From patchwork Sun Jan 27 21:47:11 2013 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Michael-Doyle Hudson X-Patchwork-Id: 14293 Return-Path: X-Original-To: patchwork@peony.canonical.com Delivered-To: patchwork@peony.canonical.com Received: from fiordland.canonical.com (fiordland.canonical.com [91.189.94.145]) by peony.canonical.com (Postfix) with ESMTP id 7A83823E57 for ; Sun, 27 Jan 2013 21:47:15 +0000 (UTC) Received: from mail-vc0-f174.google.com (mail-vc0-f174.google.com [209.85.220.174]) by fiordland.canonical.com (Postfix) with ESMTP id 18821A18A67 for ; Sun, 27 Jan 2013 21:47:15 +0000 (UTC) Received: by mail-vc0-f174.google.com with SMTP id n11so1468608vch.19 for ; Sun, 27 Jan 2013 13:47:14 -0800 (PST) X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=20120113; h=x-received:x-forwarded-to:x-forwarded-for:delivered-to:x-received :received-spf:content-type:mime-version:x-launchpad-project :x-launchpad-branch:x-launchpad-message-rationale :x-launchpad-branch-revision-number:x-launchpad-notification-type:to :from:subject:message-id:date:reply-to:sender:errors-to:precedence :x-generated-by:x-launchpad-hash:x-gm-message-state; bh=XBbu8smJ5w1ghLNUVbE/rYspe5PVTkNKc+Pf7MxgS/M=; b=noQhnZCIuk6as/p+eN2tLrJVQ1nzuU2+Nkxz+vRxVhLPrKxLuXLDF8CK9TvD3eVpWr Z8BPdxfoeRkqpliQuzwPpOlEzhARFRwNwT9afsgXvavymsmF1/ZJ9MGBTB07aBP300c+ CaGT2SKuHf0jHpY3mXfeC/pmszILgDbiMiTAdgdcHzrzKWC7EHB/X1ArF4Lyf3yCdQbx baaZBDJnH8EKhaQRY2cZ8zGA1PwoolwjEPYT0YHNG/oLbg05Kj5tQLB+1P9grK8boEJr nFBZXdo8QiWbqQViQomgAJaSso7C1S3bL5iHpdiKtA6Ryob2xtaYDFYP0xif/D+YBo8H Qhuw== X-Received: by 10.220.149.82 with SMTP id s18mr13041585vcv.14.1359323234538; Sun, 27 Jan 2013 13:47:14 -0800 (PST) X-Forwarded-To: linaro-patchwork@canonical.com X-Forwarded-For: patch@linaro.org linaro-patchwork@canonical.com Delivered-To: patches@linaro.org Received: by 10.58.145.101 with SMTP id st5csp60100veb; Sun, 27 Jan 2013 13:47:13 -0800 (PST) X-Received: by 10.180.85.103 with SMTP id g7mr6504321wiz.29.1359323233186; Sun, 27 Jan 2013 13:47:13 -0800 (PST) Received: from indium.canonical.com (indium.canonical.com. [91.189.90.7]) by mx.google.com with ESMTPS id dj8si1843912wib.69.2013.01.27.13.47.11 (version=TLSv1 cipher=RC4-SHA bits=128/128); Sun, 27 Jan 2013 13:47:13 -0800 (PST) Received-SPF: pass (google.com: best guess record for domain of bounces@canonical.com designates 91.189.90.7 as permitted sender) client-ip=91.189.90.7; Authentication-Results: mx.google.com; spf=pass (google.com: best guess record for domain of bounces@canonical.com designates 91.189.90.7 as permitted sender) smtp.mail=bounces@canonical.com Received: from ackee.canonical.com ([91.189.89.26]) by indium.canonical.com with esmtp (Exim 4.71 #1 (Debian)) id 1Tza4F-0000Bv-PR for ; Sun, 27 Jan 2013 21:47:11 +0000 Received: from ackee.canonical.com (localhost [127.0.0.1]) by ackee.canonical.com (Postfix) with ESMTP id AC848E0420 for ; Sun, 27 Jan 2013 21:47:11 +0000 (UTC) MIME-Version: 1.0 X-Launchpad-Project: lava-dispatcher X-Launchpad-Branch: ~linaro-validation/lava-dispatcher/trunk X-Launchpad-Message-Rationale: Subscriber X-Launchpad-Branch-Revision-Number: 544 X-Launchpad-Notification-Type: branch-revision To: Linaro Patch Tracker From: noreply@launchpad.net Subject: [Branch ~linaro-validation/lava-dispatcher/trunk] Rev 544: add an --output-dir argument for the dispatcher to put structured data in Message-Id: <20130127214711.1835.17407.launchpad@ackee.canonical.com> Date: Sun, 27 Jan 2013 21:47:11 -0000 Reply-To: noreply@launchpad.net Sender: bounces@canonical.com Errors-To: bounces@canonical.com Precedence: bulk X-Generated-By: Launchpad (canonical.com); Revision="16451"; Instance="launchpad-lazr.conf" X-Launchpad-Hash: 859d2f03ea8d3ced826074c574d54b30b244bca3 X-Gm-Message-State: ALoCoQlfFABp0dDva1pwS7gvN6+4R3b34aw53rivhnFyn2iB3bQC5dGnjAIRLLahd0tNWndIFJAb Merge authors: Michael Hudson-Doyle (mwhudson) Related merge proposals: https://code.launchpad.net/~mwhudson/lava-dispatcher/output-dir/+merge/144603 proposed by: Michael Hudson-Doyle (mwhudson) ------------------------------------------------------------ revno: 544 [merge] committer: Michael Hudson-Doyle branch nick: trunk timestamp: Mon 2013-01-28 10:46:16 +1300 message: add an --output-dir argument for the dispatcher to put structured data in modified: lava/dispatcher/commands.py lava_dispatcher/actions/launch_control.py lava_dispatcher/context.py lava_dispatcher/job.py --- lp:lava-dispatcher https://code.launchpad.net/~linaro-validation/lava-dispatcher/trunk You are subscribed to branch lp:lava-dispatcher. To unsubscribe from this branch go to https://code.launchpad.net/~linaro-validation/lava-dispatcher/trunk/+edit-subscription === modified file 'lava/dispatcher/commands.py' --- lava/dispatcher/commands.py 2013-01-04 14:35:51 +0000 +++ lava/dispatcher/commands.py 2013-01-23 22:28:49 +0000 @@ -53,6 +53,10 @@ type=int, help="Used internally by LAVA scheduler.") parser.add_argument( + "--output-dir", + default=None, + help="Directory to put structured output in.") + parser.add_argument( "--validate", action='store_true', help="Just validate the job file, do not execute any steps.") parser.add_argument( @@ -111,7 +115,9 @@ else: json_jobdata['target'] = self.args.target jobdata = json.dumps(json_jobdata) - job = LavaTestJob(jobdata, oob_file, config) + if self.args.output_dir and not os.path.isdir(self.args.output_dir): + os.makedirs(self.args.output_dir) + job = LavaTestJob(jobdata, oob_file, config, self.args.output_dir) #FIXME Return status if self.args.validate: === modified file 'lava_dispatcher/actions/launch_control.py' --- lava_dispatcher/actions/launch_control.py 2012-11-20 21:22:17 +0000 +++ lava_dispatcher/actions/launch_control.py 2013-01-23 22:37:05 +0000 @@ -236,6 +236,7 @@ try: result = dashboard.put_ex(json_bundle, job_name, stream) print >> self.context.oob_file, 'dashboard-put-result:', result + self.context.output.write_named_data('result-bundle', result) logging.info("Dashboard : %s" % result) except xmlrpclib.Fault, err: logging.warning("xmlrpclib.Fault occurred") === modified file 'lava_dispatcher/context.py' --- lava_dispatcher/context.py 2013-01-16 23:35:53 +0000 +++ lava_dispatcher/context.py 2013-01-23 23:10:37 +0000 @@ -19,6 +19,7 @@ # with this program; if not, see . import atexit +import logging import os import sys import tempfile @@ -29,14 +30,66 @@ from lava_dispatcher.utils import rmtree +def _write_and_flush(fobj, data): + fobj.write(data) + fobj.flush() + + +class _Forwarder(object): + """A file-like object that just forwards data written to it to a callable. + """ + + def __init__(self, callback): + self.callback = callback + + def write(self, data): + self.callback(data) + + def flush(self): + pass + + +class Outputter(object): + + def __init__(self, output_dir): + self.output_dir = output_dir + if output_dir: + self.output_txt = open(os.path.join(output_dir, 'output.txt'), 'w') + else: + self.output_txt = None + + self.logfile_read = _Forwarder(self.serial_output) + + self._log_handler = logging.StreamHandler(_Forwarder(self.log_output)) + FORMAT = '%(asctime)s %(levelname)s: %(message)s' + DATEFMT = '%Y-%m-%d %I:%M:%S %p' + self._log_handler.setFormatter( + logging.Formatter(fmt=FORMAT, datefmt=DATEFMT)) + del logging.root.handlers[:] + del logging.root.filters[:] + logging.root.addHandler(self._log_handler) + + def serial_output(self, data): + _write_and_flush(sys.stdout, data) + if self.output_txt is not None: + _write_and_flush(self.output_txt, data) + + # Currently all output is treated the same way. + log_output = serial_output + + def write_named_data(self, name, data): + if self.output_dir is None: + return + with open(os.path.join(self.output_dir, name), 'w') as outf: + outf.write(data) + + class LavaContext(object): - def __init__(self, target, dispatcher_config, oob_file, job_data): + def __init__(self, target, dispatcher_config, oob_file, job_data, output_dir): self.config = dispatcher_config self.job_data = job_data - # This is the file-like object to send serial output from the device - # to. We just send it to stdout for now, but soon we'll do something - # cleverer. - self.logfile_read = sys.stdout + self.output = Outputter(output_dir) + self.logfile_read = self.output.logfile_read device_config = get_device_config( target, dispatcher_config.config_dir) self._client = TargetBasedClient(self, device_config) === modified file 'lava_dispatcher/job.py' --- lava_dispatcher/job.py 2013-01-16 23:27:28 +0000 +++ lava_dispatcher/job.py 2013-01-23 22:37:53 +0000 @@ -111,11 +111,11 @@ class LavaTestJob(object): - def __init__(self, job_json, oob_file, config): + def __init__(self, job_json, oob_file, config, output_dir): self.job_status = 'pass' self.load_job_data(job_json) self.context = LavaContext( - self.target, config, oob_file, self.job_data) + self.target, config, oob_file, self.job_data, output_dir) def load_job_data(self, job_json): self.job_data = json.loads(job_json)