From patchwork Sun Feb 26 22:56:09 2012 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Michael-Doyle Hudson X-Patchwork-Id: 6938 Return-Path: X-Original-To: patchwork@peony.canonical.com Delivered-To: patchwork@peony.canonical.com Received: from fiordland.canonical.com (fiordland.canonical.com [91.189.94.145]) by peony.canonical.com (Postfix) with ESMTP id 4543A23EA9 for ; Sun, 26 Feb 2012 22:56:14 +0000 (UTC) Received: from mail-iy0-f180.google.com (mail-iy0-f180.google.com [209.85.210.180]) by fiordland.canonical.com (Postfix) with ESMTP id E39BFA18599 for ; Sun, 26 Feb 2012 22:56:13 +0000 (UTC) Received: by iabz7 with SMTP id z7so7466818iab.11 for ; Sun, 26 Feb 2012 14:56:13 -0800 (PST) Received: from mr.google.com ([10.42.131.129]) by 10.42.131.129 with SMTP id z1mr11405202ics.53.1330296973318 (num_hops = 1); Sun, 26 Feb 2012 14:56:13 -0800 (PST) Received: by 10.42.131.129 with SMTP id z1mr9225343ics.53.1330296973227; Sun, 26 Feb 2012 14:56:13 -0800 (PST) X-Forwarded-To: linaro-patchwork@canonical.com X-Forwarded-For: patch@linaro.org linaro-patchwork@canonical.com Delivered-To: patches@linaro.org Received: by 10.231.11.10 with SMTP id r10csp53884ibr; Sun, 26 Feb 2012 14:56:11 -0800 (PST) Received: by 10.216.138.34 with SMTP id z34mr4481927wei.27.1330296970502; Sun, 26 Feb 2012 14:56:10 -0800 (PST) Received: from indium.canonical.com (indium.canonical.com. [91.189.90.7]) by mx.google.com with ESMTPS id n55si9657092weq.16.2012.02.26.14.56.10 (version=TLSv1/SSLv3 cipher=OTHER); Sun, 26 Feb 2012 14:56:10 -0800 (PST) Received-SPF: pass (google.com: best guess record for domain of bounces@canonical.com designates 91.189.90.7 as permitted sender) client-ip=91.189.90.7; Authentication-Results: mx.google.com; spf=pass (google.com: best guess record for domain of bounces@canonical.com designates 91.189.90.7 as permitted sender) smtp.mail=bounces@canonical.com Received: from ackee.canonical.com ([91.189.89.26]) by indium.canonical.com with esmtp (Exim 4.71 #1 (Debian)) id 1S1n0j-0007Bc-Uj for ; Sun, 26 Feb 2012 22:56:09 +0000 Received: from ackee.canonical.com (localhost [127.0.0.1]) by ackee.canonical.com (Postfix) with ESMTP id DE04AE0246 for ; Sun, 26 Feb 2012 22:56:09 +0000 (UTC) MIME-Version: 1.0 X-Launchpad-Project: lava-dispatcher X-Launchpad-Branch: ~linaro-validation/lava-dispatcher/trunk X-Launchpad-Message-Rationale: Subscriber X-Launchpad-Branch-Revision-Number: 235 X-Launchpad-Notification-Type: branch-revision To: Linaro Patch Tracker From: noreply@launchpad.net Subject: [Branch ~linaro-validation/lava-dispatcher/trunk] Rev 235: combine the two submit_results actions into one, leaving a compatibility shim behind Message-Id: <20120226225609.26422.30101.launchpad@ackee.canonical.com> Date: Sun, 26 Feb 2012 22:56:09 -0000 Reply-To: noreply@launchpad.net Sender: bounces@canonical.com Errors-To: bounces@canonical.com Precedence: bulk X-Generated-By: Launchpad (canonical.com); Revision="14860"; Instance="launchpad-lazr.conf" X-Launchpad-Hash: 39a3af0ae689252504c3fc82543fa700ab540fea X-Gm-Message-State: ALoCoQlVO0Rb4qjJWBAu+UoibERcycEVTN9lsaV4axw+JRdb4klhFZ/BghUtsBdTyPN1/8lZvX6Z Merge authors: Michael Hudson-Doyle (mwhudson) Related merge proposals: https://code.launchpad.net/~mwhudson/lava-dispatcher/one-submit-results-to-rule-them-all/+merge/94498 proposed by: Michael Hudson-Doyle (mwhudson) review: Approve - Spring Zhang (qzhang) ------------------------------------------------------------ revno: 235 [merge] committer: Michael Hudson-Doyle branch nick: trunk timestamp: Mon 2012-02-27 11:54:40 +1300 message: combine the two submit_results actions into one, leaving a compatibility shim behind modified: lava_dispatcher/actions/launch_control.py lava_dispatcher/actions/lava-test.py lava_dispatcher/client/master.py lava_dispatcher/client/qemu.py lava_dispatcher/context.py --- lp:lava-dispatcher https://code.launchpad.net/~linaro-validation/lava-dispatcher/trunk You are subscribed to branch lp:lava-dispatcher. To unsubscribe from this branch go to https://code.launchpad.net/~linaro-validation/lava-dispatcher/trunk/+edit-subscription === modified file 'lava_dispatcher/actions/launch_control.py' --- lava_dispatcher/actions/launch_control.py 2012-02-07 19:16:17 +0000 +++ lava_dispatcher/actions/launch_control.py 2012-02-24 02:13:45 +0000 @@ -33,113 +33,6 @@ import xmlrpclib import traceback -class SubmitResultAction(BaseAction): - all_bundles = [] - def combine_bundles(self): - if not self.all_bundles: - return { - "test_runs": [], - "format": "Dashboard Bundle Format 1.2" - } - main_bundle = self.all_bundles.pop(0) - test_runs = main_bundle['test_runs'] - for bundle in self.all_bundles: - test_runs += bundle['test_runs'] - return main_bundle - - def submit_combine_bundles(self, status='pass', err_msg='', server=None, stream=None, token=None): - dashboard = _get_dashboard(server, token) - main_bundle = self.combine_bundles() - self.context.test_data.add_seriallog( - self.context.client.get_seriallog()) - # add gather_results result - self.context.test_data.add_result('gather_results', status, err_msg) - main_bundle['test_runs'].append(self.context.test_data.get_test_run()) - for test_run in main_bundle['test_runs']: - attributes = test_run.get('attributes', {}) - attributes.update(self.context.test_data.get_metadata()) - test_run['attributes'] = attributes - json_bundle = json.dumps(main_bundle) - job_name = self.context.job_data.get('job_name', "LAVA Results") - try: - result = dashboard.put_ex(json_bundle, job_name, stream) - print >> self.context.oob_file, 'dashboard-put-result:', result - logging.info("Dashboard : %s" %result) - except xmlrpclib.Fault, err: - logging.warning("xmlrpclib.Fault occurred") - logging.warning("Fault code: %d" % err.faultCode) - logging.warning("Fault string: %s" % err.faultString) - -class cmd_submit_results_on_host(SubmitResultAction): - def run(self, server, stream, token=None): - #Upload bundle files to dashboard - logging.info("Executing submit_results_on_host command") - bundlename_list = [] - status = 'pass' - err_msg = '' - try: - bundle_list = os.listdir(self.context.host_result_dir) - for bundle_name in bundle_list: - bundle = "%s/%s" % (self.context.host_result_dir, bundle_name) - bundlename_list.append(bundle) - f = open(bundle) - content = f.read() - f.close() - self.all_bundles.append(json.loads(content)) - except: - print traceback.format_exc() - status = 'fail' - err_msg = err_msg + " Some test case result appending failed." - - self.submit_combine_bundles(status, err_msg, server, stream, token) - - for bundle in bundlename_list: - os.remove(bundle) - shutil.rmtree(self.context.host_result_dir) - if status == 'fail': - raise OperationFailed(err_msg) - - -class cmd_submit_results(SubmitResultAction): - - def run(self, server, stream, result_disk="testrootfs", token=None): - """Submit test results to a lava-dashboard server - :param server: URL of the lava-dashboard server RPC endpoint - :param stream: Stream on the lava-dashboard server to save the result to - """ - - err_msg = None - status = 'fail' - try: - status, err_msg, result_path = self.client.retrieve_results(result_disk) - if result_path is not None: - try: - tar = tarfile.open(result_path) - for tarinfo in tar: - if os.path.splitext(tarinfo.name)[1] == ".bundle": - f = tar.extractfile(tarinfo) - content = f.read() - f.close() - self.all_bundles.append(json.loads(content)) - tar.close() - except: - logging.warning(traceback.format_exc()) - status = 'fail' - err_msg = err_msg + " Some test case result appending failed." - logging.warning(err_msg) - finally: - shutil.rmtree(os.path.dirname(result_path)) - except: - logging.exception('retrieve_results failed') - - if err_msg is None: - err_msg = '' - - self.submit_combine_bundles(status, err_msg, server, stream, token) - if status == 'fail': - raise OperationFailed(err_msg) - -#util function, see if it needs to be part of utils.py def _get_dashboard(server, token): if not server.endswith("/"): server = ''.join([server, "/"]) @@ -180,3 +73,114 @@ logging.debug("server RPC endpoint URL: %s" % server) return dashboard + + +class cmd_submit_results(BaseAction): + + def _get_bundles_from_device(self, result_disk): + err_msg = '' + status = 'fail' + device_bundles = [] + try: + status, err_msg, result_path = self.client.retrieve_results( + result_disk) + if result_path is not None: + try: + tar = tarfile.open(result_path) + for tarinfo in tar: + if os.path.splitext(tarinfo.name)[1] == ".bundle": + f = tar.extractfile(tarinfo) + content = f.read() + f.close() + device_bundles.append(json.loads(content)) + tar.close() + except: + logging.warning(traceback.format_exc()) + status = 'fail' + err_msg = err_msg + " Some test case result appending failed." + logging.warning(err_msg) + finally: + shutil.rmtree(os.path.dirname(result_path)) + except: + logging.exception('retrieve_results failed') + return device_bundles, status, err_msg + + def _get_results_from_host(self): + status = 'pass' + err_msg = '' + host_bundles = [] + try: + bundle_list = os.listdir(self.context.host_result_dir) + for bundle_name in bundle_list: + bundle = "%s/%s" % (self.context.host_result_dir, bundle_name) + f = open(bundle) + content = f.read() + f.close() + host_bundles.append(json.loads(content)) + except: + print traceback.format_exc() + status = 'fail' + err_msg = err_msg + " Some test case result appending failed." + return host_bundles, status, err_msg + + + def run(self, server, stream, result_disk="testrootfs", token=None): + all_bundles = [] + status = 'pass' + err_msg = '' + if self.context.any_device_bundles: + device_bundles, status, err_msg = self._get_bundles_from_device(result_disk) + all_bundles.extend(device_bundles) + if self.context.any_host_bundles: + host_bundles, host_status, host_err_msg = self._get_results_from_host() + all_bundles.extend(host_bundles) + if status == 'pass': + status = host_status + err_msg += host_err_msg + + self.context.test_data.add_result('gather_results', status, err_msg) + + main_bundle = self.combine_bundles(all_bundles) + + self.submit_bundle(main_bundle, server, stream, token) + + def combine_bundles(self, all_bundles): + if not all_bundles: + main_bundle = { + "test_runs": [], + "format": "Dashboard Bundle Format 1.2" + } + else: + main_bundle = all_bundles.pop(0) + test_runs = main_bundle['test_runs'] + for bundle in all_bundles: + test_runs += bundle['test_runs'] + + self.context.test_data.add_seriallog( + self.context.client.get_seriallog()) + + main_bundle['test_runs'].append(self.context.test_data.get_test_run()) + + for test_run in main_bundle['test_runs']: + attributes = test_run.get('attributes', {}) + attributes.update(self.context.test_data.get_metadata()) + test_run['attributes'] = attributes + + return main_bundle + + def submit_bundle(self, main_bundle, server, stream, token): + dashboard = _get_dashboard(server, token) + json_bundle = json.dumps(main_bundle) + job_name = self.context.job_data.get('job_name', "LAVA Results") + try: + result = dashboard.put_ex(json_bundle, job_name, stream) + print >> self.context.oob_file, 'dashboard-put-result:', result + logging.info("Dashboard : %s" %result) + except xmlrpclib.Fault, err: + logging.warning("xmlrpclib.Fault occurred") + logging.warning("Fault code: %d" % err.faultCode) + logging.warning("Fault string: %s" % err.faultString) + raise OperationFailed("could not push to dashboard") + +class cmd_submit_results_on_host(cmd_submit_results): + pass === modified file 'lava_dispatcher/actions/lava-test.py' --- lava_dispatcher/actions/lava-test.py 2012-02-07 19:16:17 +0000 +++ lava_dispatcher/actions/lava-test.py 2012-02-24 00:34:24 +0000 @@ -22,7 +22,6 @@ from datetime import datetime import logging -import traceback from lava_dispatcher.actions import BaseAction from lava_dispatcher.client.base import OperationFailed, CriticalError @@ -76,6 +75,7 @@ logging.exception("killing test failed, rebooting") self.client.boot_linaro_image() raise + self.context.any_device_bundles = True if rc is None: raise OperationFailed("test case getting return value failed") elif rc != 0: === modified file 'lava_dispatcher/client/master.py' --- lava_dispatcher/client/master.py 2012-02-26 22:17:35 +0000 +++ lava_dispatcher/client/master.py 2012-02-26 22:17:45 +0000 @@ -499,7 +499,7 @@ master_ip = session.get_master_ip() if not master_ip: err_msg = (err_msg + "Getting master image IP address failed, " - "no test case result retrived.") + "no test case result retrieved.") logging.warning(err_msg) return 'fail', err_msg, None # Set 80 as server port @@ -522,7 +522,7 @@ try: result_path = download( result_tarball, tarball_dir,False) - return 'pass', None, result_path + return 'pass', '', result_path except RuntimeError: tries += 1 if time.time() >= now + timeout: === modified file 'lava_dispatcher/client/qemu.py' --- lava_dispatcher/client/qemu.py 2012-02-09 20:29:52 +0000 +++ lava_dispatcher/client/qemu.py 2012-02-26 20:45:59 +0000 @@ -124,4 +124,4 @@ 'tar czf %s -C %s%s .' % ( tarfile, mntdir, self.context.lava_result_dir)) logging_system('rm %s%s/*.bundle' % (mntdir, self.context.lava_result_dir)) - return 'pass', None, tarfile + return 'pass', '', tarfile === modified file 'lava_dispatcher/context.py' --- lava_dispatcher/context.py 2011-12-07 01:43:24 +0000 +++ lava_dispatcher/context.py 2012-02-26 19:59:04 +0000 @@ -18,6 +18,7 @@ # along # with this program; if not, see . +import os import tempfile from lava_dispatcher.config import get_device_config @@ -44,6 +45,7 @@ self.test_data = LavaTestData() self.oob_file = oob_file self._host_result_dir = None + self.any_device_bundles = False @property def client(self): @@ -62,6 +64,11 @@ return self.config.get("LAVA_IMAGE_URL") @property + def any_host_bundles(self): + return (self._host_result_dir is not None + and len(os.listdir(self._host_result_dir)) > 0) + + @property def host_result_dir(self): if self._host_result_dir is None: self._host_result_dir = tempfile.mkdtemp()