From patchwork Tue Aug 16 00:17:12 2011 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Paul Larson X-Patchwork-Id: 3460 Return-Path: X-Original-To: patchwork@peony.canonical.com Delivered-To: patchwork@peony.canonical.com Received: from fiordland.canonical.com (fiordland.canonical.com [91.189.94.145]) by peony.canonical.com (Postfix) with ESMTP id C472823F4F for ; Tue, 16 Aug 2011 00:17:16 +0000 (UTC) Received: from mail-ey0-f170.google.com (mail-ey0-f170.google.com [209.85.215.170]) by fiordland.canonical.com (Postfix) with ESMTP id 9F4E3A18409 for ; Tue, 16 Aug 2011 00:17:16 +0000 (UTC) Received: by eyd10 with SMTP id 10so4297703eyd.29 for ; Mon, 15 Aug 2011 17:17:16 -0700 (PDT) Received: by 10.213.29.147 with SMTP id q19mr662353ebc.132.1313453834941; Mon, 15 Aug 2011 17:17:14 -0700 (PDT) X-Forwarded-To: linaro-patchwork@canonical.com X-Forwarded-For: patch@linaro.org linaro-patchwork@canonical.com Delivered-To: patches@linaro.org Received: by 10.213.102.5 with SMTP id e5cs127009ebo; Mon, 15 Aug 2011 17:17:14 -0700 (PDT) Received: from mr.google.com ([10.227.163.137]) by 10.227.163.137 with SMTP id a9mr5310777wby.74.1313453834323 (num_hops = 1); Mon, 15 Aug 2011 17:17:14 -0700 (PDT) Received: by 10.227.163.137 with SMTP id a9mr3845187wby.74.1313453833760; Mon, 15 Aug 2011 17:17:13 -0700 (PDT) Received: from indium.canonical.com (indium.canonical.com [91.189.90.7]) by mx.google.com with ESMTPS id fe3si17193042wbb.101.2011.08.15.17.17.13 (version=TLSv1/SSLv3 cipher=OTHER); Mon, 15 Aug 2011 17:17:13 -0700 (PDT) Received-SPF: pass (google.com: best guess record for domain of bounces@canonical.com designates 91.189.90.7 as permitted sender) client-ip=91.189.90.7; Authentication-Results: mx.google.com; spf=pass (google.com: best guess record for domain of bounces@canonical.com designates 91.189.90.7 as permitted sender) smtp.mail=bounces@canonical.com Received: from ackee.canonical.com ([91.189.89.26]) by indium.canonical.com with esmtp (Exim 4.71 #1 (Debian)) id 1Qt7LF-0003TZ-17 for ; Tue, 16 Aug 2011 00:17:13 +0000 Received: from ackee.canonical.com (localhost [127.0.0.1]) by ackee.canonical.com (Postfix) with ESMTP id ED8C5E0108 for ; Tue, 16 Aug 2011 00:17:12 +0000 (UTC) MIME-Version: 1.0 X-Launchpad-Project: lava-dispatcher X-Launchpad-Branch: ~linaro-validation/lava-dispatcher/trunk X-Launchpad-Message-Rationale: Subscriber X-Launchpad-Branch-Revision-Number: 84 X-Launchpad-Notification-Type: branch-revision To: Linaro Patch Tracker From: noreply@launchpad.net Subject: [Branch ~linaro-validation/lava-dispatcher/trunk] Rev 84: Improved transferring of results from Spring Zhang Message-Id: <20110816001712.14026.39221.launchpad@ackee.canonical.com> Date: Tue, 16 Aug 2011 00:17:12 -0000 Reply-To: noreply@launchpad.net Sender: bounces@canonical.com Errors-To: bounces@canonical.com Precedence: bulk X-Generated-By: Launchpad (canonical.com); Revision="13687"; Instance="initZopeless config overlay" X-Launchpad-Hash: bcc0bb84e8154df5059682ee7fa0abc2ce1ab5c1 Merge authors: Spring Zhang (qzhang) Related merge proposals: https://code.launchpad.net/~qzhang/lava-dispatcher/reliable-result-transfer/+merge/71341 proposed by: Spring Zhang (qzhang) review: Approve - Paul Larson (pwlars) review: Resubmit - Spring Zhang (qzhang) ------------------------------------------------------------ revno: 84 [merge] committer: Paul Larson branch nick: lava-dispatcher timestamp: Tue 2011-08-16 01:15:08 +0100 message: Improved transferring of results from Spring Zhang modified: lava_dispatcher/actions/__init__.py lava_dispatcher/actions/launch_control.py lava_dispatcher/android_client.py lava_dispatcher/android_config.py lava_dispatcher/client.py lava_dispatcher/config.py lava_dispatcher/utils.py --- lp:lava-dispatcher https://code.launchpad.net/~linaro-validation/lava-dispatcher/trunk You are subscribed to branch lp:lava-dispatcher. To unsubscribe from this branch go to https://code.launchpad.net/~linaro-validation/lava-dispatcher/trunk/+edit-subscription === modified file 'lava_dispatcher/actions/__init__.py' --- lava_dispatcher/actions/__init__.py 2011-06-23 19:38:42 +0000 +++ lava_dispatcher/actions/__init__.py 2011-08-16 00:15:08 +0000 @@ -34,8 +34,6 @@ class BaseAndroidAction(BaseAction): - network_interface = "eth0" - def __init__(self, context): self.context = context === modified file 'lava_dispatcher/actions/launch_control.py' --- lava_dispatcher/actions/launch_control.py 2011-06-27 04:55:08 +0000 +++ lava_dispatcher/actions/launch_control.py 2011-08-16 00:15:08 +0000 @@ -20,11 +20,15 @@ # along # with this program; if not, see . +import os import json +import shutil +import tarfile from lava_dispatcher.actions import BaseAction from lava_dispatcher.config import LAVA_RESULT_DIR, MASTER_STR, LAVA_SERVER_IP -import socket -from threading import Thread +from lava_dispatcher.config import LAVA_IMAGE_TMPDIR +from lava_dispatcher.client import NetworkError +from lava_dispatcher.utils import download import time import xmlrpclib from subprocess import call @@ -35,34 +39,21 @@ srv = xmlrpclib.ServerProxy(xmlrpc_url, allow_none=True, use_datetime=True) - client = self.client - call("cd /tmp/%s/; ls *.bundle > bundle.lst" % LAVA_RESULT_DIR, - shell=True) - - t = ResultUploader() - t.start() - call('cd /tmp/%s/; cat bundle.lst |nc %s %d' % (LAVA_RESULT_DIR, - LAVA_SERVER_IP, t.get_port()), shell=True) - t.join() - - bundle_list = t.get_data().strip().splitlines() - #Upload bundle files to server - for bundle in bundle_list: - t = ResultUploader() - t.start() - call('cat /tmp/%s/%s | nc %s %s' % (LAVA_RESULT_DIR, bundle, - LAVA_SERVER_IP, t.get_port()), shell = True) - t.join() - content = t.get_data() + #Upload bundle files to dashboard + bundle_list = os.listdir("/tmp/%s" % LAVA_RESULT_DIR) + for bundle_name in bundle_list: + bundle = "/tmp/%s/%s" % (LAVA_RESULT_DIR, bundle_name) + f = open(bundle) + content = f.read() + f.close() try: srv.put(content, bundle, stream) except xmlrpclib.Fault, err: print "xmlrpclib.Fault occurred" print "Fault code: %d" % err.faultCode print "Fault string: %s" % err.faultString - # After uploading, remove the bundle file at the host side - call('rm /tmp/%s/%s' % (LAVA_RESULT_DIR, bundle), shell=True) + os.remove(bundle) class cmd_submit_results(BaseAction): @@ -96,43 +87,41 @@ LAVA_RESULT_DIR), response = MASTER_STR) client.run_shell_command('umount /mnt/root', response = MASTER_STR) - #Upload bundle list-bundle.lst - client.run_shell_command('cd /tmp/%s' % LAVA_RESULT_DIR, - response = MASTER_STR) - client.run_shell_command('ls *.bundle > bundle.lst', - response = MASTER_STR) - - t = ResultUploader() - t.start() - #XXX: Odd problem where we sometimes get stuck here. This is just - # a hacky workaround to see if it's a race - time.sleep(60) - client.run_shell_command( - 'cat bundle.lst |nc %s %d' % (LAVA_SERVER_IP, t.get_port()), - response = MASTER_STR) - t.join() - - bundle_list = t.get_data().strip().splitlines() - + #Create tarball of all results + client.run_shell_command('cd /tmp', response=MASTER_STR) + client.run_shell_command('tar czf /tmp/lava_results.tgz -C /tmp/%s .' + % LAVA_RESULT_DIR, response=MASTER_STR) + + master_ip = client.get_master_ip() + if master_ip == None: + raise NetworkError("Getting master image IP address failed") + # Set 80 as server port + client.run_shell_command('python -m SimpleHTTPServer 80 &> /dev/null &', + response=MASTER_STR) + time.sleep(3) + + result_tarball = "http://%s/lava_results.tgz" % master_ip + tarball_dir = mkdtemp(dir=LAVA_IMAGE_TMPDIR) + os.chmod(tarball_dir, 0755) + #FIXME: need to consider exception? + result_path = download(result_tarball, tarball_dir) + id = client.proc.expect([MASTER_STR, pexpect.TIMEOUT, pexpect.EOF], + timeout=3) + client.run_shell_command('kill %1', response=MASTER_STR) + + tar = tarfile.open(result_path) + for tarinfo in tar: + if os.path.splitext(tarinfo.name)[1] == ".bundle": + f = tar.extractfile(tarinfo) + content = f.read() + f.close() + self.all_bundles.append(json.loads(content)) + tar.close() + shutil.rmtree(tarball_dir) + #flush the serial log client.run_shell_command("") - #Upload bundle files to server - for bundle in bundle_list: - t = ResultUploader() - t.start() - #XXX: Odd problem where we sometimes get stuck here. This is just - # a hacky workaround to see if it's a race - time.sleep(60) - client.run_shell_command( - 'cat /tmp/%s/%s | nc %s %s' % (LAVA_RESULT_DIR, bundle, - LAVA_SERVER_IP, t.get_port()), - response = MASTER_STR) - t.join() - content = t.get_data() - - self.all_bundles.append(json.loads(content)) - main_bundle = self.combine_bundles() self.context.test_data.add_seriallog( self.context.client.get_seriallog()) @@ -156,32 +145,3 @@ test_runs += bundle['test_runs'] return main_bundle -class ResultUploader(Thread): - """ - Simple HTTP Server for uploading bundles - """ - def __init__(self): - """ - if no filename specified, just get uploaded data - """ - Thread.__init__(self) - self.data = "" - self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - self.s.bind(('', 0)) - - def get_port(self): - return self.s.getsockname()[1] - - def get_data(self): - return self.data - - def run(self): - self.s.listen(1) - conn, addr = self.s.accept() - while(1): - #10KB per time - data = conn.recv(10240) - if not data: - break - self.data = self.data + data - self.s.close() === modified file 'lava_dispatcher/android_client.py' --- lava_dispatcher/android_client.py 2011-07-20 04:45:23 +0000 +++ lava_dispatcher/android_client.py 2011-08-16 00:15:08 +0000 @@ -106,17 +106,18 @@ def check_adb_status(self): # XXX: IP could be assigned in other way in the validation farm - network_interface = self.board.network_interface + network_interface = self.board.default_network_interface try: self.run_shell_command('netcfg %s dhcp' % \ - network_interface, response = TESTER_STR, timeout = 60) + default_network_interface, response = TESTER_STR, + timeout = 60) except: - print "netcfg %s dhcp exception" % network_interface + print "netcfg %s dhcp exception" % default_network_interface return False # Check network ip and setup adb connection ip_pattern = "(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})" - cmd = "ifconfig %s" % network_interface + cmd = "ifconfig %s" % default_network_interface self.proc.sendline('') self.proc.sendline(cmd) try: === modified file 'lava_dispatcher/android_config.py' --- lava_dispatcher/android_config.py 2011-07-21 16:47:16 +0000 +++ lava_dispatcher/android_config.py 2011-08-16 00:15:08 +0000 @@ -31,7 +31,6 @@ "init=/init androidboot.console=ttyO2'", "boot"] type = "beagle" - network_interface = "eth0" class PandaBoard(Board): @@ -47,7 +46,6 @@ "init=/init androidboot.console=ttyO2'", "boot"] type = "panda" - network_interface = "eth0" #Here, it still needs to maintain a map from boardid to board, for there is only === modified file 'lava_dispatcher/client.py' --- lava_dispatcher/client.py 2011-07-21 16:47:16 +0000 +++ lava_dispatcher/client.py 2011-08-16 00:15:08 +0000 @@ -126,6 +126,25 @@ return raise NetworkError + def get_master_ip(self): + #get master image ip address + #tty device uses minimal match, see pexpect wiki + #pattern1 = ".*\n(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})" + pattern1 = "(\d?\d?\d?\.\d?\d?\d?\.\d?\d?\d?\.\d?\d?\d?)" + cmd = ("ifconfig %s | grep 'inet addr' | awk -F: '{print $2}' |" + "awk '{print $1}'" % self.board.default_network_interface) + self.proc.sendline(cmd) + #if running from ipython, it needs another Enter, don't know why + #self.proc.sendline("") + id = self.proc.expect([pattern1, pexpect.EOF, + pexpect.TIMEOUT], timeout=5) + print "\nid=%s" %id + if id == 0: + ip = self.proc.match.groups()[0] + return ip + else: + return None + def export_display(self): #export the display, ignore errors on non-graphical images self.run_shell_command("su - linaro -c 'DISPLAY=:0 xhost local:'", === modified file 'lava_dispatcher/config.py' --- lava_dispatcher/config.py 2011-07-21 16:47:16 +0000 +++ lava_dispatcher/config.py 2011-08-16 00:15:08 +0000 @@ -32,6 +32,7 @@ boot_part = 1 # root partition number, counting from 1 root_part = 2 + default_network_interface = "eth0" class BeagleBoard(Board): uboot_cmds = ["mmc init", === modified file 'lava_dispatcher/utils.py' --- lava_dispatcher/utils.py 2011-06-27 04:55:08 +0000 +++ lava_dispatcher/utils.py 2011-08-15 10:44:10 +0000 @@ -32,7 +32,7 @@ filename = os.path.join(path,filename) fd = open(filename, "w") try: - response = urllib2.urlopen(urllib2.quote(url, safe=":/")) + response = urllib2.urlopen(urllib2.quote(url, safe=":/"), timeout=5) fd = open(filename, 'wb') shutil.copyfileobj(response,fd,0x10000) fd.close()