From patchwork Sun Jul 22 20:01:10 2012 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Andy Doan X-Patchwork-Id: 10165 Return-Path: X-Original-To: patchwork@peony.canonical.com Delivered-To: patchwork@peony.canonical.com Received: from fiordland.canonical.com (fiordland.canonical.com [91.189.94.145]) by peony.canonical.com (Postfix) with ESMTP id 015112402A for ; Sun, 22 Jul 2012 20:01:14 +0000 (UTC) Received: from mail-gh0-f180.google.com (mail-gh0-f180.google.com [209.85.160.180]) by fiordland.canonical.com (Postfix) with ESMTP id 9B1BFA1965E for ; Sun, 22 Jul 2012 20:01:14 +0000 (UTC) Received: by ghbz12 with SMTP id z12so5277353ghb.11 for ; Sun, 22 Jul 2012 13:01:14 -0700 (PDT) X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=20120113; h=x-forwarded-to:x-forwarded-for:delivered-to:received-spf :content-type:mime-version:x-launchpad-project:x-launchpad-branch :x-launchpad-message-rationale:x-launchpad-branch-revision-number :x-launchpad-notification-type:to:from:subject:message-id:date :reply-to:sender:errors-to:precedence:x-generated-by :x-launchpad-hash:x-gm-message-state; bh=AgRUrMtKmgMt49MEURjOGcP2PsY3/4HRywjgeei+j3Q=; b=nOvk8917ZaRVm3W7q6n5b7Yw1AdIXxbamqw3IJQC+p7chTk2VHhjoJ/+YLf6bmMg/G wPCjRJ1RjjEJh1v27LFliLfg/G3lWzsVL1hBCIRUkQNq6aGpnsmuLIl8J4pCiiQh/Qp/ qVWBhwhVxZWuPhBpaNQRD33jNy1GtqO///wDjZOPlhgW//4xdW5KI0ns/Fm+KuLVFoWr m/htohfG+klxX8hxn8r/QonXij11HjWLXAcYbFew826W78cPQBW8M1ai0SsuCR0E/Vno /WATZnJLByBK038w/BDMZsUjtl67f/AhwAaWWqFulHVUiJdmJbHC59P5n/QDhXfWOC7f VQPg== Received: by 10.50.207.65 with SMTP id lu1mr8914154igc.33.1342987273570; Sun, 22 Jul 2012 13:01:13 -0700 (PDT) X-Forwarded-To: linaro-patchwork@canonical.com X-Forwarded-For: patch@linaro.org linaro-patchwork@canonical.com Delivered-To: patches@linaro.org Received: by 10.231.153.7 with SMTP id i7csp48109ibw; Sun, 22 Jul 2012 13:01:12 -0700 (PDT) Received: by 10.180.78.5 with SMTP id x5mr13719967wiw.13.1342987271933; Sun, 22 Jul 2012 13:01:11 -0700 (PDT) Received: from indium.canonical.com (indium.canonical.com. [91.189.90.7]) by mx.google.com with ESMTPS id u15si7587782wek.51.2012.07.22.13.01.10 (version=TLSv1/SSLv3 cipher=OTHER); Sun, 22 Jul 2012 13:01:11 -0700 (PDT) Received-SPF: pass (google.com: best guess record for domain of bounces@canonical.com designates 91.189.90.7 as permitted sender) client-ip=91.189.90.7; Authentication-Results: mx.google.com; spf=pass (google.com: best guess record for domain of bounces@canonical.com designates 91.189.90.7 as permitted sender) smtp.mail=bounces@canonical.com Received: from ackee.canonical.com ([91.189.89.26]) by indium.canonical.com with esmtp (Exim 4.71 #1 (Debian)) id 1St2L0-0002Yy-6M for ; Sun, 22 Jul 2012 20:01:10 +0000 Received: from ackee.canonical.com (localhost [127.0.0.1]) by ackee.canonical.com (Postfix) with ESMTP id 206E2E00A9 for ; Sun, 22 Jul 2012 20:01:10 +0000 (UTC) MIME-Version: 1.0 X-Launchpad-Project: lava-dispatcher X-Launchpad-Branch: ~linaro-validation/lava-dispatcher/trunk X-Launchpad-Message-Rationale: Subscriber X-Launchpad-Branch-Revision-Number: 354 X-Launchpad-Notification-Type: branch-revision To: Linaro Patch Tracker From: noreply@launchpad.net Subject: [Branch ~linaro-validation/lava-dispatcher/trunk] Rev 354: fix remaining bugs with temp dirs and download logic Message-Id: <20120722200110.6776.42810.launchpad@ackee.canonical.com> Date: Sun, 22 Jul 2012 20:01:10 -0000 Reply-To: noreply@launchpad.net Sender: bounces@canonical.com Errors-To: bounces@canonical.com Precedence: bulk X-Generated-By: Launchpad (canonical.com); Revision="15654"; Instance="launchpad-lazr.conf" X-Launchpad-Hash: c0259f47ba0cdec698c88511638a137f30566ef6 X-Gm-Message-State: ALoCoQmk4MhhDCbYehbQdYMMOrNpFDLN+6Ble04KmJOPEbIrENsOj/NCuj4sONiMU6qf23nNuHpJ Merge authors: Andy Doan (doanac) Related merge proposals: https://code.launchpad.net/~doanac/lava-dispatcher/tempdir-madness/+merge/116141 proposed by: Andy Doan (doanac) review: Approve - Dave Pigott (dpigott) ------------------------------------------------------------ revno: 354 [merge] committer: Andy Doan branch nick: lava-dispatcher timestamp: Sun 2012-07-22 14:57:32 -0500 message: fix remaining bugs with temp dirs and download logic modified: lava_dispatcher/client/base.py lava_dispatcher/client/lmc_utils.py lava_dispatcher/client/master.py lava_dispatcher/client/qemu.py lava_dispatcher/utils.py --- lp:lava-dispatcher https://code.launchpad.net/~linaro-validation/lava-dispatcher/trunk You are subscribed to branch lp:lava-dispatcher. To unsubscribe from this branch go to https://code.launchpad.net/~linaro-validation/lava-dispatcher/trunk/+edit-subscription === modified file 'lava_dispatcher/client/base.py' --- lava_dispatcher/client/base.py 2012-07-17 20:02:17 +0000 +++ lava_dispatcher/client/base.py 2012-07-21 22:56:42 +0000 @@ -19,14 +19,19 @@ # along # with this program; if not, see . +import atexit import commands import contextlib +import logging +import os import pexpect +import shutil import sys import time +import traceback + from cStringIO import StringIO -import traceback -import logging +from tempfile import mkdtemp class CommandRunner(object): @@ -424,6 +429,15 @@ def get_seriallog(self): return self.sio.getvalue() + def get_www_scratch_dir(self): + ''' returns a temporary directory available for downloads that's gets + deleted when the process exits ''' + + d = mkdtemp(dir=self.context.lava_image_tmpdir) + atexit.register(shutil.rmtree, d) + os.chmod(d, 0755) + return d + # Android stuff def get_android_adb_interface(self): === modified file 'lava_dispatcher/client/lmc_utils.py' --- lava_dispatcher/client/lmc_utils.py 2012-06-29 01:26:45 +0000 +++ lava_dispatcher/client/lmc_utils.py 2012-07-22 00:43:26 +0000 @@ -9,14 +9,14 @@ import sys from lava_dispatcher.client.base import CriticalError +from lava_dispatcher.downloader import ( + download_image, + ) from lava_dispatcher.utils import ( - download, logging_system, ) -def refresh_hwpack(client, kernel_matrix, hwpack): - lava_proxy = client.context.lava_proxy - LAVA_IMAGE_TMPDIR = client.context.lava_image_tmpdir +def _refresh_hwpack(client, tarball_dir, kernel_matrix, hwpack): logging.info("Deploying new kernel") new_kernel = kernel_matrix[0] deb_prefix = kernel_matrix[1] @@ -26,17 +26,14 @@ raise CriticalError("New kernel only support deb kernel package!") # download package to local - tarball_dir = mkdtemp(dir=LAVA_IMAGE_TMPDIR) - os.chmod(tarball_dir, 0755) - kernel_path = download(new_kernel, tarball_dir, lava_proxy) - hwpack_path = download(hwpack, tarball_dir, lava_proxy) + kernel_path = download_image(new_kernel, client.context, outdir, decompress=False) + hwpack_path = download_image(hwpack, client.context, outdir, decompress=False) cmd = ("sudo linaro-hwpack-replace -t %s -p %s -r %s" % (hwpack_path, kernel_path, deb_prefix)) rc, output = getstatusoutput(cmd) if rc: - shutil.rmtree(tarball_dir) raise RuntimeError("linaro-hwpack-replace failed: %s" % output) #fix it:l-h-r doesn't make a output option to specify the output hwpack, @@ -51,13 +48,13 @@ return new_hwpack_path -def generate_image(client, hwpack_url, rootfs_url, kernel_matrix, rootfstype=None): +def generate_image(client, hwpack_url, rootfs_url, kernel_matrix, + outdir, rootfstype=None): """Generate image from a hwpack and rootfs url :param hwpack_url: url of the Linaro hwpack to download :param rootfs_url: url of the Linaro image to download """ - lava_proxy = client.context.lava_proxy LAVA_IMAGE_TMPDIR = client.context.lava_image_tmpdir LAVA_IMAGE_URL = client.context.lava_image_url logging.info("preparing to deploy on %s" % client.hostname) @@ -65,20 +62,18 @@ logging.info(" rootfs: %s" % rootfs_url) if kernel_matrix: logging.info(" package: %s" % kernel_matrix[0]) - hwpack_url = refresh_hwpack(kernel_matrix, hwpack_url) + hwpack_url = _refresh_hwpack(client, outdir, kernel_matrix, hwpack_url) #make new hwpack downloadable hwpack_url = hwpack_url.replace(LAVA_IMAGE_TMPDIR, '') hwpack_url = '/'.join(u.strip('/') for u in [ LAVA_IMAGE_URL, hwpack_url]) logging.info(" hwpack with new kernel: %s" % hwpack_url) - tarball_dir = mkdtemp(dir=LAVA_IMAGE_TMPDIR) - os.chmod(tarball_dir, 0755) - #fix me: if url is not http-prefix, copy it to tarball_dir + logging.info("Downloading the %s file" % hwpack_url) - hwpack_path = download(hwpack_url, tarball_dir, lava_proxy) + hwpack_path = download_image(hwpack_url, client.context, outdir, decompress=False) logging.info("Downloading the %s file" % rootfs_url) - rootfs_path = download(rootfs_url, tarball_dir, lava_proxy) + rootfs_path = download_image(rootfs_url, client.context, outdir, decompress=False) logging.info("linaro-media-create version information") cmd = "sudo linaro-media-create -v" @@ -87,7 +82,7 @@ metadata['target.linaro-media-create-version'] = output client.context.test_data.add_metadata(metadata) - image_file = os.path.join(tarball_dir, "lava.img") + image_file = os.path.join(outdir, "lava.img") logging.info("client.device_type = %s" %client.device_type) @@ -98,13 +93,18 @@ cmd += ' --rootfs ' + rootfstype logging.info("Executing the linaro-media-create command") logging.info(cmd) - try: - _run_linaro_media_create(cmd) - except: - shutil.rmtree(tarball_dir) - raise + + _run_linaro_media_create(cmd) return image_file +def generate_fastmodel_image(hwpack, rootfs, odir, size="2000M"): + cmd = ("flock /var/lock/lava-lmc.lck sudo linaro-media-create " + "--dev fastmodel --output-directory %s --image-size %s " + "--hwpack %s --binary %s --hwpack-force-yes" % + (odir, size, hwpack, rootfs) ) + logging.info("Generating fastmodel image with: %s" % cmd) + _run_linaro_media_create(cmd) + def generate_android_image(device, boot, data, system, ofile, size="2000M"): cmd = ("flock /var/lock/lava-lmc.lck sudo linaro-android-media-create " "--dev %s --image_file %s --image_size %s " @@ -244,3 +244,4 @@ state = next_state_names[match_id] if state is None: return + === modified file 'lava_dispatcher/client/master.py' --- lava_dispatcher/client/master.py 2012-07-20 22:18:48 +0000 +++ lava_dispatcher/client/master.py 2012-07-22 01:40:55 +0000 @@ -25,7 +25,6 @@ import re import shutil import subprocess -from tempfile import mkdtemp import time import traceback import atexit @@ -37,7 +36,6 @@ download_image, ) from lava_dispatcher.utils import ( - download, logging_spawn, logging_system, string_to_list, @@ -472,11 +470,6 @@ shutil.copy(boot_tgz, c_boot_tgz) shutil.copy(root_tgz, c_root_tgz) - def _download(self, url, directory): - lava_proxy = self.context.lava_proxy - lava_cookies = self.context.lava_cookies - return download(url, directory, lava_proxy, lava_cookies) - def deploy_linaro(self, hwpack=None, rootfs=None, image=None, kernel_matrix=None, rootfstype='ext3'): LAVA_IMAGE_TMPDIR = self.context.lava_image_tmpdir @@ -493,12 +486,12 @@ # generate image if needed try: + tarball_dir = self.get_www_scratch_dir() if image is None: - image_file = generate_image(self, hwpack, rootfs, kernel_matrix) + image_file = generate_image(self, hwpack, rootfs, + kernel_matrix, tarball_dir) boot_tgz, root_tgz = self._generate_tarballs(image_file) else: - tarball_dir = mkdtemp(dir=LAVA_IMAGE_TMPDIR) - atexit.register(shutil.rmtree, tarball_dir) os.chmod(tarball_dir, 0755) lava_cachedir = self.context.lava_cachedir if self.context.job_data.get('health_check', False): @@ -621,7 +614,6 @@ self.sio.write(tb) raise CriticalError("Android deployment failed") finally: - shutil.rmtree(self.tarball_dir) logging.info("Android image deployment exiting") def _download_tarballs(self, boot_url, system_url, data_url, pkg_url=None): @@ -632,18 +624,14 @@ :param data_url: url of the Linaro Android data tarball to download :param pkg_url: url of the custom kernel tarball to download """ - lava_proxy = self.context.lava_proxy - LAVA_IMAGE_TMPDIR = self.context.lava_image_tmpdir - self.tarball_dir = mkdtemp(dir=LAVA_IMAGE_TMPDIR) - tarball_dir = self.tarball_dir - os.chmod(tarball_dir, 0755) + tarball_dir = self.get_www_scratch_dir() logging.info("Downloading the image files") - boot_path = download(boot_url, tarball_dir, lava_proxy) - system_path = download(system_url, tarball_dir, lava_proxy) - data_path = download(data_url, tarball_dir, lava_proxy) + boot_path = download_image(boot_url, self.context, tarball_dir, decompress=False) + system_path = download_image(system_url, self.context, tarball_dir, decompress=False) + data_path = download_image(data_url, self.context, tarball_dir, decompress=False) if pkg_url: - pkg_path = download(pkg_url, tarball_dir, lava_proxy) + pkg_path = download_image(pkg_url, self.context, tarball_dir, decompress=False) else: pkg_path = None logging.info("Downloaded the image files") @@ -740,8 +728,7 @@ time.sleep(3) result_tarball = "http://%s/lava_results.tgz" % master_ip - tarball_dir = mkdtemp(dir=self.context.lava_image_tmpdir) - os.chmod(tarball_dir, 0755) + tarball_dir = self.get_www_scratch_dir() # download test result with a retry mechanism # set retry timeout to 5 mins @@ -752,7 +739,8 @@ while True: try: - result_path = download(result_tarball, tarball_dir) + result_path = download_image(result_tarball, + self.context, tarball_dir, decompress=False) return 'pass', '', result_path except RuntimeError: tries += 1 === modified file 'lava_dispatcher/client/qemu.py' --- lava_dispatcher/client/qemu.py 2012-06-29 01:26:45 +0000 +++ lava_dispatcher/client/qemu.py 2012-07-21 22:56:42 +0000 @@ -49,7 +49,9 @@ def deploy_linaro(self, hwpack=None, rootfs=None, image=None, kernel_matrix=None, rootfstype='ext3'): if image is None: - image_file = generate_image(self, hwpack, rootfs, kernel_matrix, rootfstype) + odir = self.get_www_scratch_dir() + image_file = generate_image(self, hwpack, rootfs, kernel_matrix, + odir, rootfstype) else: image_file = download_image(image, self.context) self._lava_image = image_file === modified file 'lava_dispatcher/utils.py' --- lava_dispatcher/utils.py 2012-07-09 23:10:24 +0000 +++ lava_dispatcher/utils.py 2012-07-22 01:42:10 +0000 @@ -24,37 +24,11 @@ import os import sys import shutil -import urllib2 import urlparse from shlex import shlex import pexpect -def download(url, path="", proxy=None, cookies=None, verbose_failure=1): - urlpath = urlparse.urlsplit(url).path - filename = os.path.basename(urlpath) - if path: - filename = os.path.join(path, filename) - fd = open(filename, "w") - try: - if proxy: - handlers = [urllib2.ProxyHandler({'http': '%s' % proxy})] - else: - handlers = [] - opener = urllib2.build_opener(*handlers) - if cookies: - opener.addheaders.append(('Cookie', cookies)) - response = opener.open(urllib2.quote(url, safe=":/"), timeout=30) - fd = open(filename, 'wb') - shutil.copyfileobj(response, fd, 0x10000) - fd.close() - response.close() - except: - if verbose_failure: - logging.exception("download '%s' failed" % url) - raise RuntimeError("Could not retrieve %s" % url) - return filename - def link_or_copy_file(src, dest): try: dir = os.path.dirname(dest)