From patchwork Thu Apr 26 03:43:11 2012 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Andy Doan X-Patchwork-Id: 8136 Return-Path: X-Original-To: patchwork@peony.canonical.com Delivered-To: patchwork@peony.canonical.com Received: from fiordland.canonical.com (fiordland.canonical.com [91.189.94.145]) by peony.canonical.com (Postfix) with ESMTP id 1A94723E13 for ; Thu, 26 Apr 2012 03:43:14 +0000 (UTC) Received: from mail-iy0-f180.google.com (mail-iy0-f180.google.com [209.85.210.180]) by fiordland.canonical.com (Postfix) with ESMTP id B0099A182F2 for ; Thu, 26 Apr 2012 03:43:13 +0000 (UTC) Received: by iage36 with SMTP id e36so1457653iag.11 for ; Wed, 25 Apr 2012 20:43:13 -0700 (PDT) X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=20120113; h=x-forwarded-to:x-forwarded-for:delivered-to:received-spf :content-type:mime-version:x-launchpad-project:x-launchpad-branch :x-launchpad-message-rationale:x-launchpad-branch-revision-number :x-launchpad-notification-type:to:from:subject:message-id:date :reply-to:sender:errors-to:precedence:x-generated-by :x-launchpad-hash:x-gm-message-state; bh=PLISVnQEDGh7fPuwjcAXmMKc6ST2HL6JoDlwGNrww7c=; b=mh86WqoGQnAqzfAPKDhKIc1hyVG2A7fPpxADAS5WQO2GPjWKbiKSPZIJP6nVZQ7Q38 QhExap1fCIPn0/ygITCP8UN+tBvaHCmRVRw4BPdpcLYiVp21WmYwDbOLJFBoRx3wlpXa kQ1FNskZLsHJrVAgq6X7t7mCFk2xAxzw4Ah3SOrJSgfIkp47v94NQQpwROsmfXrKCRce dz1XC4YqwDBsBUzP2vRP2wHcYVkRWejCcIrqXRV+UjNHjE1oBwiRA+9at1G6IpBa4TCO 2Y5cFqD48WoqkGikgsyFEiodsVr7CzAJhYCBvTzRfa21OpSOC1YZN9XpUjWqN0eroyvG t1IA== Received: by 10.50.168.106 with SMTP id zv10mr3192534igb.55.1335411793165; Wed, 25 Apr 2012 20:43:13 -0700 (PDT) X-Forwarded-To: linaro-patchwork@canonical.com X-Forwarded-For: patch@linaro.org linaro-patchwork@canonical.com Delivered-To: patches@linaro.org Received: by 10.231.137.198 with SMTP id x6csp305712ibt; Wed, 25 Apr 2012 20:43:12 -0700 (PDT) Received: by 10.216.135.223 with SMTP id u73mr3075579wei.117.1335411791851; Wed, 25 Apr 2012 20:43:11 -0700 (PDT) Received: from indium.canonical.com (indium.canonical.com. [91.189.90.7]) by mx.google.com with ESMTPS id d7si2259571wid.26.2012.04.25.20.43.11 (version=TLSv1/SSLv3 cipher=OTHER); Wed, 25 Apr 2012 20:43:11 -0700 (PDT) Received-SPF: pass (google.com: best guess record for domain of bounces@canonical.com designates 91.189.90.7 as permitted sender) client-ip=91.189.90.7; Authentication-Results: mx.google.com; spf=pass (google.com: best guess record for domain of bounces@canonical.com designates 91.189.90.7 as permitted sender) smtp.mail=bounces@canonical.com Received: from ackee.canonical.com ([91.189.89.26]) by indium.canonical.com with esmtp (Exim 4.71 #1 (Debian)) id 1SNFbr-0007Jz-AY for ; Thu, 26 Apr 2012 03:43:11 +0000 Received: from ackee.canonical.com (localhost [127.0.0.1]) by ackee.canonical.com (Postfix) with ESMTP id 44259E1AB9 for ; Thu, 26 Apr 2012 03:43:11 +0000 (UTC) MIME-Version: 1.0 X-Launchpad-Project: lava-dispatcher X-Launchpad-Branch: ~linaro-validation/lava-dispatcher/trunk X-Launchpad-Message-Rationale: Subscriber X-Launchpad-Branch-Revision-Number: 277 X-Launchpad-Notification-Type: branch-revision To: Linaro Patch Tracker From: noreply@launchpad.net Subject: [Branch ~linaro-validation/lava-dispatcher/trunk] Rev 277: merge cache-rootfs-boot-tarballs Message-Id: <20120426034311.12853.3168.launchpad@ackee.canonical.com> Date: Thu, 26 Apr 2012 03:43:11 -0000 Reply-To: noreply@launchpad.net Sender: bounces@canonical.com Errors-To: bounces@canonical.com Precedence: bulk X-Generated-By: Launchpad (canonical.com); Revision="15149"; Instance="launchpad-lazr.conf" X-Launchpad-Hash: 811b790eae490a0a89e12e0e4075851b42315c50 X-Gm-Message-State: ALoCoQm/8ASimxWOdY6nsj9GfZ2TWVM7Wai2NDcpdZbfi07iOVCr4Yd+OxZaDGJjsTDepyBNuZ1Z Merge authors: Le Chi Thu le.chi.thu@linaro.org Related merge proposals: https://code.launchpad.net/~le-chi-thu/lava-dispatcher/cache-tarballs-v1/+merge/102181 proposed by: Le Chi Thu (le-chi-thu) review: Approve - Zygmunt Krynicki (zkrynicki) review: Resubmit - Le Chi Thu (le-chi-thu) ------------------------------------------------------------ revno: 277 [merge] committer: Andy Doan branch nick: lava-dispatcher timestamp: Wed 2012-04-25 22:38:59 -0500 message: merge cache-rootfs-boot-tarballs modified: lava_dispatcher/client/master.py lava_dispatcher/utils.py --- lp:lava-dispatcher https://code.launchpad.net/~linaro-validation/lava-dispatcher/trunk You are subscribed to branch lp:lava-dispatcher. To unsubscribe from this branch go to https://code.launchpad.net/~linaro-validation/lava-dispatcher/trunk/+edit-subscription === modified file 'lava_dispatcher/client/master.py' --- lava_dispatcher/client/master.py 2012-04-17 15:40:25 +0000 +++ lava_dispatcher/client/master.py 2012-04-26 03:38:59 +0000 @@ -30,6 +30,7 @@ import traceback import pexpect +import errno from lava_dispatcher.utils import ( download, @@ -37,7 +38,7 @@ logging_spawn, logging_system, string_to_list, - ) + url_to_cache, link_or_copy_file) from lava_dispatcher.client.base import ( CommandRunner, CriticalError, @@ -58,13 +59,13 @@ :param partno: The index of the partition in the image :param tarfile: path and filename of the tgz to output """ + with image_partition_mounted(image, partno) as mntdir: cmd = "sudo tar -C %s -czf %s ." % (mntdir, tarfile) rc = logging_system(cmd) if rc: raise RuntimeError("Failed to create tarball: %s" % tarfile) - def _deploy_tarball_to_board(session, tarball_url, dest, timeout=-1): decompression_char = '' if tarball_url.endswith('.gz') or tarball_url.endswith('.tgz'): @@ -289,31 +290,129 @@ return uncompressed_name return image_file + def _tarball_url_to_cache(self, url, cachedir): + cache_loc = url_to_cache(url, cachedir) + # can't have a folder name same as file name. replacing '.' with '.' + return os.path.join(cache_loc.replace('.','-'), "tarballs") + + def _are_tarballs_cached(self, image, lava_cachedir): + cache_loc = self._tarball_url_to_cache(image, lava_cachedir) + cached = os.path.exists(os.path.join(cache_loc, "boot.tgz")) and \ + os.path.exists(os.path.join(cache_loc, "root.tgz")) + + if cached: + return True; + + # Check if there is an other lava-dispatch instance have start to cache the same image + # see the _about_to_cache_tarballs + if not os.path.exists(os.path.join(cache_loc, "tarballs-cache-ongoing")): + return False + + # wait x minute for caching is done. + waittime=20 + + logging.info("Waiting for the other instance of lava-dispatcher to finish the caching of %s", image) + while waittime > 0: + if not os.path.exists(os.path.join(cache_loc, "tarballs-cache-ongoing")): + waittime = 0 + else: + time.sleep(60) + waittime = waittime - 1 + if (waittime % 5) == 0: + logging.info("%d minute left..." % waittime) + + return os.path.exists(os.path.join(cache_loc, "boot.tgz")) and \ + os.path.exists(os.path.join(cache_loc, "root.tgz")) + + def _get_cached_tarballs(self, image, tarball_dir, lava_cachedir): + cache_loc = self._tarball_url_to_cache(image, lava_cachedir) + + boot_tgz = os.path.join(tarball_dir,"boot.tgz") + root_tgz = os.path.join(tarball_dir,"root.tgz") + link_or_copy_file(os.path.join(cache_loc, "root.tgz"), root_tgz) + link_or_copy_file(os.path.join(cache_loc, "boot.tgz"), boot_tgz) + + return (boot_tgz,root_tgz) + + def _about_to_cache_tarballs(self, image, lava_cachedir): + # create this folder to indicate this instance of lava-dispatcher is caching this image. + # see _are_tarballs_cached + # return false if unable to create the directory. The caller should not cache the tarballs + cache_loc = self._tarball_url_to_cache(image, lava_cachedir) + path = os.path.join(cache_loc, "tarballs-cache-ongoing") + try: + os.makedirs(path) + except OSError as exc: # Python >2.5 + if exc.errno == errno.EEXIST: + # other dispatcher process already caching - concurrency issue + return False + else: + raise + return True + + def _cache_tarballs(self, image, boot_tgz, root_tgz, lava_cachedir): + cache_loc = self._tarball_url_to_cache(image, lava_cachedir) + if not os.path.exists(cache_loc): + os.makedirs(cache_loc) + c_boot_tgz = os.path.join(cache_loc, "boot.tgz") + c_root_tgz = os.path.join(cache_loc, "root.tgz") + shutil.copy(boot_tgz, c_boot_tgz) + shutil.copy(root_tgz, c_root_tgz) + path = os.path.join(cache_loc, "tarballs-cache-ongoing") + if os.path.exists(path): + shutil.rmtree(path) def deploy_linaro(self, hwpack=None, rootfs=None, image=None, kernel_matrix=None, use_cache=True, rootfstype='ext3'): LAVA_IMAGE_TMPDIR = self.context.lava_image_tmpdir LAVA_IMAGE_URL = self.context.lava_image_url + + # validate in parameters + if image is None: + if hwpack is None or rootfs is None: + raise CriticalError( + "must specify both hwpack and rootfs when not specifying image") + else: + if hwpack is not None or rootfs is not None or kernel_matrix is not None: + raise CriticalError( + "cannot specify hwpack or rootfs when specifying image") + + # generate image if needed try: if image is None: - if hwpack is None or rootfs is None: - raise CriticalError( - "must specify both hwpack and rootfs when not specifying image") - else: - image_file = generate_image(self, hwpack, rootfs, kernel_matrix, use_cache) + image_file = generate_image(self, hwpack, rootfs, kernel_matrix, use_cache) + boot_tgz, root_tgz = self._generate_tarballs(image_file) else: - if hwpack is not None or rootfs is not None or kernel_matrix is not None: - raise CriticalError( - "cannot specify hwpack or rootfs when specifying image") tarball_dir = mkdtemp(dir=LAVA_IMAGE_TMPDIR) os.chmod(tarball_dir, 0755) if use_cache: lava_cachedir = self.context.lava_cachedir - image_file = download_with_cache(image, tarball_dir, lava_cachedir) + if self._are_tarballs_cached(image, lava_cachedir): + logging.info("Reusing cached tarballs") + boot_tgz, root_tgz = self._get_cached_tarballs(image, tarball_dir, lava_cachedir) + else: + logging.info("Downloading and caching the tarballs") + # in some corner case, there can be more than one lava-dispatchers execute + # caching of same tarballs exact at the same time. One of them will successfully + # get the lock directory. The rest will skip the caching if _about_to_cache_tarballs + # return false. + should_cache = self._about_to_cache_tarballs(image, lava_cachedir) + image_file = download_with_cache(image, tarball_dir, lava_cachedir) + image_file = self.decompress(image_file) + boot_tgz, root_tgz = self._generate_tarballs(image_file) + if should_cache: + self._cache_tarballs(image, boot_tgz, root_tgz, lava_cachedir) else: image_file = download(image, tarball_dir) - image_file = self.decompress(image_file) - boot_tgz, root_tgz = self._generate_tarballs(image_file) + image_file = self.decompress(image_file) + boot_tgz, root_tgz = self._generate_tarballs(image_file) + # remove the cached tarballs + cache_loc = self._tarball_url_to_cache(image, lava_cachedir) + shutil.rmtree(cache_loc, ignore_errors = true) + # remove the cached image files + cache_loc = url_to_cache + shutil.rmtree(cache_loc, ignore_errors = true) + except CriticalError: raise except: @@ -322,6 +421,7 @@ self.sio.write(tb) raise CriticalError("Deployment tarballs preparation failed") + # deploy the boot image and rootfs to target logging.info("Booting master image") try: self.boot_master_image() === modified file 'lava_dispatcher/utils.py' --- lava_dispatcher/utils.py 2012-03-19 18:39:24 +0000 +++ lava_dispatcher/utils.py 2012-04-18 22:36:15 +0000 @@ -48,6 +48,27 @@ raise RuntimeError("Could not retrieve %s" % url) return filename +def link_or_copy_file(src, dest): + try: + dir = os.path.dirname(dest) + if not os.path.exists(dir): + os.makedirs(dir) + os.link(src, dest) + except OSError, err: + if err.errno == errno.EXDEV: + shutil.copy(src, dest) + if err.errno == errno.EEXIST: + logging.debug("Cached copy of %s already exists" % dest) + else: + logging.exception("os.link '%s' with '%s' failed" % (src, dest)) + +def copy_file(src, dest): + dir = os.path.dirname(dest) + if not os.path.exists(dir): + os.makedirs(dir) + shutil.copy(src, dest) + + # XXX: duplication, we have similar code in lava-test, we need to move that to # lava.utils -> namespace as standalone package def download_with_cache(url, path="", cachedir=""): @@ -55,31 +76,10 @@ if os.path.exists(cache_loc): filename = os.path.basename(cache_loc) file_location = os.path.join(path, filename) - try: - os.link(cache_loc, file_location) - except OSError, err: - if err.errno == errno.EXDEV: - shutil.copy(cache_loc, file_location) - if err.errno == errno.EEXIST: - logging.debug("Cached copy of %s already exists" % url) - else: - logging.exception("os.link '%s' with '%s' failed" % (cache_loc, - file_location)) + link_or_copy_file(cache_loc, file_location) else: file_location = download(url, path) - try: - cache_dir = os.path.dirname(cache_loc) - if not os.path.exists(cache_dir): - os.makedirs(cache_dir) - os.link(file_location, cache_loc) - except OSError, err: - #errno.EXDEV(18) is Invalid cross-device link - if err.errno == errno.EXDEV: - shutil.copy(file_location, cache_loc) - if err.errno == errno.EEXIST: - logging.debug("Cached copy of %s already exists" % url) - else: - logging.exception("os.link failed") + copy_file(file_location, cache_loc) return file_location