From patchwork Mon Jun 11 20:08:12 2012 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Andy Doan X-Patchwork-Id: 9211 Return-Path: X-Original-To: patchwork@peony.canonical.com Delivered-To: patchwork@peony.canonical.com Received: from fiordland.canonical.com (fiordland.canonical.com [91.189.94.145]) by peony.canonical.com (Postfix) with ESMTP id 9DA6923EB4 for ; Mon, 11 Jun 2012 20:08:16 +0000 (UTC) Received: from mail-yx0-f180.google.com (mail-yx0-f180.google.com [209.85.213.180]) by fiordland.canonical.com (Postfix) with ESMTP id 50BFDA18705 for ; Mon, 11 Jun 2012 20:08:16 +0000 (UTC) Received: by yenq6 with SMTP id q6so3139841yen.11 for ; Mon, 11 Jun 2012 13:08:15 -0700 (PDT) X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=20120113; h=x-forwarded-to:x-forwarded-for:delivered-to:received-spf :content-type:mime-version:x-launchpad-project:x-launchpad-branch :x-launchpad-message-rationale:x-launchpad-branch-revision-number :x-launchpad-notification-type:to:from:subject:message-id:date :reply-to:sender:errors-to:precedence:x-generated-by :x-launchpad-hash:x-gm-message-state; bh=1YFrvLSIzPUbcpewKsUl+blAvZ9ERuPv5dpffvgEpfQ=; b=dcPC0jJ4ok+zFGAqu/yocEvLdkltFV8j4iTvPRSwvbDMV+l5UwxcFfyV84s274fZ8N qRxaXu0nkHxqwmMDQIvWRUQJYpOFJm5MXvx2xt9QKpDmOhIzFErWKKga5w6Cf5ZAXrab iKQJziWwGsJh+cF1G6rs7F+gGBxsC0WWSa9d381Q1flWtbheI7KEKUdCUtA7XovbSSWi zzQF9nqrMgbcR/GDzvooTEBVIJ3jqlfzjMLt7WzoaA0DfqEOxL3Rud2801A+Gujtmx0w fyzH54mMjQDCR0CEmh4VOFAl4EHHNRTASUjaJj14mDfTX7lyIzZXVgdV0z+CmTfIIlUt EleA== Received: by 10.50.46.232 with SMTP id y8mr7039535igm.57.1339445295601; Mon, 11 Jun 2012 13:08:15 -0700 (PDT) X-Forwarded-To: linaro-patchwork@canonical.com X-Forwarded-For: patch@linaro.org linaro-patchwork@canonical.com Delivered-To: patches@linaro.org Received: by 10.231.24.148 with SMTP id v20csp128846ibb; Mon, 11 Jun 2012 13:08:14 -0700 (PDT) Received: by 10.216.194.93 with SMTP id l71mr7543282wen.169.1339445293982; Mon, 11 Jun 2012 13:08:13 -0700 (PDT) Received: from indium.canonical.com (indium.canonical.com. [91.189.90.7]) by mx.google.com with ESMTPS id n21si25550377wee.115.2012.06.11.13.08.12 (version=TLSv1/SSLv3 cipher=OTHER); Mon, 11 Jun 2012 13:08:13 -0700 (PDT) Received-SPF: pass (google.com: best guess record for domain of bounces@canonical.com designates 91.189.90.7 as permitted sender) client-ip=91.189.90.7; Authentication-Results: mx.google.com; spf=pass (google.com: best guess record for domain of bounces@canonical.com designates 91.189.90.7 as permitted sender) smtp.mail=bounces@canonical.com Received: from ackee.canonical.com ([91.189.89.26]) by indium.canonical.com with esmtp (Exim 4.71 #1 (Debian)) id 1SeAuK-0007hm-Pk for ; Mon, 11 Jun 2012 20:08:12 +0000 Received: from ackee.canonical.com (localhost [127.0.0.1]) by ackee.canonical.com (Postfix) with ESMTP id B41F6E01F2 for ; Mon, 11 Jun 2012 20:08:12 +0000 (UTC) MIME-Version: 1.0 X-Launchpad-Project: lava-dispatcher X-Launchpad-Branch: ~linaro-validation/lava-dispatcher/trunk X-Launchpad-Message-Rationale: Subscriber X-Launchpad-Branch-Revision-Number: 321 X-Launchpad-Notification-Type: branch-revision To: Linaro Patch Tracker From: noreply@launchpad.net Subject: [Branch ~linaro-validation/lava-dispatcher/trunk] Rev 321: download code fixes for proxy and cookies Message-Id: <20120611200812.22680.2634.launchpad@ackee.canonical.com> Date: Mon, 11 Jun 2012 20:08:12 -0000 Reply-To: noreply@launchpad.net Sender: bounces@canonical.com Errors-To: bounces@canonical.com Precedence: bulk X-Generated-By: Launchpad (canonical.com); Revision="15376"; Instance="launchpad-lazr.conf" X-Launchpad-Hash: 24248d038c700ec305a14f2684c3ca44442df728 X-Gm-Message-State: ALoCoQlvMe0bFmazDglrpfEyyiEjhcQjJwLiCQN1TUCNfxXSJ12tCroyLs/Tp3o1aYR5G5y1Rbq8 Merge authors: Andy Doan (doanac) Related merge proposals: https://code.launchpad.net/~doanac/lava-dispatcher/download-fixes/+merge/109255 proposed by: Andy Doan (doanac) review: Approve - Michael Hudson-Doyle (mwhudson) ------------------------------------------------------------ revno: 321 [merge] committer: Andy Doan branch nick: lava-dispatcher timestamp: Mon 2012-06-11 14:54:52 -0500 message: download code fixes for proxy and cookies modified: lava_dispatcher/client/master.py lava_dispatcher/context.py lava_dispatcher/utils.py --- lp:lava-dispatcher https://code.launchpad.net/~linaro-validation/lava-dispatcher/trunk You are subscribed to branch lp:lava-dispatcher. To unsubscribe from this branch go to https://code.launchpad.net/~linaro-validation/lava-dispatcher/trunk/+edit-subscription === modified file 'lava_dispatcher/client/master.py' --- lava_dispatcher/client/master.py 2012-06-08 06:22:05 +0000 +++ lava_dispatcher/client/master.py 2012-06-11 19:54:52 +0000 @@ -35,7 +35,6 @@ from lava_dispatcher.utils import ( download, - download_with_cache, logging_spawn, logging_system, string_to_list, @@ -439,6 +438,11 @@ if os.path.exists(path): shutil.rmtree(path) + def _download(self, url, directory): + lava_proxy = self.context.lava_proxy + lava_cookies = self.context.lava_cookies + return download(url, directory, lava_proxy, lava_cookies) + def deploy_linaro(self, hwpack=None, rootfs=None, image=None, kernel_matrix=None, use_cache=True, rootfstype='ext3'): LAVA_IMAGE_TMPDIR = self.context.lava_image_tmpdir @@ -473,14 +477,13 @@ # get the lock directory. The rest will skip the caching if _about_to_cache_tarballs # return false. should_cache = self._about_to_cache_tarballs(image, lava_cachedir) - image_file = download_with_cache(image, tarball_dir, lava_cachedir) + image_file = self._download(image, tarball_dir) image_file = self.decompress(image_file) boot_tgz, root_tgz = self._generate_tarballs(image_file) if should_cache: self._cache_tarballs(image, boot_tgz, root_tgz, lava_cachedir) else: - lava_proxy = self.context.lava_proxy - image_file = download(image, tarball_dir, lava_proxy) + image_file = self._download(image, tarball_dir) image_file = self.decompress(image_file) boot_tgz, root_tgz = self._generate_tarballs(image_file) # remove the cached tarballs === modified file 'lava_dispatcher/context.py' --- lava_dispatcher/context.py 2012-05-08 05:37:59 +0000 +++ lava_dispatcher/context.py 2012-06-07 23:38:28 +0000 @@ -63,6 +63,10 @@ return proxy @property + def lava_cookies(self): + return self.config.get("LAVA_COOKIES", None) + + @property def lava_image_tmpdir(self): return self.config.get("LAVA_IMAGE_TMPDIR") === modified file 'lava_dispatcher/utils.py' --- lava_dispatcher/utils.py 2012-05-08 05:56:59 +0000 +++ lava_dispatcher/utils.py 2012-06-07 23:38:28 +0000 @@ -29,8 +29,7 @@ import pexpect - -def download(url, path="", proxy=None, verbose_failure=1): +def download(url, path="", proxy=None, cookies=None, verbose_failure=1): urlpath = urlparse.urlsplit(url).path filename = os.path.basename(urlpath) if path: @@ -42,6 +41,8 @@ else: handlers = [] opener = urllib2.build_opener(*handlers) + if cookies: + opener.addheaders.append(('Cookie', cookies)) response = opener.open(urllib2.quote(url, safe=":/"), timeout=30) fd = open(filename, 'wb') shutil.copyfileobj(response, fd, 0x10000) @@ -74,20 +75,6 @@ shutil.copy(src, dest) -# XXX: duplication, we have similar code in lava-test, we need to move that to -# lava.utils -> namespace as standalone package -def download_with_cache(url, path="", cachedir=""): - cache_loc = url_to_cache(url, cachedir) - if os.path.exists(cache_loc): - filename = os.path.basename(cache_loc) - file_location = os.path.join(path, filename) - link_or_copy_file(cache_loc, file_location) - else: - file_location = download(url, path) - copy_file(file_location, cache_loc) - return file_location - - def url_to_cache(url, cachedir): url_parts = urlparse.urlsplit(url) path = os.path.join(cachedir, url_parts.netloc,