=== modified file 'doc/QUICKSTART'
@@ -95,6 +95,14 @@
$ cat ~/.config/lava-dispatcher/lava-dispatcher.conf
LAVA_SERVER_IP = 192.168.88.77
+You can set LAVA_PROXY if you have a proxy available in
+~/.config/lava-dispatcher/lava-dispatcher.conf, squid proxy service
+is preferred, if no, please leave it blank::
+
+ $ cat ~/.config/lava-dispatcher/lava-dispatcher.conf
+ LAVA_PROXY = http://192.168.88.77:3128/
+
+
You will need to add a configuration file for your device. It can be
extremely simple, just identifying the type of the device::
=== modified file 'doc/changes.rst'
@@ -1,11 +1,17 @@
Version History
***************
+.. _version_0_7:
+
+Version 0.7 (Milestone 12.05, Unreleased)
+=========================================
+
+* Use squid proxy for caching mechanism
+
.. _version_0_6:
-Version 0.6
-===========
-
+Version 0.6 (Milestone 12.04)
+=============================
* Merge 0.5.12 bugfix release
* Config options for interrupting boot process
@@ -56,9 +62,9 @@
* Changes for virtual express support:
* Add in a standard vexpress config for UEFI
* Make changes to allow for different boot interception message
- configuration
+ configuration
* Increase timeouts for some stages of deployment (mkfs ext3) to
- account for vexpress (lack of) speed.
+ account for vexpress (lack of) speed.
.. _version_0_5_7:
=== modified file 'doc/conf.py'
@@ -61,7 +61,7 @@
# The short X.Y version.
version = '0.4'
# The full version, including alpha/beta/rc tags.
-release = '0.4.1'
+release = '0.6'
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
=== added file 'doc/configuration.rst'
@@ -0,0 +1,18 @@
+.. _configuration:
+
+Configuration
+=============
+
+Guideline
+^^^^^^^^^
+
+Please refer to README in lava-dispatcher source code::
+
+ lava-dispatcher/lava_dispatcher/default-config/lava-dispatcher/README
+
+Or browse on the `web page <http://bazaar.launchpad.net/~linaro-validation/lava-dispatcher/trunk/view/head:/lava_dispatcher/default-config/lava-dispatcher/README>`_
+
+Don't forget to setup LAVA_SERVER_IP, and LAVA_PROXY if there is one proxy,
+which will prompt the performance explicitly.
+
+You can set up a squid service, refer to :ref:`proxy`
=== modified file 'doc/index.rst'
@@ -40,7 +40,7 @@
* Make use of the output of LAVA test, which is Linaro Dashboard Bundle format,
upload test results to the LAVA Dashboard for result archiving and analysis.
-.. seealso:: See what's new in :ref:`version_0_5_3`
+.. seealso:: See what's new in :ref:`version_0_6`
.. todo::
@@ -76,6 +76,7 @@
:maxdepth: 2
installation.rst
+ configuration.rst
jobfile.rst
usage.rst
changes.rst
=== modified file 'doc/installation.rst'
@@ -175,6 +175,47 @@
There are several installation options available:
+Using pip
+---------
+
+To install from pip::
+
+ pip install lava-dispatcher
+
+To upgrade from pip::
+
+ pip install --upgrade lava-dispatcher
+
+Using lava-deployment-tool
+--------------------------
+
+To install from lava-deployment-tool, first checkout lava-deployment-tool::
+
+ bzr branch lp:lava-deployment-tool
+
+Refer to README in lava-deployment-tool, make sure in "./lava-deployment-tool
+bundle" commands, requirements.txt includes lava-dispatcher.
+
+lava-dispatcher can be found in /srv/lava/instances/$LAVA_INSTANCE/bin.
+
+To use lava-dispatcher, activate virtualenv::
+
+ cd /srv/lava/instances/$LAVA_INSTANCE
+ . bin/activate
+
+Using source tarball
+--------------------
+
+To install from source you must first obtain a source tarball from bazzar
+branch or from `Launchpad <https://launchpad.net/lava-dispatcher/+download>`_::
+
+ bzr branch lp:lava-dispatcher
+
+To install the package unpack the tarball and run::
+
+ sudo python setup.py install
+
+
Using Ubuntu PPAs
-----------------
@@ -193,16 +234,3 @@
Finally you can install the package, it is called `lava-dispatcher`::
sudo apt-get install lava-dispatcher
-
-
-Using source tarball
---------------------
-
-To install from source you must first obtain a source tarball from bazzar
-branch or from `Launchpad <https://launchpad.net/lava-dispatcher/+download>`_::
-
- bzr branch lp:lava-dispatcher
-
-To install the package unpack the tarball and run::
-
- sudo python setup.py install
=== modified file 'lava_dispatcher/actions/lava-test.py'
@@ -122,6 +122,13 @@
with self.client.reliable_session() as session:
+ lava_proxy = self.client.context.lava_proxy
+ if lava_proxy:
+ session.run("sh -c 'export http_proxy=%s'" % lava_proxy)
+ session.run("echo 'Acquire::http::proxy \"%s\";' > /etc/apt/apt.conf.d/30proxy" % lava_proxy)
+ else:
+ session.run("echo '' > /etc/apt/apt.conf.d/30proxy")
+
_install_lava_test(self.client, session)
if install_python:
=== modified file 'lava_dispatcher/client/base.py'
@@ -68,6 +68,8 @@
.expect().
:param timeout: How long to wait for 'response' (if specified) and the
shell prompt, defaulting to forever.
+ :param failok: The command can fail or not, if it is set False and
+ command fail, an OperationFail exception will raise
:return: The exit value of the command, if wait_for_rc not explicitly
set to False during construction.
"""
@@ -376,6 +378,20 @@
def deploy_linaro(self, hwpack, rootfs, kernel_matrix=None, use_cache=True, rootfstype='ext3'):
raise NotImplementedError(self.deploy_linaro)
+ def setup_proxy(self, prompt_str):
+ lava_proxy = self.context.lava_proxy
+ if lava_proxy:
+ logging.info("Setting up http proxy")
+ # haven't included Android support yet
+ self.proc.sendline("export http_proxy=%s" % lava_proxy)
+ self.proc.expect(prompt_str, timeout=10)
+ self.proc.sendline("echo 'Acquire::http::proxy \"%s\";' > /etc/apt/apt.conf.d/30proxy" % lava_proxy)
+ self.proc.expect(prompt_str, timeout=10)
+ else:
+ self.proc.sendline("echo '' > /etc/apt/apt.conf.d/30proxy")
+ self.proc.expect(prompt_str, timeout=10)
+
+
def boot_master_image(self):
raise NotImplementedError(self.boot_master_image)
@@ -394,6 +410,7 @@
self.proc.sendline('export PS1="$PS1 [rc=$(echo \$?)]: "')
self.proc.expect(self.tester_str, timeout=10)
+ self.setup_proxy(self.tester_str)
logging.info("System is in test image now")
def get_seriallog(self):
@@ -409,6 +426,7 @@
self._boot_linaro_android_image()
self.in_test_shell(timeout=900)
self.proc.sendline("export PS1=\"root@linaro: \"")
+ #TODO: set up proxy
if self.config.get("enable_network_after_boot_android"):
time.sleep(1)
=== modified file 'lava_dispatcher/client/lmc_utils.py'
@@ -12,12 +12,11 @@
from lava_dispatcher.client.base import CriticalError
from lava_dispatcher.utils import (
download,
- download_with_cache,
logging_system,
)
def refresh_hwpack(client, kernel_matrix, hwpack, use_cache=True):
- lava_cachedir = client.context.lava_cachedir
+ lava_proxy = client.context.lava_proxy
LAVA_IMAGE_TMPDIR = client.context.lava_image_tmpdir
logging.info("Deploying new kernel")
new_kernel = kernel_matrix[0]
@@ -31,11 +30,11 @@
tarball_dir = mkdtemp(dir=LAVA_IMAGE_TMPDIR)
os.chmod(tarball_dir, 0755)
if use_cache:
- kernel_path = download_with_cache(new_kernel, tarball_dir, lava_cachedir)
- hwpack_path = download_with_cache(hwpack, tarball_dir, lava_cachedir)
+ proxy = lava_proxy
else:
- kernel_path = download(new_kernel, tarball_dir)
- hwpack_path = download(hwpack, tarball_dir)
+ proxy = None
+ kernel_path = download(new_kernel, tarball_dir, proxy)
+ hwpack_path = download(hwpack, tarball_dir, proxy)
cmd = ("sudo linaro-hwpack-replace -t %s -p %s -r %s"
% (hwpack_path, kernel_path, deb_prefix))
@@ -63,7 +62,7 @@
:param hwpack_url: url of the Linaro hwpack to download
:param rootfs_url: url of the Linaro image to download
"""
- lava_cachedir = client.context.lava_cachedir
+ lava_proxy = client.context.lava_proxy
LAVA_IMAGE_TMPDIR = client.context.lava_image_tmpdir
LAVA_IMAGE_URL = client.context.lava_image_url
logging.info("preparing to deploy on %s" % client.hostname)
@@ -81,17 +80,15 @@
os.chmod(tarball_dir, 0755)
#fix me: if url is not http-prefix, copy it to tarball_dir
if use_cache:
- logging.info("Downloading the %s file using cache" % hwpack_url)
- hwpack_path = download_with_cache(hwpack_url, tarball_dir, lava_cachedir)
-
- logging.info("Downloading the %s file using cache" % rootfs_url)
- rootfs_path = download_with_cache(rootfs_url, tarball_dir, lava_cachedir)
+ proxy = lava_proxy
else:
- logging.info("Downloading the %s file" % hwpack_url)
- hwpack_path = download(hwpack_url, tarball_dir)
-
- logging.info("Downloading the %s file" % rootfs_url)
- rootfs_path = download(rootfs_url, tarball_dir)
+ proxy = None
+
+ logging.info("Downloading the %s file" % hwpack_url)
+ hwpack_path = download(hwpack_url, tarball_dir, proxy)
+
+ logging.info("Downloading the %s file" % rootfs_url)
+ rootfs_path = download(rootfs_url, tarball_dir, proxy)
logging.info("linaro-media-create version information")
cmd = "sudo linaro-media-create -v"
=== modified file 'lava_dispatcher/client/master.py'
@@ -34,11 +34,9 @@
from lava_dispatcher.utils import (
download,
- download_with_cache,
logging_spawn,
logging_system,
- string_to_list,
- url_to_cache, link_or_copy_file)
+ string_to_list)
from lava_dispatcher.client.base import (
CommandRunner,
CriticalError,
@@ -290,82 +288,11 @@
return uncompressed_name
return image_file
- def _tarball_url_to_cache(self, url, cachedir):
- cache_loc = url_to_cache(url, cachedir)
- # can't have a folder name same as file name. replacing '.' with '.'
- return os.path.join(cache_loc.replace('.','-'), "tarballs")
-
- def _are_tarballs_cached(self, image, lava_cachedir):
- cache_loc = self._tarball_url_to_cache(image, lava_cachedir)
- cached = os.path.exists(os.path.join(cache_loc, "boot.tgz")) and \
- os.path.exists(os.path.join(cache_loc, "root.tgz"))
-
- if cached:
- return True;
-
- # Check if there is an other lava-dispatch instance have start to cache the same image
- # see the _about_to_cache_tarballs
- if not os.path.exists(os.path.join(cache_loc, "tarballs-cache-ongoing")):
- return False
-
- # wait x minute for caching is done.
- waittime=20
-
- logging.info("Waiting for the other instance of lava-dispatcher to finish the caching of %s", image)
- while waittime > 0:
- if not os.path.exists(os.path.join(cache_loc, "tarballs-cache-ongoing")):
- waittime = 0
- else:
- time.sleep(60)
- waittime = waittime - 1
- if (waittime % 5) == 0:
- logging.info("%d minute left..." % waittime)
-
- return os.path.exists(os.path.join(cache_loc, "boot.tgz")) and \
- os.path.exists(os.path.join(cache_loc, "root.tgz"))
-
- def _get_cached_tarballs(self, image, tarball_dir, lava_cachedir):
- cache_loc = self._tarball_url_to_cache(image, lava_cachedir)
-
- boot_tgz = os.path.join(tarball_dir,"boot.tgz")
- root_tgz = os.path.join(tarball_dir,"root.tgz")
- link_or_copy_file(os.path.join(cache_loc, "root.tgz"), root_tgz)
- link_or_copy_file(os.path.join(cache_loc, "boot.tgz"), boot_tgz)
-
- return (boot_tgz,root_tgz)
-
- def _about_to_cache_tarballs(self, image, lava_cachedir):
- # create this folder to indicate this instance of lava-dispatcher is caching this image.
- # see _are_tarballs_cached
- # return false if unable to create the directory. The caller should not cache the tarballs
- cache_loc = self._tarball_url_to_cache(image, lava_cachedir)
- path = os.path.join(cache_loc, "tarballs-cache-ongoing")
- try:
- os.makedirs(path)
- except OSError as exc: # Python >2.5
- if exc.errno == errno.EEXIST:
- # other dispatcher process already caching - concurrency issue
- return False
- else:
- raise
- return True
-
- def _cache_tarballs(self, image, boot_tgz, root_tgz, lava_cachedir):
- cache_loc = self._tarball_url_to_cache(image, lava_cachedir)
- if not os.path.exists(cache_loc):
- os.makedirs(cache_loc)
- c_boot_tgz = os.path.join(cache_loc, "boot.tgz")
- c_root_tgz = os.path.join(cache_loc, "root.tgz")
- shutil.copy(boot_tgz, c_boot_tgz)
- shutil.copy(root_tgz, c_root_tgz)
- path = os.path.join(cache_loc, "tarballs-cache-ongoing")
- if os.path.exists(path):
- shutil.rmtree(path)
-
def deploy_linaro(self, hwpack=None, rootfs=None, image=None,
kernel_matrix=None, use_cache=True, rootfstype='ext3'):
LAVA_IMAGE_TMPDIR = self.context.lava_image_tmpdir
LAVA_IMAGE_URL = self.context.lava_image_url
+ lava_proxy = self.context.lava_proxy
# validate in parameters
if image is None:
@@ -385,34 +312,10 @@
else:
tarball_dir = mkdtemp(dir=LAVA_IMAGE_TMPDIR)
os.chmod(tarball_dir, 0755)
- if use_cache:
- lava_cachedir = self.context.lava_cachedir
- if self._are_tarballs_cached(image, lava_cachedir):
- logging.info("Reusing cached tarballs")
- boot_tgz, root_tgz = self._get_cached_tarballs(image, tarball_dir, lava_cachedir)
- else:
- logging.info("Downloading and caching the tarballs")
- # in some corner case, there can be more than one lava-dispatchers execute
- # caching of same tarballs exact at the same time. One of them will successfully
- # get the lock directory. The rest will skip the caching if _about_to_cache_tarballs
- # return false.
- should_cache = self._about_to_cache_tarballs(image, lava_cachedir)
- image_file = download_with_cache(image, tarball_dir, lava_cachedir)
- image_file = self.decompress(image_file)
- boot_tgz, root_tgz = self._generate_tarballs(image_file)
- if should_cache:
- self._cache_tarballs(image, boot_tgz, root_tgz, lava_cachedir)
- else:
- image_file = download(image, tarball_dir)
- image_file = self.decompress(image_file)
- boot_tgz, root_tgz = self._generate_tarballs(image_file)
- # remove the cached tarballs
- cache_loc = self._tarball_url_to_cache(image, lava_cachedir)
- shutil.rmtree(cache_loc, ignore_errors = true)
- # remove the cached image files
- cache_loc = url_to_cache
- shutil.rmtree(cache_loc, ignore_errors = true)
-
+ proxy = lava_proxy if use_cache else None
+ image_file = download(image, tarball_dir, proxy)
+ image_file = self.decompress(image_file)
+ boot_tgz, root_tgz = self._generate_tarballs(image_file)
except CriticalError:
raise
except:
@@ -525,29 +428,21 @@
:param pkg_url: url of the custom kernel tarball to download
:param use_cache: whether or not to use the cached copy (if it exists)
"""
- lava_cachedir = self.context.lava_cachedir
+ lava_proxy = self.context.lava_proxy
LAVA_IMAGE_TMPDIR = self.context.lava_image_tmpdir
self.tarball_dir = mkdtemp(dir=LAVA_IMAGE_TMPDIR)
tarball_dir = self.tarball_dir
os.chmod(tarball_dir, 0755)
logging.info("Downloading the image files")
- if use_cache:
- boot_path = download_with_cache(boot_url, tarball_dir, lava_cachedir)
- system_path = download_with_cache(system_url, tarball_dir, lava_cachedir)
- data_path = download_with_cache(data_url, tarball_dir, lava_cachedir)
- if pkg_url:
- pkg_path = download_with_cache(pkg_url, tarball_dir)
- else:
- pkg_path = None
+ proxy = lava_proxy if use_cache else None
+ boot_path = download(boot_url, tarball_dir, proxy)
+ system_path = download(system_url, tarball_dir, proxy)
+ data_path = download(data_url, tarball_dir, proxy)
+ if pkg_url:
+ pkg_path = download(pkg_url, tarball_dir, proxy)
else:
- boot_path = download(boot_url, tarball_dir)
- system_path = download(system_url, tarball_dir)
- data_path = download(data_url, tarball_dir)
- if pkg_url:
- pkg_path = download(pkg_url, tarball_dir)
- else:
- pkg_path = None
+ pkg_path = None
logging.info("Downloaded the image files")
return boot_path, system_path, data_path, pkg_path
@@ -567,6 +462,7 @@
self._in_master_shell(300)
self.proc.sendline('export PS1="$PS1 [rc=$(echo \$?)]: "')
self.proc.expect(self.master_str, timeout=10, lava_no_logging=1)
+ self.setup_proxy(self.master_str)
logging.info("System is in master image now")
def _format_testpartition(self, session, fstype):
@@ -651,8 +547,7 @@
while True:
try:
- result_path = download(
- result_tarball, tarball_dir,False)
+ result_path = download(result_tarball, tarball_dir)
return 'pass', '', result_path
except RuntimeError:
tries += 1
@@ -682,7 +577,8 @@
with self._master_session() as master_session:
directory = '/mnt/' + partition
master_session.run('mkdir -p %s' % directory)
- master_session.run('mount /dev/disk/by-label/%s %s' % (partition, directory))
+ master_session.run('mount /dev/disk/by-label/%s %s' % (
+ partition, directory))
master_session.run(
'cp -f %s/etc/resolv.conf %s/etc/resolv.conf.bak' % (
directory, directory))
=== modified file 'lava_dispatcher/context.py'
@@ -56,6 +56,13 @@
return self.config.get("LAVA_SERVER_IP")
@property
+ def lava_proxy(self):
+ proxy = self.config.get("LAVA_PROXY", "")
+ if proxy == "":
+ proxy = None
+ return proxy
+
+ @property
def lava_image_tmpdir(self):
return self.config.get("LAVA_IMAGE_TMPDIR")
@@ -77,7 +84,3 @@
@property
def lava_result_dir(self):
return self.config.get("LAVA_RESULT_DIR")
-
- @property
- def lava_cachedir(self):
- return self.config.get("LAVA_CACHEDIR")
=== modified file 'lava_dispatcher/default-config/lava-dispatcher/README'
@@ -15,7 +15,9 @@
* lava-dispatcher.conf
This file defines global settings of the dispatcher. You will
- almost certainly need to customize LAVA_SERVER_IP for your install.
+ almost certainly need to customize LAVA_SERVER_IP and LAVA_PROXY
+ for your install. LAVA_PROXY could be empty if no proxy server
+ available.
* device-defaults.conf
@@ -42,4 +44,4 @@
* logging.conf
- This file defines settings for Python logging
\ No newline at end of file
+ This file defines settings for Python logging
=== modified file 'lava_dispatcher/default-config/lava-dispatcher/lava-dispatcher.conf'
@@ -5,6 +5,10 @@
# This is the IP the device downloads the image parts from.
LAVA_SERVER_IP = 192.168.1.10
+# This is the address and port of cache proxy service, format is like:
+# LAVA_PROXY = http://192.168.1.10:3128/
+LAVA_PROXY =
+
# Location for rootfs/boot tarballs extracted from images
LAVA_IMAGE_TMPDIR = /linaro/images/tmp
@@ -14,9 +18,6 @@
# Location on the device for storing test results.
LAVA_RESULT_DIR = /lava/results
-# Location for caching downloaded artifacts such as hwpacks and images
-LAVA_CACHEDIR = /linaro/images/cache
-
# Python logging level to use
# 10 = DEBUG
# 20 = INFO
@@ -30,4 +31,4 @@
# The qemu command to use. Called 'default_qemu_binary' because we
# want to allow testing custom qemu binaries soon.
-default_qemu_binary = qemu
\ No newline at end of file
+default_qemu_binary = qemu
=== modified file 'lava_dispatcher/utils.py'
@@ -30,14 +30,19 @@
import pexpect
-def download(url, path="", verbose_failure=1):
+def download(url, path="", proxy=None, verbose_failure=1):
urlpath = urlparse.urlsplit(url).path
filename = os.path.basename(urlpath)
if path:
filename = os.path.join(path, filename)
fd = open(filename, "w")
try:
- response = urllib2.urlopen(urllib2.quote(url, safe=":/"), timeout=30)
+ if proxy:
+ handlers = [urllib2.ProxyHandler({'http': '%s' % proxy})]
+ else:
+ handlers = []
+ opener = urllib2.build_opener(*handlers)
+ response = opener.open(urllib2.quote(url, safe=":/"), timeout=30)
fd = open(filename, 'wb')
shutil.copyfileobj(response, fd, 0x10000)
fd.close()
@@ -48,46 +53,9 @@
raise RuntimeError("Could not retrieve %s" % url)
return filename
-def link_or_copy_file(src, dest):
- try:
- dir = os.path.dirname(dest)
- if not os.path.exists(dir):
- os.makedirs(dir)
- os.link(src, dest)
- except OSError, err:
- if err.errno == errno.EXDEV:
- shutil.copy(src, dest)
- if err.errno == errno.EEXIST:
- logging.debug("Cached copy of %s already exists" % dest)
- else:
- logging.exception("os.link '%s' with '%s' failed" % (src, dest))
-
-def copy_file(src, dest):
- dir = os.path.dirname(dest)
- if not os.path.exists(dir):
- os.makedirs(dir)
- shutil.copy(src, dest)
-
-
# XXX: duplication, we have similar code in lava-test, we need to move that to
# lava.utils -> namespace as standalone package
-def download_with_cache(url, path="", cachedir=""):
- cache_loc = url_to_cache(url, cachedir)
- if os.path.exists(cache_loc):
- filename = os.path.basename(cache_loc)
- file_location = os.path.join(path, filename)
- link_or_copy_file(cache_loc, file_location)
- else:
- file_location = download(url, path)
- copy_file(file_location, cache_loc)
- return file_location
-
-
-def url_to_cache(url, cachedir):
- url_parts = urlparse.urlsplit(url)
- path = os.path.join(cachedir, url_parts.netloc,
- url_parts.path.lstrip(os.sep))
- return path
+# def download_with_cache(url, path="", cachedir=""):
def string_to_list(string):