From patchwork Tue Sep 11 15:42:10 2012 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: James Tunnicliffe X-Patchwork-Id: 11326 Return-Path: X-Original-To: patchwork@peony.canonical.com Delivered-To: patchwork@peony.canonical.com Received: from fiordland.canonical.com (fiordland.canonical.com [91.189.94.145]) by peony.canonical.com (Postfix) with ESMTP id 099BB23E41 for ; Tue, 11 Sep 2012 15:42:15 +0000 (UTC) Received: from mail-ie0-f180.google.com (mail-ie0-f180.google.com [209.85.223.180]) by fiordland.canonical.com (Postfix) with ESMTP id 56867A1A1A1 for ; Tue, 11 Sep 2012 15:42:14 +0000 (UTC) Received: by ieak11 with SMTP id k11so1112332iea.11 for ; Tue, 11 Sep 2012 08:42:13 -0700 (PDT) X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=20120113; h=x-forwarded-to:x-forwarded-for:delivered-to:received-spf :content-type:mime-version:x-launchpad-project:x-launchpad-branch :x-launchpad-message-rationale:x-launchpad-branch-revision-number :x-launchpad-notification-type:to:from:subject:message-id:date :reply-to:sender:errors-to:precedence:x-generated-by :x-launchpad-hash:x-gm-message-state; bh=fAn0Kfalj2swXtwL85YVtB+/Slkx6vDicvvuGmIAfGU=; b=U+rMfzLyrRDo14HGGJwDEK67Hesz9umCCqmmzwhz8y0BAy6J6gig44EdCYaQDpnaSV LXqLOiDw0HA/eh2ZNl3gOYXajwaHOjW2k8sZFm+3WMGSZpcK94hKrBsy5HU+EWvu9slG KTwhrUlhxeFkiR5lUrE8BD9iMMWc967MX1JZtnvfuAkUHOU4ayJmFS/db6V+AYbEy1cv ZivWS17HgfcLKGDHSiPTBOpX0MfX8oqHwJlUagZ2XZaXt+UOKABx7BsFccdtVByZBp91 kS1G/41QEntGfVMwQ6Tszu/DVOqXAkVz/5ayWqBE2EXfNkX1EYWJhNefTcV0bqTEdzSG 9vpA== Received: by 10.50.45.162 with SMTP id o2mr17617096igm.0.1347378133692; Tue, 11 Sep 2012 08:42:13 -0700 (PDT) X-Forwarded-To: linaro-patchwork@canonical.com X-Forwarded-For: patch@linaro.org linaro-patchwork@canonical.com Delivered-To: patches@linaro.org Received: by 10.50.184.232 with SMTP id ex8csp20467igc; Tue, 11 Sep 2012 08:42:12 -0700 (PDT) Received: by 10.216.242.196 with SMTP id i46mr9933378wer.140.1347378131679; Tue, 11 Sep 2012 08:42:11 -0700 (PDT) Received: from indium.canonical.com (indium.canonical.com. [91.189.90.7]) by mx.google.com with ESMTPS id e47si23451533wep.3.2012.09.11.08.42.10 (version=TLSv1/SSLv3 cipher=OTHER); Tue, 11 Sep 2012 08:42:11 -0700 (PDT) Received-SPF: pass (google.com: best guess record for domain of bounces@canonical.com designates 91.189.90.7 as permitted sender) client-ip=91.189.90.7; Authentication-Results: mx.google.com; spf=pass (google.com: best guess record for domain of bounces@canonical.com designates 91.189.90.7 as permitted sender) smtp.mail=bounces@canonical.com Received: from ackee.canonical.com ([91.189.89.26]) by indium.canonical.com with esmtp (Exim 4.71 #1 (Debian)) id 1TBSbK-00016L-QM for ; Tue, 11 Sep 2012 15:42:10 +0000 Received: from ackee.canonical.com (localhost [127.0.0.1]) by ackee.canonical.com (Postfix) with ESMTP id BA871E0148 for ; Tue, 11 Sep 2012 15:42:10 +0000 (UTC) MIME-Version: 1.0 X-Launchpad-Project: linaro-image-tools X-Launchpad-Branch: ~linaro-image-tools/linaro-image-tools/trunk X-Launchpad-Message-Rationale: Subscriber X-Launchpad-Branch-Revision-Number: 560 X-Launchpad-Notification-Type: branch-revision To: Linaro Patch Tracker From: noreply@launchpad.net Subject: [Branch ~linaro-image-tools/linaro-image-tools/trunk] Rev 560: Add support for copy_files field in metadata Message-Id: <20120911154210.10664.54738.launchpad@ackee.canonical.com> Date: Tue, 11 Sep 2012 15:42:10 -0000 Reply-To: noreply@launchpad.net Sender: bounces@canonical.com Errors-To: bounces@canonical.com Precedence: bulk X-Generated-By: Launchpad (canonical.com); Revision="15930"; Instance="launchpad-lazr.conf" X-Launchpad-Hash: ac465b3932f1ff3d1468d1d99f9441facd573611 X-Gm-Message-State: ALoCoQk4pCxyjkVfjpjO2M7dg6mYW1xA0zS+H31lM+FDkg18smSX1Nqx4tUXgD6zXqnpXM0lk2P4 Merge authors: James Tunnicliffe (dooferlad) Related merge proposals: https://code.launchpad.net/~dooferlad/linaro-image-tools/copy-files-new-syntax/+merge/123761 proposed by: James Tunnicliffe (dooferlad) review: Approve - Paul Sokolovsky (pfalcon) ------------------------------------------------------------ revno: 560 [merge] committer: James Tunnicliffe branch nick: linaro-image-tools timestamp: Tue 2012-09-11 16:41:47 +0100 message: Add support for copy_files field in metadata modified: linaro_image_tools/hwpack/builder.py linaro_image_tools/hwpack/config.py linaro_image_tools/hwpack/tests/test_builder.py linaro_image_tools/media_create/boards.py linaro_image_tools/media_create/tests/test_media_create.py --- lp:linaro-image-tools https://code.launchpad.net/~linaro-image-tools/linaro-image-tools/trunk You are subscribed to branch lp:linaro-image-tools. To unsubscribe from this branch go to https://code.launchpad.net/~linaro-image-tools/linaro-image-tools/trunk/+edit-subscription === modified file 'linaro_image_tools/hwpack/builder.py' --- linaro_image_tools/hwpack/builder.py 2012-09-03 17:12:23 +0000 +++ linaro_image_tools/hwpack/builder.py 2012-09-11 14:28:20 +0000 @@ -37,29 +37,13 @@ ) from linaro_image_tools.hwpack.hwpack_fields import ( - FILE_FIELD, PACKAGE_FIELD, - SPL_FILE_FIELD, SPL_PACKAGE_FIELD, - COPY_FILES_FIELD, ) # The fields that hold packages to be installed. PACKAGE_FIELDS = [PACKAGE_FIELD, SPL_PACKAGE_FIELD] -# Specification of files (boot related) to extract: -# : (, -# ) -# if is None, it will be for -# global bootloader, or - for board-specific -# bootloader -EXTRACT_FILES = {FILE_FIELD: (PACKAGE_FIELD, None), - SPL_FILE_FIELD: (SPL_PACKAGE_FIELD, None), - COPY_FILES_FIELD: (PACKAGE_FIELD, None)} - - logger = logging.getLogger(__name__) - - LOCAL_ARCHIVE_LABEL = 'hwpack-local' @@ -166,57 +150,72 @@ # Eliminate duplicates. return list(set(boot_packages)) - def extract_bootloader_files(self, board, bootloader_name, - bootloader_conf): - for key, value in bootloader_conf.iteritems(): - if key in EXTRACT_FILES: - package_field, dest_path = EXTRACT_FILES[key] - if not dest_path: - dest_path = bootloader_name - if board: - dest_path += "-" + board - # Dereference package field to get actual package name - package = bootloader_conf.get(package_field) - src_files = value - - # Process scalar and list fields consistently below - field_value_scalar = False - if type(src_files) != type([]): - src_files = [src_files] - field_value_scalar = True - - package_ref = self.find_fetched_package( - self.packages, package) - added_files = [] - for f in src_files: - added_files.append(self.add_file_to_hwpack( - package_ref, f, dest_path)) - # Store within-hwpack file paths with the same - # scalar/list type as original field. - if field_value_scalar: - assert len(added_files) == 1 - added_files = added_files[0] - bootloader_conf[key] = added_files - - def extract_files(self, config_dictionary, is_bootloader_config, - board=None): - """Extract (boot) files based on EXTRACT_FILES spec and put - them into hwpack.""" - self.remove_packages = [] - if is_bootloader_config: - for bootl_name, bootl_conf in config_dictionary.iteritems(): - self.extract_bootloader_files(board, bootl_name, bootl_conf) - else: - # This is board config - for board, board_conf in config_dictionary.iteritems(): - bootloaders = board_conf['bootloaders'] - self.extract_files(bootloaders, True, board) - - # Clean up no longer needed packages. - for package in self.remove_packages: - if package in self.packages: - self.packages.remove(package) - self.remove_packages = [] + def do_extract_file(self, package, source_path, dest_path): + """Extract specified file from package to dest_path.""" + package_ref = self.find_fetched_package(self.packages, package) + return self.add_file_to_hwpack(package_ref, source_path, dest_path) + + def do_extract_files(self): + """Go through a bootloader config, search for files to extract.""" + base_dest_path = "" + if self.config.board: + base_dest_path = self.config.board + base_dest_path = os.path.join(base_dest_path, self.config.bootloader) + # Extract bootloader file + if self.config.bootloader_package and self.config.bootloader_file: + dest_path = os.path.join(base_dest_path, + os.path.dirname(self.config.bootloader_file)) + self.do_extract_file(self.config.bootloader_package, + self.config.bootloader_file, + dest_path) + + # Extract SPL file + if self.config.spl_package and self.config.spl_file: + dest_path = os.path.join(base_dest_path, + os.path.dirname(self.config.spl_file)) + self.do_extract_file(self.config.spl_package, + self.config.spl_file, + dest_path) + + def foreach_boards_and_bootloaders(self, function): + """Call function for each board + bootloader combination in metadata""" + if self.config.bootloaders is not None: + for bootloader in self.config.bootloaders: + self.config.set_board(None) + self.config.set_bootloader(bootloader) + function() + + if self.config.boards is not None: + for board in self.config.boards: + if self.config.bootloaders is not None: + for bootloader in self.config.bootloaders: + self.config.set_board(board) + self.config.set_bootloader(bootloader) + function() + + def extract_files(self): + """Find bootloaders in config that may contain files to extract.""" + if float(self.config.format.format_as_string) < 3.0: + # extract files was introduced in version 3 configurations and is + # a null operation for earlier configuration files + return + + self.foreach_boards_and_bootloaders(self.do_extract_files) + + def do_find_copy_files_packages(self): + """Find packages referenced by copy_files (single board, bootloader)""" + copy_files = self.config.bootloader_copy_files + if copy_files: + self.copy_files_packages.extend(copy_files.keys()) + + def find_copy_files_packages(self): + """Find all packages referenced by copy_files sections in metadata.""" + self.copy_files_packages = [] + self.foreach_boards_and_bootloaders( + self.do_find_copy_files_packages) + packages = self.copy_files_packages + del(self.copy_files_packages) + return packages def build(self): for architecture in self.config.architectures: @@ -242,6 +241,8 @@ if self.config.boards is not None: self.packages.extend(self.find_bootloader_packages( self.config.boards)) + + self.packages.extend(self.find_copy_files_packages()) else: if self.config.bootloader_package is not None: self.packages.append(self.config.bootloader_package) @@ -268,14 +269,9 @@ # On a v3 hwpack, all the values we need to check are # in the bootloaders and boards section, so we loop # through both of them changing what is necessary. + if self.config.format.format_as_string == '3.0': - if self.config.bootloaders is not None: - self.extract_files(self.config.bootloaders, - True) - metadata.bootloaders = self.config.bootloaders - if self.config.boards is not None: - self.extract_files(self.config.boards, False) - metadata.boards = self.config.boards + self.extract_files() else: bootloader_package = None if self.config.bootloader_file is not None: === modified file 'linaro_image_tools/hwpack/config.py' --- linaro_image_tools/hwpack/config.py 2012-08-30 10:39:02 +0000 +++ linaro_image_tools/hwpack/config.py 2012-09-11 14:28:20 +0000 @@ -21,6 +21,7 @@ import ConfigParser from operator import attrgetter +import os import re import string import yaml @@ -116,6 +117,8 @@ translate_v2_to_v3[BOOTLOADER_IN_BOOT_PART_KEY] = IN_BOOT_PART_FIELD BOOTLOADER_DD_KEY = 'u_boot_dd' translate_v2_to_v3[BOOTLOADER_DD_KEY] = DD_FIELD + last_used_keys = [] + board = None def __init__(self, fp, bootloader=None, board=None, allow_unset_bootloader=False): @@ -354,9 +357,56 @@ return self._get_bootloader_option(SPL_IN_BOOT_PART_FIELD) @property - def boot_copy_files(self): - """Extra files to copy to boot partition.""" - return self._get_bootloader_option(COPY_FILES_FIELD) + def bootloader_copy_files(self): + """Extra files to copy to boot partition. + + This can be stored in several formats. We always present in a common + one: {source_package: [{source_file_path: dest_file_path}]. + dest_file_path (in the above example) is always absolute. + """ + #copy_files: + # source_package: + # - source_file_path : dest_file_path + # - source_file_without_explicit_destination + #copy_files: + # - file1 + # - file2: dest_path + # + # Note that the list of files is always that - a list. + + copy_files = self._get_bootloader_option(COPY_FILES_FIELD) + + if copy_files is None: + return None + + if not isinstance(copy_files, dict): + copy_files = {self.bootloader_package: copy_files} + + for package in copy_files: + new_list = [] + for value in copy_files[package]: + if not isinstance(value, dict): + dest_path = "/boot" + source_path = value + else: + if len(value.keys()) > 1: + raise HwpackConfigError("copy_files entry found with" + "more than one destination") + source_path = value.keys()[0] + dest_path = value[source_path] + + if not dest_path.startswith("/boot"): + # Target path should be relative, or start with /boot - we + # don't support to copying to anywhere other than /boot. + if dest_path[0] == "/": + raise HwpackConfigError("copy_files destinations must" + "be relative to /boot or start with /boot.") + dest_path = os.path.join("/boot", dest_path) + + new_list.append({source_path: dest_path}) + copy_files[package] = new_list + + return copy_files @property def spl_dd(self): @@ -426,8 +476,22 @@ key = self._v2_key_to_v3(key) if result is not None: result = result.get(key, None) + self.last_used_keys = keys return result + def get_last_used_keys(self): + """Used so you can work out which boards + boot loader was used. + + Configuration data is stored in a dictionary. This returns a list of + keys used to traverse into the dictionary the last time an item was + looked up. + + This can be used to see where a bit of information came from - we + store data that may be indexed differently depending on which board + and bootloader are set. + """ + return self.last_used_keys + def get_option(self, name): """Return the value of an attribute by name. === modified file 'linaro_image_tools/hwpack/tests/test_builder.py' --- linaro_image_tools/hwpack/tests/test_builder.py 2012-09-04 10:42:27 +0000 +++ linaro_image_tools/hwpack/tests/test_builder.py 2012-09-11 14:28:20 +0000 @@ -31,7 +31,7 @@ HardwarePackBuilder, logger as builder_logger, ) -from linaro_image_tools.hwpack.config import HwpackConfigError, Config +from linaro_image_tools.hwpack.config import HwpackConfigError from linaro_image_tools.hwpack.hardwarepack import Metadata from linaro_image_tools.hwpack.packages import ( FetchedPackage, @@ -55,7 +55,6 @@ MockSomethingFixture, MockCmdRunnerPopenFixture, ) -from StringIO import StringIO class ConfigFileMissingTests(TestCase): @@ -431,11 +430,17 @@ Equals("Local package 'bar' not included")) def test_global_and_board_bootloader(self): - package_names = ['package0', 'package1'] - files = {package_names[0]: - ["usr/lib/u-boot/omap4_panda/u-boot.img", - "usr/share/doc/u-boot-linaro-omap4-panda/copyright"], - package_names[1]: ["usr/lib/u-boot/omap4_panda/u-boot.img"]} + package_names = ['package0', 'package1', 'package2', 'package3'] + files = { + package_names[0]: + ["usr/lib/u-boot/omap4_panda/u-boot.img", + "usr/share/doc/u-boot-linaro-omap4-panda/copyright"], + package_names[1]: + ["usr/lib/u-boot/omap4_panda/u-boot.img", + "some/path/file"], + package_names[2]: [], + package_names[3]: [], + } config_v3 = self.config_v3 + "\n".join([ "bootloaders:", @@ -443,13 +448,17 @@ self.bootloader_config, " file: " + files[package_names[0]][0], " copy_files:", - " - " + files[package_names[0]][1], + " " + package_names[2] + ":", + " - some_file", "boards:", " board1:", " bootloaders:", " u_boot:", " package: %s", " file: " + files[package_names[1]][0], + " copy_files:", + " " + package_names[3] + ":", + " - some_file", " in_boot_part: true", "sources:", " ubuntu: %s"]) @@ -480,10 +489,6 @@ config_file_fixture = self.useFixture(ConfigFileFixture(config_v3)) - # Parse the config - config = Config(StringIO(config_v3)) - config.set_bootloader("u_boot") - # Build a hardware pack builder = HardwarePackBuilder( config_file_fixture.filename, "1.0", @@ -491,6 +496,9 @@ for package in available_packages]) builder.build() + stored_package_names = [p.name for p in builder.packages] + for package_name in package_names: + self.assertIn(package_name, stored_package_names) # Read the contents of the hardware pack, making sure it is as expected tf = tarfile.open("hwpack_ahwpack_1.0_armel.tar.gz", mode="r:gz") @@ -499,11 +507,9 @@ # files this is " " so they can be # uniquely identified. expected_files = [ - ("u_boot/u-boot.img", + ("u_boot/" + files[package_names[0]][0], package_names[0] + " " + files[package_names[0]][0]), - ("u_boot/copyright", - package_names[0] + " " + files[package_names[0]][1]), - ("u_boot-board1/u-boot.img", + ("board1/u_boot/" + files[package_names[1]][0], package_names[1] + " " + files[package_names[1]][0])] for expected_file, contents in expected_files: === modified file 'linaro_image_tools/media_create/boards.py' --- linaro_image_tools/media_create/boards.py 2012-09-05 09:29:14 +0000 +++ linaro_image_tools/media_create/boards.py 2012-09-10 17:50:26 +0000 @@ -37,6 +37,7 @@ import string import logging from linaro_image_tools.hwpack.config import Config +from linaro_image_tools.hwpack.builder import PackageUnpacker from parted import Device @@ -128,6 +129,7 @@ self.hwpack_tarfiles = [] self.bootloader = bootloader self.board = board + self.tempdirs = {} class FakeSecHead(object): """ Add a fake section header to the metadata file. @@ -162,9 +164,15 @@ if self.tempdir is not None and os.path.exists(self.tempdir): shutil.rmtree(self.tempdir) - def get_field(self, field): + for name in self.tempdirs: + tempdir = self.tempdirs[name] + if tempdir is not None and os.path.exists(tempdir): + shutil.rmtree(tempdir) + + def get_field(self, field, return_keys=False): data = None hwpack_with_data = None + keys = None for hwpack_tarfile in self.hwpack_tarfiles: metadata = hwpack_tarfile.extractfile(self.metadata_filename) lines = metadata.readlines() @@ -181,9 +189,13 @@ new_data) data = new_data hwpack_with_data = hwpack_tarfile + if return_keys: + keys = parser.get_last_used_keys() except ConfigParser.NoOptionError: continue + if return_keys: + return data, hwpack_with_data, keys return data, hwpack_with_data def get_format(self): @@ -207,7 +219,8 @@ file reference(s) :return: path to a file or list of paths to files """ - file_names, hwpack_tarfile = self.get_field(file_alias) + file_names, hwpack_tarfile, keys = self.get_field(file_alias, + return_keys=True) if not file_names: return file_names single = False @@ -215,7 +228,24 @@ single = True file_names = [file_names] out_files = [] + + # Depending on if board and/or bootloader were used to look up the + # file we are getting, we need to prepend those names to the path + # to get the correct extracted file from the hardware pack. + config_names = [("board", "boards"), ("bootloader", "bootloaders")] + base_path = "" + if keys: + # If keys is non-empty, we have a V3 config option that was + # modified by the bootloader and/or boot option... + for name, key in config_names: + if self.get_field(name): + value = self.get_field(name)[0] + if keys[0] == key: + base_path = os.path.join(base_path, value) + keys = keys[1:] + for f in file_names: + f = os.path.join(base_path, f) hwpack_tarfile.extract(f, self.tempdir) f = os.path.join(self.tempdir, f) out_files.append(f) @@ -223,6 +253,104 @@ return out_files[0] return out_files + def list_packages(self): + """Return list of (package names, TarFile object containing them)""" + packages = [] + for tf in self.hwpack_tarfiles: + for name in tf.getnames(): + if name.startswith("pkgs/") and name.endswith(".deb"): + packages.append((tf, name)) + return packages + + def find_package_for(self, name, version=None, revision=None, + architecture=None): + """Find a package that matches the name, version, rev and arch given. + + Packages are named according to the debian specification: + http://www.debian.org/doc/manuals/debian-faq/ch-pkg_basics.en.html + _-_.deb + DebianRevisionNumber seems to be optional. + Use this spec to return a package matching the requirements given. + """ + for tar_file, package in self.list_packages(): + file_name = os.path.basename(package) + dpkg_chunks = re.search("^(.+)_(.+)_(.+)\.deb$", + file_name) + assert dpkg_chunks, "Could not split package file name into"\ + "__.deb" + + pkg_name = dpkg_chunks.group(1) + pkg_version = dpkg_chunks.group(2) + pkg_architecture = dpkg_chunks.group(3) + + ver_chunks = re.search("^(.+)-(.+)$", pkg_version) + if ver_chunks: + pkg_version = ver_chunks.group(1) + pkg_revision = ver_chunks.group(2) + else: + pkg_revision = None + + if name != pkg_name: + continue + if version != None and str(version) != pkg_version: + continue + if revision != None and str(revision) != pkg_revision: + continue + if (architecture != None and + str(architecture) != pkg_architecture): + continue + + # Got a matching package - return its path inside the tarball + return tar_file, package + + # Failed to find a matching package - return None + return None + + def get_file_from_package(self, file_path, package_name, + package_version=None, package_revision=None, + package_architecture=None): + """Extract named file from package specified by name, ver, rev, arch. + + File is extracted from the package matching the given specification + to a temporary directory. The absolute path to the extracted file is + returned. + """ + + package_info = self.find_package_for(package_name, + package_version, + package_revision, + package_architecture) + if package_info is None: + return None + tar_file, package = package_info + + # Avoid unpacking hardware pack more than once by assigning each one + # its own tempdir to unpack into. + # TODO: update logic that uses self.tempdir so we can get rid of this + # by sharing nicely. + if not package in self.tempdirs: + self.tempdirs[package] = tempfile.mkdtemp() + tempdir = self.tempdirs[package] + + # We extract everything in the hardware pack so we don't have to worry + # about chasing links (extract a link, find where it points to, extract + # that...). This is slower, but more reliable. + tar_file.extractall(tempdir) + package_path = os.path.join(tempdir, package) + + with PackageUnpacker() as self.package_unpacker: + extracted_file = self.package_unpacker.get_file(package_path, + file_path) + after_tmp = re.sub(self.package_unpacker.tempdir, "", + extracted_file).lstrip("/\\") + extract_dir = os.path.join(tempdir, "extracted", + os.path.dirname(after_tmp)) + os.makedirs(extract_dir) + shutil.move(extracted_file, extract_dir) + extracted_file = os.path.join(extract_dir, + os.path.basename(extracted_file)) + return extracted_file + class BoardConfig(object): board = None @@ -465,6 +593,13 @@ cls.SAMSUNG_V310_BL2_START = (cls.SAMSUNG_V310_ENV_START + cls.SAMSUNG_V310_ENV_LEN) + cls.bootloader_copy_files = cls.hardwarepack_handler.get_field( + "bootloader_copy_files")[0] + + cls.bootloader = cls.hardwarepack_handler.get_field( + "bootloader") + cls.board = board + @classmethod def get_file(cls, file_alias, default=None): # XXX remove the 'default' parameter when V1 support is removed! @@ -793,7 +928,6 @@ bootloader_parts_dir = os.path.join(chroot_dir, parts_dir) cmd_runner.run(['mkdir', '-p', boot_disk]).wait() with partition_mounted(boot_partition, boot_disk): - boot_files = [] with cls.hardwarepack_handler: if cls.bootloader_file_in_boot_part: # @@ -813,22 +947,48 @@ assert bootloader_bin is not None, ( "bootloader binary could not be found") - boot_files.append(bootloader_bin) - - copy_files = cls.get_file('boot_copy_files') - if copy_files: - boot_files.extend(copy_files) - - for f in boot_files: proc = cmd_runner.run( - ['cp', '-v', f, boot_disk], as_root=True) + ['cp', '-v', bootloader_bin, boot_disk], as_root=True) proc.wait() + # Handle copy_files field. + cls.copy_files(boot_disk) + cls.make_boot_files( bootloader_parts_dir, is_live, is_lowmem, consoles, chroot_dir, rootfs_id, boot_disk, boot_device_or_file) @classmethod + def copy_files(cls, boot_disk): + """Handle the copy_files metadata field.""" + + # Extract anything specified by copy_files sections + # self.bootloader_copy_files is always of the form: + # {'source_package': + # [ + # {'source_path': 'dest_path'} + # ] + # } + if cls.bootloader_copy_files is None: + return + + for source_package, file_list in cls.bootloader_copy_files.iteritems(): + for file_info in file_list: + for source_path, dest_path in file_info.iteritems(): + source = cls.hardwarepack_handler.get_file_from_package( + source_path, source_package) + dest_path = dest_path.lstrip("/\\") + dirname = os.path.dirname(dest_path) + dirname = os.path.join(boot_disk, dirname) + if not os.path.exists(dirname): + cmd_runner.run(["mkdir", "-p", dirname], + as_root=True).wait() + proc = cmd_runner.run( + ['cp', '-v', source, + os.path.join(boot_disk, dest_path)], as_root=True) + proc.wait() + + @classmethod def _get_kflavor_files(cls, path): """Search for kernel, initrd and optional dtb in path.""" if cls.kernel_flavors is None: === modified file 'linaro_image_tools/media_create/tests/test_media_create.py' --- linaro_image_tools/media_create/tests/test_media_create.py 2012-09-04 13:53:43 +0000 +++ linaro_image_tools/media_create/tests/test_media_create.py 2012-09-10 17:50:26 +0000 @@ -36,6 +36,7 @@ from testtools import TestCase from linaro_image_tools import cmd_runner +from linaro_image_tools.hwpack.packages import PackageMaker import linaro_image_tools.media_create from linaro_image_tools.media_create import ( android_boards, @@ -124,6 +125,7 @@ ) from linaro_image_tools.utils import find_command, preferred_tools_dir +from linaro_image_tools.hwpack.testing import ContextManagerFixture chroot_args = " ".join(cmd_runner.CHROOT_ARGS) sudo_args = " ".join(cmd_runner.SUDO_ARGS) @@ -309,36 +311,95 @@ test_file = hp.get_file('bootloader_file') self.assertEquals(data, open(test_file, 'r').read()) - def test_get_file_v3(self): - # Test that get_file() works as expected with hwpackv3 and - # supports its new file fields. - metadata = textwrap.dedent("""\ - format: 3.0 - name: ahwpack - version: 4 - architecture: armel - origin: linaro - bootloaders: - u_boot: - file: a_file - copy_files: - - file1 - - file2 - uefi: - file: b_file - """) - files = {'FORMAT': '3.0\n', 'metadata': metadata, - 'a_file': 'a_file content', 'file1': 'file1 content', - 'file2': 'file2 content'} - tarball = self.add_to_tarball(files.items()) - hp = HardwarepackHandler([tarball], bootloader='u_boot') - with hp: - test_file = hp.get_file('bootloader_file') - self.assertEquals(files['a_file'], open(test_file, 'r').read()) - test_files = hp.get_file('boot_copy_files') - self.assertEquals(len(test_files), 2) - self.assertEquals(files['file1'], open(test_files[0], 'r').read()) - self.assertEquals(files['file2'], open(test_files[1], 'r').read()) + def test_list_packages(self): + metadata = ("format: 3.0\nname: ahwpack\nversion: 4\narchitecture: " + "armel\norigin: linaro\n") + format = "3.0\n" + tarball = self.add_to_tarball([ + ("FORMAT", format), + ("metadata", metadata), + ("pkgs/foo_1-1_all.deb", ''), + ("pkgs/bar_1-1_all.deb", ''), + ]) + + hp = HardwarepackHandler([tarball], board='panda', bootloader='uefi') + with hp: + packages = hp.list_packages() + names = [p[1] for p in packages] + self.assertIn('pkgs/foo_1-1_all.deb', names) + self.assertIn('pkgs/bar_1-1_all.deb', names) + self.assertEqual(len(packages), 2) + + def test_find_package_for(self): + metadata = ("format: 3.0\nname: ahwpack\nversion: 4\narchitecture: " + "armel\norigin: linaro\n") + format = "3.0\n" + tarball = self.add_to_tarball([ + ("FORMAT", format), + ("metadata", metadata), + ("pkgs/foo_1-3_all.deb", ''), + ("pkgs/foo_2-5_arm.deb", ''), + ("pkgs/bar_1-3_arm.deb", ''), + ]) + + hp = HardwarepackHandler([tarball], board='panda', bootloader='uefi') + with hp: + self.assertEqual(hp.find_package_for("foo")[1], + "pkgs/foo_1-3_all.deb") + self.assertEqual(hp.find_package_for("bar")[1], + "pkgs/bar_1-3_arm.deb") + self.assertEqual(hp.find_package_for("foo", version=2)[1], + "pkgs/foo_2-5_arm.deb") + self.assertEqual(hp.find_package_for("foo", version=2, + revision=5)[1], + "pkgs/foo_2-5_arm.deb") + self.assertEqual(hp.find_package_for("foo", version=2, revision=5, + architecture="arm")[1], + "pkgs/foo_2-5_arm.deb") + self.assertEqual(hp.find_package_for("foo", architecture="arm")[1], + "pkgs/foo_2-5_arm.deb") + self.assertEqual(hp.find_package_for("foo", architecture="all")[1], + "pkgs/foo_1-3_all.deb") + + def test_get_file_from_package(self): + metadata = ("format: 3.0\nname: ahwpack\nversion: 4\narchitecture: " + "armel\norigin: linaro\n") + format = "3.0\n" + + names = ['package0', 'package1', 'package2'] + files = {names[0]: + ["usr/lib/u-boot/omap4_panda/u-boot.img", + "usr/share/doc/u-boot-linaro-omap4-panda/copyright"], + names[1]: ["usr/lib/u-boot/omap4_panda/u-boot2.img", + "foo/bar", + "flim/flam"], + names[2]: ["some/path/config"]} + + # Generate some test packages + maker = PackageMaker() + self.useFixture(ContextManagerFixture(maker)) + + tarball_content = [("FORMAT", format), ("metadata", metadata)] + + package_names = [] + for package_name in names: + # The files parameter to make_package is a list of files to create. + # These files are text files containing package_name and their + # path. Since package_name is different for each package, this + # gives each file a unique content. + deb_file_path = maker.make_package(package_name, '1.0', {}, + files=files[package_name]) + name = os.path.basename(deb_file_path) + tarball_content.append((os.path.join("pkgs", name), + open(deb_file_path).read())) + package_names.append(name) + + tarball = self.add_to_tarball(tarball_content) + + hp = HardwarepackHandler([tarball], board='panda', bootloader='uefi') + with hp: + path = hp.get_file_from_package("some/path/config", "package2") + self.assertTrue(path.endswith("some/path/config")) class TestSetMetadata(TestCaseWithFixtures): @@ -574,6 +635,23 @@ self.assertRaises( AssertionError, config.set_metadata, 'ahwpack.tar.gz') + def test_sets_copy_files(self): + self.useFixture(MockSomethingFixture( + linaro_image_tools.media_create.boards, 'HardwarepackHandler', + self.MockHardwarepackHandler)) + field_to_test = 'bootloader_copy_files' + data_to_set = {'package': + [{"source1": "dest1"}, + {"source2": "dest2"}]} + self.MockHardwarepackHandler.metadata_dict = { + field_to_test: data_to_set, + } + + class config(BoardConfig): + pass + config.set_metadata('ahwpack.tar.gz') + self.assertEquals(data_to_set, config.bootloader_copy_files) + class TestGetMLOFile(TestCaseWithFixtures): @@ -2937,6 +3015,16 @@ self.popen_fixture = self.useFixture(MockCmdRunnerPopenFixture()) self.useFixture(MockSomethingFixture( self.config, 'make_boot_files', self.save_args)) + self.config.hardwarepack_handler.get_file_from_package = \ + self.get_file_from_package + self.config.bootloader_copy_files = None + + def get_file_from_package(self, source_path, source_package): + if source_package in self.config.bootloader_copy_files: + for file_info in self.config.bootloader_copy_files[source_package]: + if source_path in file_info: + return source_path + return None def prepare_config_v3(self, config): class c(config): @@ -2949,6 +3037,13 @@ self.config.hardwarepack_handler.get_format = lambda: '3.0' self.config.hardwarepack_handler.get_file = \ lambda file_alias: ['file1', 'file2'] + self.config.hardwarepack_handler.get_file_from_package = \ + self.get_file_from_package + self.config.bootloader_copy_files = { + "package1": + [{"file1": "/boot/"}, + {"file2": "/boot/grub/renamed"}]} + self.popen_fixture = self.useFixture(MockCmdRunnerPopenFixture()) self.useFixture(MockSomethingFixture( self.config, 'make_boot_files', self.save_args)) @@ -2984,6 +3079,7 @@ self.prepare_config(boards.BoardConfig) self.config.bootloader_flavor = "bootloader_flavor" self.config.bootloader_file_in_boot_part = True + self.config.bootloader = "u_boot" self.call_populate_boot(self.config) expected_calls = self.expected_calls[:] expected_calls.insert(2, @@ -3003,7 +3099,7 @@ expected_calls, self.popen_fixture.mock.commands_executed) self.assertEquals(self.expected_args, self.saved_args) - def test_populate_boot_copy_files(self): + def test_populate_bootloader_copy_files(self): self.prepare_config_v3(boards.BoardConfig) self.config.bootloader_flavor = "bootloader_flavor" # Test that copy_files works per spec (puts stuff in boot partition) @@ -3011,12 +3107,35 @@ self.config.bootloader_file_in_boot_part = False self.call_populate_boot(self.config) expected_calls = self.expected_calls[:] - expected_calls.insert(2, + expected_calls.insert(2, '%s mkdir -p boot_disk/boot' % sudo_args) + expected_calls.insert(3, '%s cp -v file1 ' - 'boot_disk' % sudo_args) - expected_calls.insert(3, + 'boot_disk/boot/' % sudo_args) + expected_calls.insert(4, '%s mkdir -p boot_disk/boot/grub' % sudo_args) + expected_calls.insert(5, '%s cp -v file2 ' - 'boot_disk' % sudo_args) + 'boot_disk/boot/grub/renamed' % sudo_args) + self.assertEquals( + expected_calls, self.popen_fixture.mock.commands_executed) + self.assertEquals(self.expected_args, self.saved_args) + + def test_populate_bootloader_copy_files_bootloader_set(self): + self.prepare_config_v3(boards.BoardConfig) + self.config.bootloader_flavor = "bootloader_flavor" + # Test that copy_files works per spec (puts stuff in boot partition) + # even if bootloader not in_boot_part. + self.config.bootloader_file_in_boot_part = False + self.config.bootloader = "u_boot" + self.call_populate_boot(self.config) + expected_calls = self.expected_calls[:] + expected_calls.insert(2, '%s mkdir -p boot_disk/boot' % sudo_args) + expected_calls.insert(3, + '%s cp -v file1 ' + 'boot_disk/boot/' % sudo_args) + expected_calls.insert(4, '%s mkdir -p boot_disk/boot/grub' % sudo_args) + expected_calls.insert(5, + '%s cp -v file2 ' + 'boot_disk/boot/grub/renamed' % sudo_args) self.assertEquals( expected_calls, self.popen_fixture.mock.commands_executed) self.assertEquals(self.expected_args, self.saved_args)