=== modified file 'fetch_image_ui.py'
@@ -1264,10 +1264,8 @@
#--- Event(s) ---
def event_start(self, event):
- if event == "download OS":
- self.downloading_files_status.SetLabel("Downloading OS")
- elif event == "download hwpack":
- self.downloading_files_status.SetLabel("Downloading Hardware Pack")
+ if event == "download":
+ self.downloading_files_status.SetLabel("Downloading")
elif event == "unpack":
self.unpacking_files_status.SetLabel("Running")
elif event == "installing packages":
@@ -1279,7 +1277,7 @@
install_unsigned_packages = self.unsigned_packages_query(packages)
if install_unsigned_packages == False:
- self.file_handler.kill_create_media()
+ # TODO: Tidy up other threads
sys.exit(1)
else:
self.lmc_thread.send_to_create_process("y")
@@ -1292,9 +1290,7 @@
print "Unhandled start event:", event
def event_end(self, event):
- if event == "download OS":
- self.downloading_files_status.SetLabel("Done (1/2)")
- elif event == "download hwpack":
+ if event == "download":
self.downloading_files_status.SetLabel("Done")
elif event == "unpack":
self.unpacking_files_status.SetLabel("Done")
@@ -1309,12 +1305,7 @@
def event_update(self, task, update_type, value):
if task == "download":
- if update_type == "name":
- self.downloading_files_status.SetLabel("Downloading")
- self.old_time = time.time()
- self.old_bytes_downloaded = 0
-
- elif update_type == "progress":
+ if update_type == "progress":
self.total_bytes_downloaded += value
time_difference = time.time() - self.old_time
@@ -1374,12 +1365,17 @@
/ self.total_bytes_to_download)
elif update_type == "total bytes":
+ self.old_time = time.time()
+ self.old_bytes_downloaded = 0
self.total_bytes_to_download = value
self.total_bytes_downloaded = 0
self.speeds = [] # keep an array of speeds used to calculate
# the estimated time remaining - by not just using the
# current speed we can stop the ETA bouncing around too much.
+ elif update_type == "message":
+ self.downloading_files_status.SetLabel(value)
+
def event_combo_box_release(self, evt):
pass
@@ -1543,7 +1539,7 @@
app = wx.PySimpleApp() # Start the application
if app:
pass # We don't use this directly. Stop pyflakes complaining!
-
+
w = TestDriveWizard('Simple Wizard')
return w.go()
=== modified file 'linaro-media-create'
@@ -45,7 +45,7 @@
from linaro_image_tools.utils import (
ensure_command,
is_arm_host,
- verify_file_integrity,
+ check_file_integrity_and_log_errors,
)
# Just define the global variables
@@ -57,7 +57,6 @@
# Registered as the first atexit handler as we want this to be the last
# handler to execute.
-@atexit.register
def cleanup_tempdir():
"""Remove TEMP_DIR with all its contents.
@@ -111,10 +110,15 @@
sig_file_list = args.hwpacksigs[:]
if args.binarysig is not None:
sig_file_list.append(args.binarysig)
- verified_files = verify_file_integrity(sig_file_list)
- for verified_file in verified_files:
- print 'Hash verification of file %s OK.' % verified_file
+ # Check that the signatures that we have been provided (if any) match
+ # the hwpack and OS binaries we have been provided. If they don't, quit.
+ files_ok, verified_files = check_file_integrity_and_log_errors(
+ sig_file_list, args.binary, args.hwpacks)
+ if not files_ok:
+ sys.exit(1)
+
+ atexit.register(cleanup_tempdir)
media = Media(args.device)
if media.is_block_device:
if not board_config.supports_writing_to_mmc:
@@ -138,8 +142,8 @@
lmc_dir = os.path.dirname(__file__)
if lmc_dir == '':
lmc_dir = None
- install_hwpacks(
- ROOTFS_DIR, TMP_DIR, lmc_dir, args.hwpack_force_yes, verified_files, *hwpacks)
+ install_hwpacks(ROOTFS_DIR, TMP_DIR, lmc_dir, args.hwpack_force_yes,
+ verified_files, *hwpacks)
if args.rootfs == 'btrfs':
install_packages(ROOTFS_DIR, TMP_DIR, "btrfs-tools")
=== modified file 'linaro_image_tools/FetchImage.py'
@@ -35,6 +35,8 @@
import datetime
import threading
import subprocess
+import BeautifulSoup
+import utils
class FileHandler():
@@ -68,12 +70,12 @@
hwpack_file,
settings,
tools_dir,
+ hwpack_verified,
run_in_gui=False):
import linaro_image_tools.utils
- args = []
- args.append("pkexec")
+ args = ["pkexec"]
# Prefer a local linaro-media-create (from a bzr checkout for instance)
# to an installed one
@@ -89,6 +91,15 @@
if run_in_gui:
args.append("--nocheck-mmc")
+ if hwpack_verified:
+ # We verify the hwpack before calling linaro-media-create because
+ # these tools run linaro-media-create as root and to get the GPG
+ # signature verification to work, root would need to have GPG set
+ # up with the Canonical Linaro Image Build Automatic Signing Key
+ # imported. It seems far more likely that users will import keys
+ # as themselves, not root!
+ args.append("--hwpack-force-yes")
+
if 'rootfs' in settings and settings['rootfs']:
args.append("--rootfs")
args.append(settings['rootfs'])
@@ -115,22 +126,59 @@
return args
+ def get_sig_files(self, downloaded_files):
+ """
+ Find sha1sum.txt files in downloaded_files, append ".asc" and return
+ list.
+
+ The reason for not just searching for .asc files is because if they
+ don't exist, utils.verify_file_integrity(sig_files) won't check the
+ sha1sums that do exist, and they are useful to us. Trying to check
+ a GPG signature that doesn't exist just results in the signature check
+ failing, which is correct and we act accordingly.
+ """
+
+ sig_files = []
+
+ for filename in downloaded_files.values():
+ if re.search(r"sha1sums\.txt$", filename):
+ sig_files.append(filename + ".asc")
+
+ return sig_files
+
def create_media(self, image_url, hwpack_url, settings, tools_dir):
"""Create a command line for linaro-media-create based on the settings
provided then run linaro-media-create, either in a separate thread
(GUI mode) or in the current one (CLI mode)."""
- to_download = [(image_url, "OS"),
- (hwpack_url, "hwpack")]
+ sha1sums_urls = self.sha1sum_file_download_list(image_url,
+ hwpack_url,
+ settings)
+
+ sha1sum_files = self.download_files(sha1sums_urls,
+ settings)
+
+ to_download = self.generate_download_list(image_url,
+ hwpack_url,
+ sha1sum_files)
+
downloaded_files = self.download_files(to_download,
settings)
- args = self.build_lmc_command(downloaded_files['OS'],
- downloaded_files['hwpack'],
- settings,
- tools_dir)
-
- self.create_process = subprocess.Popen(args)
+ sig_files = self.get_sig_files(downloaded_files)
+
+ verified_files, gpg_sig_ok = utils.verify_file_integrity(sig_files)
+
+ hwpack = os.path.basename(downloaded_files[hwpack_url])
+ hwpack_verified = (hwpack in verified_files) and gpg_sig_ok
+
+ lmc_command = self.build_lmc_command(downloaded_files[image_url],
+ downloaded_files[hwpack_url],
+ settings,
+ tools_dir,
+ hwpack_verified)
+
+ self.create_process = subprocess.Popen(lmc_command)
self.create_process.wait()
class LinaroMediaCreate(threading.Thread):
@@ -163,20 +211,43 @@
needs to be re-directed to the GUI
"""
- to_download = [(self.image_url, "OS"),
- (self.hwpack_url, "hwpack")]
+ sha1sums_urls = self.file_handler.sha1sum_file_download_list(
+ self.image_url,
+ self.hwpack_url,
+ self.settings)
+
+ self.event_queue.put(("update",
+ "download",
+ "message",
+ "Calculating download size"))
+
+ sha1sum_files = self.file_handler.download_files(sha1sums_urls,
+ self.settings)
+
+ to_download = self.file_handler.generate_download_list(
+ self.image_url,
+ self.hwpack_url,
+ sha1sum_files)
downloaded_files = self.file_handler.download_files(
to_download,
self.settings,
self.event_queue)
+ sig_files = self.file_handler.get_sig_files(downloaded_files)
+
+ verified_files, gpg_sig_ok = utils.verify_file_integrity(sig_files)
+
+ hwpack = os.path.basename(downloaded_files[self.hwpack_url])
+ hwpack_verified = (hwpack in verified_files) and gpg_sig_ok
+
lmc_command = self.file_handler.build_lmc_command(
- downloaded_files['OS'],
- downloaded_files['hwpack'],
- self.settings,
- self.tools_dir,
- True)
+ downloaded_files[self.image_url],
+ downloaded_files[self.hwpack_url],
+ self.settings,
+ self.tools_dir,
+ hwpack_verified,
+ True)
self.create_process = subprocess.Popen(lmc_command,
stdin=subprocess.PIPE,
@@ -291,6 +362,129 @@
return None
+ def list_files_in_dir_of_url(self, url):
+ """
+ return a directory listing of the directory that url sits in
+ """
+
+ url = os.path.dirname(url)
+ response = self.urllib2_open(url)
+ page = response.read()
+
+ dir = []
+ # Use BeautifulSoup to parse the HTML and iterate over all 'a' tags
+ soup = BeautifulSoup.BeautifulSoup(page)
+ for link in soup.findAll('a'):
+ for attr, value in link.attrs:
+ if( attr == "href"
+ and not re.search(r'[\?=;/]', value)):
+ # Ignore links that don't look like plain files
+ dir.append('/'.join([url, value]))
+
+ return dir
+
+ def sha1sum_file_download_list(self,
+ image_url,
+ hwpack_url,
+ settings):
+ """
+ Need more than just the hwpack and OS image if we want signature
+ verification to work, we need sha1sums, signatures for sha1sums
+ and manifest files.
+
+ Note that this code is a bit sloppy and may result in downloading
+ more sums than are strictly required, but the
+ files are small and the wrong files won't be used.
+ """
+
+ downloads_list = []
+
+ # Get list of files in OS image directory
+ image_dir = self.list_files_in_dir_of_url(image_url)
+ hwpack_dir = self.list_files_in_dir_of_url(hwpack_url)
+
+ for link in image_dir:
+ if re.search("sha1sums\.txt$", link):
+ downloads_list.append(link)
+
+ for link in hwpack_dir:
+ if( re.search(settings['hwpack'], link)
+ and re.search("sha1sums\.txt$", link)):
+ downloads_list.append(link)
+
+ return downloads_list
+
+ def generate_download_list(self,
+ image_url,
+ hwpack_url,
+ sha1sum_file_names):
+ """
+ Generate a list of files based on what is in the sums files that
+ we have downloaded. We may have downloaded some sig files based on
+ a rather sloppy file name match that we don't want to use. For the
+ hwpack sig files, check to see if the hwpack listed matches the hwpack
+ URL. If it doesn't ignore it.
+
+ 1. Download sig file(s) that match the hardware spec (done by this
+ point).
+ 2. Find which sig file really matches the hardware pack we have
+ downloaded. (this function calculates this list)
+ 3. Download all the files listed in the sig file (done by another func)
+
+ We go through this process because sometimes a directory will have
+ more than one hardware pack that will match the hardware pack name,
+ for example panda and panda-x11 will both match "panda". These checks
+ make sure we only try and validate the signatures of the files that
+ we should be downloading and not try and validatate a signature of a
+ file that there is no reason for us to download, which would result in
+ an an invalid warning about installing unsigned packages when running
+ linaro-media-create.
+ """
+
+ downloads_list = [image_url, hwpack_url]
+
+ for sha1sum_file_name in sha1sum_file_names.values():
+ sha1sum_file = open(sha1sum_file_name)
+
+ common_prefix = None
+
+ files = []
+ for line in sha1sum_file:
+ line = line.split() # line[1] is now the file name
+
+ if line[1] == os.path.basename(image_url):
+ # Found a line that matches an image or hwpack URL - keep
+ # the contents of this sig file
+ common_prefix = os.path.dirname(image_url)
+
+ if line[1] == os.path.basename(hwpack_url):
+ # Found a line that matches an image or hwpack URL - keep
+ # the contents of this sig file
+ common_prefix = os.path.dirname(hwpack_url)
+
+ if common_prefix:
+ for file_name in files:
+ downloads_list.append('/'.join([common_prefix,
+ file_name]))
+
+ dir = self.list_files_in_dir_of_url(common_prefix + '/')
+
+ # We include the sha1sum files that pointed to the files that
+ # we are going to download so the full file list can be parsed
+ # later. The files won't be re-downloaded because they will be
+ # cached.
+ file_name = os.path.basename(sha1sum_file_name)
+ downloads_list.append('/'.join([common_prefix, file_name]))
+ signed_sums = os.path.basename(sha1sum_file_name) + ".asc"
+
+ for link in dir:
+ if re.search(signed_sums, link):
+ downloads_list.append(link)
+
+ sha1sum_file.close()
+
+ return downloads_list
+
def download_files(self,
downloads_list,
settings,
@@ -304,7 +498,7 @@
bytes_to_download = 0
- for url, name in downloads_list:
+ for url in downloads_list:
file_name, file_path = self.name_and_path_from_url(url)
file_name = file_path + os.sep + file_name
@@ -318,19 +512,13 @@
response.close()
if event_queue:
+ event_queue.put(("start", "download"))
event_queue.put(("update",
"download",
"total bytes",
bytes_to_download))
- for url, name in downloads_list:
- if event_queue:
- event_queue.put(("start", "download " + name))
- event_queue.put(("update",
- "download",
- "name",
- name))
-
+ for url in downloads_list:
path = None
try:
path = self.download(url,
@@ -341,14 +529,14 @@
print "Unexpected error:", sys.exc_info()[0]
logging.error("Unable to download " + url + " - aborting.")
- if event_queue:
- event_queue.put(("end", "download " + name))
-
if path == None: # User hit cancel when downloading
sys.exit(0)
- downloaded_files[name] = path
- logging.info("Have downloaded {0} to {1}".format(name, path))
+ downloaded_files[url] = path
+ logging.info("Have downloaded {0} to {1}".format(url, path))
+
+ if event_queue:
+ event_queue.put(("end", "download"))
return downloaded_files
=== modified file 'linaro_image_tools/cmd_runner.py'
@@ -72,6 +72,7 @@
def __init__(self, args, env=None, **kwargs):
self._my_args = args
+ self.except_on_cmd_fail=True
if env is None:
env = os.environ.copy()
env['LC_ALL'] = 'C'
@@ -81,19 +82,40 @@
sanitize_path(env)
super(Popen, self).__init__(args, env=env, **kwargs)
+ def communicate(self, input=None):
+ self.except_on_cmd_fail = False
+ stdout, stderr = super(Popen, self).communicate(input)
+ self.except_on_cmd_fail = True
+
+ if self.returncode != 0:
+ raise SubcommandNonZeroReturnValue(self._my_args,
+ self.returncode,
+ stdout,
+ stderr)
+ return stdout, stderr
+
def wait(self):
returncode = super(Popen, self).wait()
- if returncode != 0:
+ if returncode != 0 and self.except_on_cmd_fail:
raise SubcommandNonZeroReturnValue(self._my_args, returncode)
return returncode
class SubcommandNonZeroReturnValue(Exception):
- def __init__(self, command, return_value):
+ def __init__(self, command, return_value, stdout=None, stderr=None):
self.command = command
self.retval = return_value
+ self.stdout = stdout
+ self.stderr = stderr
def __str__(self):
- return 'Sub process "%s" returned a non-zero value: %d' % (
+ message = 'Sub process "%s" returned a non-zero value: %d' % (
self.command, self.retval)
+
+ if self.stdout:
+ message += '\nstdout was\n{0}'.format(self.stdout)
+ if self.stderr:
+ message += '\nstderr was\n{0}'.format(self.stderr)
+
+ return message
\ No newline at end of file
=== modified file 'linaro_image_tools/tests/test_utils.py'
@@ -21,6 +21,7 @@
import stat
import subprocess
import sys
+import logging
from linaro_image_tools import cmd_runner, utils
from linaro_image_tools.testing import TestCaseWithFixtures
@@ -36,6 +37,7 @@
preferred_tools_dir,
UnableToFindPackageProvidingCommand,
verify_file_integrity,
+ check_file_integrity_and_log_errors,
)
@@ -59,6 +61,37 @@
def wait(self):
return self.returncode
+
+ class MockCmdRunnerPopen_sha1sum_fail(object):
+ def __call__(self, cmd, *args, **kwargs):
+ self.returncode = 0
+ return self
+
+ def communicate(self, input=None):
+ self.wait()
+ return ': ERROR\n'.join(
+ TestVerifyFileIntegrity.filenames_in_shafile) + ': ERROR\n', ''
+
+ def wait(self):
+ return self.returncode
+
+
+ class MockCmdRunnerPopen_wait_fails(object):
+ def __call__(self, cmd, *args, **kwargs):
+ self.returncode = 0
+ return self
+
+ def communicate(self, input=None):
+ self.wait()
+ return ': OK\n'.join(
+ TestVerifyFileIntegrity.filenames_in_shafile) + ': OK\n', ''
+
+ def wait(self):
+ stdout = ': OK\n'.join(
+ TestVerifyFileIntegrity.filenames_in_shafile) + ': OK\n'
+ raise cmd_runner.SubcommandNonZeroReturnValue([], 1, stdout, None)
+
+
def test_verify_files(self):
fixture = self.useFixture(MockCmdRunnerPopenFixture())
hash_filename = "dummy-file.txt"
@@ -74,9 +107,57 @@
self.MockCmdRunnerPopen()))
hash_filename = "dummy-file.txt"
signature_filename = hash_filename + ".asc"
- verified_files = verify_file_integrity([signature_filename])
- self.assertEqual(self.filenames_in_shafile, verified_files)
-
+ verified_files, _ = verify_file_integrity([signature_filename])
+ self.assertEqual(self.filenames_in_shafile, verified_files)
+
+ def test_check_file_integrity_and_print_errors(self):
+ self.useFixture(MockSomethingFixture(cmd_runner, 'Popen',
+ self.MockCmdRunnerPopen()))
+ hash_filename = "dummy-file.txt"
+ signature_filename = hash_filename + ".asc"
+ result, verified_files = check_file_integrity_and_log_errors(
+ [signature_filename],
+ self.filenames_in_shafile[0],
+ [self.filenames_in_shafile[1]])
+ self.assertEqual(self.filenames_in_shafile, verified_files)
+
+ # The sha1sums are faked as passing and all commands return 0, so
+ # it should look like GPG passed
+ self.assertTrue(result)
+
+ def test_check_file_integrity_and_print_errors_fail_sha1sum(self):
+ logging.getLogger().setLevel(100) # Disable logging messages to screen
+ self.useFixture(MockSomethingFixture(cmd_runner, 'Popen',
+ self.MockCmdRunnerPopen_sha1sum_fail()))
+ hash_filename = "dummy-file.txt"
+ signature_filename = hash_filename + ".asc"
+ result, verified_files = check_file_integrity_and_log_errors(
+ [signature_filename],
+ self.filenames_in_shafile[0],
+ [self.filenames_in_shafile[1]])
+ self.assertEqual([], verified_files)
+
+ # The sha1sums are faked as failing and all commands return 0, so
+ # it should look like GPG passed
+ self.assertFalse(result)
+ logging.getLogger().setLevel(logging.WARNING)
+
+ def test_check_file_integrity_and_print_errors_fail_gpg(self):
+ logging.getLogger().setLevel(100) # Disable logging messages to screen
+ self.useFixture(MockSomethingFixture(cmd_runner, 'Popen',
+ self.MockCmdRunnerPopen_wait_fails()))
+ hash_filename = "dummy-file.txt"
+ signature_filename = hash_filename + ".asc"
+ result, verified_files = check_file_integrity_and_log_errors(
+ [signature_filename],
+ self.filenames_in_shafile[0],
+ [self.filenames_in_shafile[1]])
+ self.assertEqual([], verified_files)
+
+ # The sha1sums are faked as passing and all commands return 1, so
+ # it should look like GPG failed
+ self.assertFalse(result)
+ logging.getLogger().setLevel(logging.WARNING)
class TestEnsureCommand(TestCaseWithFixtures):
=== modified file 'linaro_image_tools/utils.py'
@@ -20,6 +20,8 @@
import os
import platform
import subprocess
+import re
+import logging
try:
from CommandNotFound import CommandNotFound
@@ -39,20 +41,68 @@
Each of the sha1 files will be checked using sha1sums. All files listed in
the sha1 hash file must be found in the same directory as the hash file.
"""
+
+ gpg_sig_ok = True
+
verified_files = []
for sig_file in sig_file_list:
hash_file = sig_file[0:-len('.asc')]
- cmd_runner.run(['gpg', '--verify', sig_file]).wait()
+
+ try:
+ cmd_runner.run(['gpg', '--verify', sig_file]).wait()
+ except cmd_runner.SubcommandNonZeroReturnValue:
+ gpg_sig_ok = False
+
if os.path.dirname(hash_file) == '':
sha_cwd = None
else:
sha_cwd = os.path.dirname(hash_file)
- sha1sums_out, _ = cmd_runner.run(['sha1sum', '-c', hash_file],
- stdout=subprocess.PIPE, cwd=sha_cwd
- ).communicate()
- verified_files.extend(sha1sums_out.replace(': OK', '').splitlines())
- return verified_files
-
+
+ try:
+ sha1sums_out, _ = cmd_runner.Popen(
+ ['sha1sum', '-c', hash_file],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ cwd=sha_cwd
+ ).communicate()
+ except cmd_runner.SubcommandNonZeroReturnValue as inst:
+ sha1sums_out = inst.stdout
+
+ for line in sha1sums_out.splitlines():
+ sha1_check = re.search(r'^(.*):\s+OK', line)
+ if sha1_check:
+ verified_files.append(sha1_check.group(1))
+
+ return verified_files, gpg_sig_ok
+
+def check_file_integrity_and_log_errors(sig_file_list, binary, hwpacks):
+ """
+ Wrapper around verify_file_integrity that prints error messages to stderr
+ if verify_file_integrity finds any problems.
+ """
+ verified_files, gpg_sig_pass = verify_file_integrity(sig_file_list)
+
+ # Check the outputs from verify_file_integrity
+ # Abort if anything fails.
+ if len(sig_file_list):
+ if not gpg_sig_pass:
+ logging.error("GPG signature verification failed.")
+ return False, []
+
+ if not os.path.basename(binary) in verified_files:
+ logging.error("OS Binary verification failed")
+ return False, []
+
+ for hwpack in hwpacks:
+ if not os.path.basename(hwpack) in verified_files:
+ logging.error("Hwpack {0} verification failed".format(hwpack))
+ return False, []
+
+ for verified_file in verified_files:
+ logging.info('Hash verification of file {0} OK.'.format(
+ verified_file))
+
+ return True, verified_files
def install_package_providing(command):
"""Install a package which provides the given command.