From patchwork Wed Aug 17 21:28:14 2011 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Michael-Doyle Hudson X-Patchwork-Id: 3492 Return-Path: X-Original-To: patchwork@peony.canonical.com Delivered-To: patchwork@peony.canonical.com Received: from fiordland.canonical.com (fiordland.canonical.com [91.189.94.145]) by peony.canonical.com (Postfix) with ESMTP id 33FFD23F46 for ; Wed, 17 Aug 2011 21:28:19 +0000 (UTC) Received: from mail-ey0-f170.google.com (mail-ey0-f170.google.com [209.85.215.170]) by fiordland.canonical.com (Postfix) with ESMTP id 850D0A18481 for ; Wed, 17 Aug 2011 21:28:17 +0000 (UTC) Received: by eyd10 with SMTP id 10so1115095eyd.29 for ; Wed, 17 Aug 2011 14:28:17 -0700 (PDT) Received: by 10.213.14.67 with SMTP id f3mr1493075eba.56.1313616497162; Wed, 17 Aug 2011 14:28:17 -0700 (PDT) X-Forwarded-To: linaro-patchwork@canonical.com X-Forwarded-For: patch@linaro.org linaro-patchwork@canonical.com Delivered-To: patches@linaro.org Received: by 10.213.102.5 with SMTP id e5cs44156ebo; Wed, 17 Aug 2011 14:28:16 -0700 (PDT) Received: by 10.216.186.66 with SMTP id v44mr4753495wem.0.1313616495711; Wed, 17 Aug 2011 14:28:15 -0700 (PDT) Received: from indium.canonical.com (indium.canonical.com [91.189.90.7]) by mx.google.com with ESMTPS id s6si3984558wec.130.2011.08.17.14.28.15 (version=TLSv1/SSLv3 cipher=OTHER); Wed, 17 Aug 2011 14:28:15 -0700 (PDT) Received-SPF: pass (google.com: best guess record for domain of bounces@canonical.com designates 91.189.90.7 as permitted sender) client-ip=91.189.90.7; Authentication-Results: mx.google.com; spf=pass (google.com: best guess record for domain of bounces@canonical.com designates 91.189.90.7 as permitted sender) smtp.mail=bounces@canonical.com Received: from ackee.canonical.com ([91.189.89.26]) by indium.canonical.com with esmtp (Exim 4.71 #1 (Debian)) id 1Qtnep-00045f-1S for ; Wed, 17 Aug 2011 21:28:15 +0000 Received: from ackee.canonical.com (localhost [127.0.0.1]) by ackee.canonical.com (Postfix) with ESMTP id EDEB4E0A09 for ; Wed, 17 Aug 2011 21:28:14 +0000 (UTC) MIME-Version: 1.0 X-Launchpad-Project: lava-scheduler X-Launchpad-Branch: ~linaro-validation/lava-scheduler/trunk X-Launchpad-Message-Rationale: Subscriber X-Launchpad-Branch-Revision-Number: 61 X-Launchpad-Notification-Type: branch-revision To: Linaro Patch Tracker From: noreply@launchpad.net Subject: [Branch ~linaro-validation/lava-scheduler/trunk] Rev 61: record and display the url of the bundle submitted by a job Message-Id: <20110817212814.12361.71911.launchpad@ackee.canonical.com> Date: Wed, 17 Aug 2011 21:28:14 -0000 Reply-To: noreply@launchpad.net Sender: bounces@canonical.com Errors-To: bounces@canonical.com Precedence: bulk X-Generated-By: Launchpad (canonical.com); Revision="13697"; Instance="initZopeless config overlay" X-Launchpad-Hash: f9fef101fb2e69e5f87c87e661c81ddb7c20ca85 Merge authors: Michael Hudson-Doyle (mwhudson) Related merge proposals: https://code.launchpad.net/~mwhudson/lava-scheduler/record-bundle-sha1/+merge/71809 proposed by: Michael Hudson-Doyle (mwhudson) review: Approve - Zygmunt Krynicki (zkrynicki) ------------------------------------------------------------ revno: 61 [merge] committer: Michael-Doyle Hudson branch nick: trunk timestamp: Thu 2011-08-18 09:26:40 +1200 message: record and display the url of the bundle submitted by a job added: lava_scheduler_app/migrations/0008_auto__add_field_testjob_results_link.py modified: fake-dispatcher lava_scheduler_app/models.py lava_scheduler_app/templates/lava_scheduler_app/job.html lava_scheduler_daemon/board.py lava_scheduler_daemon/dbjobsource.py lava_scheduler_daemon/main.py lava_scheduler_daemon/tests/test_board.py --- lp:lava-scheduler https://code.launchpad.net/~linaro-validation/lava-scheduler/trunk You are subscribed to branch lp:lava-scheduler. To unsubscribe from this branch go to https://code.launchpad.net/~linaro-validation/lava-scheduler/trunk/+edit-subscription === modified file 'fake-dispatcher' --- fake-dispatcher 2011-07-27 00:22:15 +0000 +++ fake-dispatcher 2011-08-17 03:09:36 +0000 @@ -1,10 +1,15 @@ #!/bin/sh echo starting processing $1 echo error >&2 -for i in `seq 30`; do +for i in `seq 3`; do sleep 2 echo $i cat $1 echo done +echo dashboard-put-result: http://disney.com >&3 +for i in `seq 3 6`; do +sleep 2 +echo $i +done echo ending === added file 'lava_scheduler_app/migrations/0008_auto__add_field_testjob_results_link.py' --- lava_scheduler_app/migrations/0008_auto__add_field_testjob_results_link.py 1970-01-01 00:00:00 +0000 +++ lava_scheduler_app/migrations/0008_auto__add_field_testjob_results_link.py 2011-08-17 03:02:01 +0000 @@ -0,0 +1,86 @@ +# encoding: utf-8 +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + +class Migration(SchemaMigration): + + def forwards(self, orm): + + # Adding field 'TestJob.results_link' + db.add_column('lava_scheduler_app_testjob', 'results_link', self.gf('django.db.models.fields.CharField')(default=None, max_length=400, null=True, blank=True), keep_default=False) + + + def backwards(self, orm): + + # Deleting field 'TestJob.results_link' + db.delete_column('lava_scheduler_app_testjob', 'results_link') + + + models = { + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'lava_scheduler_app.device': { + 'Meta': {'object_name': 'Device'}, + 'current_job': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['lava_scheduler_app.TestJob']", 'unique': 'True', 'null': 'True', 'blank': 'True'}), + 'device_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['lava_scheduler_app.DeviceType']"}), + 'hostname': ('django.db.models.fields.CharField', [], {'max_length': '200', 'primary_key': 'True'}), + 'status': ('django.db.models.fields.IntegerField', [], {'default': '1'}) + }, + 'lava_scheduler_app.devicetype': { + 'Meta': {'object_name': 'DeviceType'}, + 'name': ('django.db.models.fields.SlugField', [], {'max_length': '50', 'primary_key': 'True', 'db_index': 'True'}) + }, + 'lava_scheduler_app.testjob': { + 'Meta': {'object_name': 'TestJob'}, + 'actual_device': ('django.db.models.fields.related.ForeignKey', [], {'default': 'None', 'related_name': "'+'", 'null': 'True', 'to': "orm['lava_scheduler_app.Device']"}), + 'results_link': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '400', 'null': 'True', 'blank': 'True'}), + 'definition': ('django.db.models.fields.TextField', [], {}), + 'end_time': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'log_file': ('django.db.models.fields.files.FileField', [], {'default': 'None', 'max_length': '100', 'null': 'True'}), + 'requested_device': ('django.db.models.fields.related.ForeignKey', [], {'default': 'None', 'related_name': "'+'", 'null': 'True', 'to': "orm['lava_scheduler_app.Device']"}), + 'requested_device_type': ('django.db.models.fields.related.ForeignKey', [], {'default': 'None', 'related_name': "'+'", 'null': 'True', 'to': "orm['lava_scheduler_app.DeviceType']"}), + 'start_time': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'status': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'submit_time': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'submitter': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}) + } + } + + complete_apps = ['lava_scheduler_app'] === modified file 'lava_scheduler_app/models.py' --- lava_scheduler_app/models.py 2011-07-26 04:38:13 +0000 +++ lava_scheduler_app/models.py 2011-08-17 03:02:01 +0000 @@ -135,6 +135,9 @@ log_file = models.FileField( upload_to='lava-logs', default=None, null=True) + results_link = models.CharField( + max_length=400, default=None, null=True, blank=True) + def __unicode__(self): r = "%s test job" % self.get_status_display() if self.requested_device: === modified file 'lava_scheduler_app/templates/lava_scheduler_app/job.html' --- lava_scheduler_app/templates/lava_scheduler_app/job.html 2011-07-27 00:23:24 +0000 +++ lava_scheduler_app/templates/lava_scheduler_app/job.html 2011-08-17 03:30:25 +0000 @@ -45,6 +45,14 @@
On device:
{{ job.actual_device }}
{% endif %} + + {% if job.results_link %} +

+ + Results + +

+ {% endif %}
=== modified file 'lava_scheduler_daemon/board.py' --- lava_scheduler_daemon/board.py 2011-08-01 15:27:33 +0000 +++ lava_scheduler_daemon/board.py 2011-08-16 04:07:08 +0000 @@ -5,24 +5,51 @@ from twisted.internet.protocol import ProcessProtocol from twisted.internet import defer - +from twisted.protocols.basic import LineReceiver + + +def catchall_errback(logger): + def eb(failure): + logger.exception(failure.value) + return eb + + +class OOBDataProtocol(LineReceiver): + + logger = logging.getLogger(__name__ + '.OOBDataProtocol') + + delimiter = '\n' + + def __init__(self, source, board_name): + self.source = source + self.board_name = board_name + + def lineReceived(self, line): + if ':' not in line: + self.logger.error('malformed oob data: %r' % line) + return + key, value = line.split(':', 1) + self.source.jobOobData( + self.board_name, key, value.lstrip()).addErrback( + catchall_errback(self.logger)) class DispatcherProcessProtocol(ProcessProtocol): logger = logging.getLogger(__name__ + '.DispatcherProcessProtocol') - def __init__(self, deferred, log_file): - print log_file + def __init__(self, deferred, log_file, source, board_name): self.deferred = deferred self.log_file = log_file + self.source = source + self.oob_data = OOBDataProtocol(source, board_name) - def outReceived(self, text): - self.log_file.write(text) + def childDataReceived(self, childFD, data): + if childFD == 3: + self.oob_data.dataReceived(data) + self.log_file.write(data) self.log_file.flush() - errReceived = outReceived - def processEnded(self, reason): # This discards the process exit value. self.log_file.close() @@ -33,9 +60,11 @@ logger = logging.getLogger(__name__ + '.Job') - def __init__(self, job_data, dispatcher, reactor): + def __init__(self, job_data, dispatcher, source, board_name, reactor): self.job_data = job_data self.dispatcher = dispatcher + self.source = source + self.board_name = board_name self.reactor = reactor self._json_file = None @@ -46,9 +75,11 @@ with os.fdopen(fd, 'wb') as f: json.dump(json_data, f) self.reactor.spawnProcess( - DispatcherProcessProtocol(d, log_file), self.dispatcher, - args=[self.dispatcher, self._json_file], - childFDs={0:0, 1:'r', 2:'r'}, env=None) + DispatcherProcessProtocol( + d, log_file, self.source, self.board_name), + self.dispatcher, args=[ + self.dispatcher, self._json_file, '--oob-fd', '3'], + childFDs={0:0, 1:'r', 2:'r', 3:'r'}, env=None) d.addBoth(self._exited) return d @@ -61,7 +92,6 @@ class Board(object): """ - A board runs jobs. A board can be in four main states: * stopped (S) @@ -192,7 +222,8 @@ return self.logger.info("starting job %r", job_data) self.running_job = self.job_cls( - job_data, self.dispatcher, self.reactor) + job_data, self.dispatcher, self.source, self.board_name, + self.reactor) d = self.running_job.run() d.addCallbacks(self._cbJobFinished, self._ebJobFinished) === modified file 'lava_scheduler_daemon/dbjobsource.py' --- lava_scheduler_daemon/dbjobsource.py 2011-07-27 10:10:10 +0000 +++ lava_scheduler_daemon/dbjobsource.py 2011-08-17 03:09:36 +0000 @@ -95,3 +95,16 @@ def jobCompleted(self, board_name): return deferToThread(self.jobCompleted_impl, board_name) + + @transaction.commit_on_success() + def jobOobData_impl(self, board_name, key, value): + self.logger.info( + "oob data received for %s: %s: %s", board_name, key, value) + if key == 'dashboard-put-result': + device = Device.objects.get(hostname=board_name) + device.current_job.results_link = value + device.current_job.save() + + def jobOobData(self, board_name, key, value): + return deferToThread(self.jobOobData_impl, board_name, key, value) + === modified file 'lava_scheduler_daemon/main.py' --- lava_scheduler_daemon/main.py 2011-07-27 06:59:51 +0000 +++ lava_scheduler_daemon/main.py 2011-08-17 03:18:13 +0000 @@ -14,7 +14,10 @@ if sys.argv[1:] == ['--use-fake']: dispatcher = os.path.join( - os.path.dirname(os.path.abspath(__file__)), 'fake-dispatcher') + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + 'fake-dispatcher') + elif sys.argv[1:2] == ['--dispatcher'] and len(sys.argv) == 3: + dispatcher = sys.argv[2] elif sys.argv[1:]: print >>sys.stderr, "invalid options %r" % sys.argv[1:] sys.exit(1) === modified file 'lava_scheduler_daemon/tests/test_board.py' --- lava_scheduler_daemon/tests/test_board.py 2011-07-26 00:34:21 +0000 +++ lava_scheduler_daemon/tests/test_board.py 2011-08-05 10:50:10 +0000 @@ -36,10 +36,12 @@ class TestJob(object): - def __init__(self, job_data, dispatcher, reactor): + def __init__(self, job_data, dispatcher, source, board_name, reactor): self.json_data = job_data self.dispatcher = dispatcher self.reactor = reactor + self.source = source + self.board_name = board_name self.deferred = defer.Deferred() def run(self):