| # Copyright 2011 Google Inc. All Rights Reserved. |
| # |
| |
| __author__ = 'kbaclawski@google.com (Krystian Baclawski)' |
| |
| from collections import namedtuple |
| import glob |
| import gzip |
| import os.path |
| import pickle |
| import time |
| import xmlrpclib |
| |
| from django import forms |
| from django.http import HttpResponseRedirect |
| from django.shortcuts import render_to_response |
| from django.template import Context |
| from django.views import static |
| |
| Link = namedtuple('Link', 'href name') |
| |
| |
| def GetServerConnection(): |
| return xmlrpclib.Server('http://localhost:8000') |
| |
| |
| def MakeDefaultContext(*args): |
| context = Context({'links': [ |
| Link('/job-group', 'Job Groups'), Link('/machine', 'Machines') |
| ]}) |
| |
| for arg in args: |
| context.update(arg) |
| |
| return context |
| |
| |
| class JobInfo(object): |
| |
| def __init__(self, job_id): |
| self._job = pickle.loads(GetServerConnection().GetJob(job_id)) |
| |
| def GetAttributes(self): |
| job = self._job |
| |
| group = [Link('/job-group/%d' % job.group.id, job.group.label)] |
| |
| predecessors = [Link('/job/%d' % pred.id, pred.label) |
| for pred in job.predecessors] |
| |
| successors = [Link('/job/%d' % succ.id, succ.label) |
| for succ in job.successors] |
| |
| machines = [Link('/machine/%s' % mach.hostname, mach.hostname) |
| for mach in job.machines] |
| |
| logs = [Link('/job/%d/log' % job.id, 'Log')] |
| |
| commands = enumerate(job.PrettyFormatCommand().split('\n'), start=1) |
| |
| return {'text': [('Label', job.label), ('Directory', job.work_dir)], |
| 'link': [('Group', group), ('Predecessors', predecessors), |
| ('Successors', successors), ('Machines', machines), |
| ('Logs', logs)], |
| 'code': [('Command', commands)]} |
| |
| def GetTimeline(self): |
| return [{'started': evlog.GetTimeStartedFormatted(), |
| 'state_from': evlog.event.from_, |
| 'state_to': evlog.event.to_, |
| 'elapsed': evlog.GetTimeElapsedRounded()} |
| for evlog in self._job.timeline.GetTransitionEventHistory()] |
| |
| def GetLog(self): |
| log_path = os.path.join(self._job.logs_dir, |
| '%s.gz' % self._job.log_filename_prefix) |
| |
| try: |
| log = gzip.open(log_path, 'r') |
| except IOError: |
| content = [] |
| else: |
| # There's a good chance that file is not closed yet, so EOF handling |
| # function and CRC calculation will fail, thus we need to monkey patch the |
| # _read_eof method. |
| log._read_eof = lambda: None |
| |
| def SplitLine(line): |
| prefix, msg = line.split(': ', 1) |
| datetime, stream = prefix.rsplit(' ', 1) |
| |
| return datetime, stream, msg |
| |
| content = map(SplitLine, log.readlines()) |
| finally: |
| log.close() |
| |
| return content |
| |
| |
| class JobGroupInfo(object): |
| |
| def __init__(self, job_group_id): |
| self._job_group = pickle.loads(GetServerConnection().GetJobGroup( |
| job_group_id)) |
| |
| def GetAttributes(self): |
| group = self._job_group |
| |
| home_dir = [Link('/job-group/%d/files/' % group.id, group.home_dir)] |
| |
| return {'text': [('Label', group.label), |
| ('Time submitted', time.ctime(group.time_submitted)), |
| ('State', group.status), |
| ('Cleanup on completion', group.cleanup_on_completion), |
| ('Cleanup on failure', group.cleanup_on_failure)], |
| 'link': [('Directory', home_dir)]} |
| |
| def _GetJobStatus(self, job): |
| status_map = {'SUCCEEDED': 'success', 'FAILED': 'failure'} |
| return status_map.get(str(job.status), None) |
| |
| def GetJobList(self): |
| return [{'id': job.id, |
| 'label': job.label, |
| 'state': job.status, |
| 'status': self._GetJobStatus(job), |
| 'elapsed': job.timeline.GetTotalTime()} |
| for job in self._job_group.jobs] |
| |
| def GetHomeDirectory(self): |
| return self._job_group.home_dir |
| |
| def GetReportList(self): |
| job_dir_pattern = os.path.join(self._job_group.home_dir, 'job-*') |
| |
| filenames = [] |
| |
| for job_dir in glob.glob(job_dir_pattern): |
| filename = os.path.join(job_dir, 'report.html') |
| |
| if os.access(filename, os.F_OK): |
| filenames.append(filename) |
| |
| reports = [] |
| |
| for filename in sorted(filenames, key=lambda f: os.stat(f).st_ctime): |
| try: |
| with open(filename, 'r') as report: |
| reports.append(report.read()) |
| except IOError: |
| pass |
| |
| return reports |
| |
| |
| class JobGroupListInfo(object): |
| |
| def __init__(self): |
| self._all_job_groups = pickle.loads(GetServerConnection().GetAllJobGroups()) |
| |
| def _GetJobGroupState(self, group): |
| return str(group.status) |
| |
| def _GetJobGroupStatus(self, group): |
| status_map = {'SUCCEEDED': 'success', 'FAILED': 'failure'} |
| return status_map.get(self._GetJobGroupState(group), None) |
| |
| def GetList(self): |
| return [{'id': group.id, |
| 'label': group.label, |
| 'submitted': time.ctime(group.time_submitted), |
| 'state': self._GetJobGroupState(group), |
| 'status': self._GetJobGroupStatus(group)} |
| for group in self._all_job_groups] |
| |
| def GetLabelList(self): |
| return sorted(set(group.label for group in self._all_job_groups)) |
| |
| |
| def JobPageHandler(request, job_id): |
| job = JobInfo(int(job_id)) |
| |
| ctx = MakeDefaultContext({ |
| 'job_id': job_id, |
| 'attributes': job.GetAttributes(), |
| 'timeline': job.GetTimeline() |
| }) |
| |
| return render_to_response('job.html', ctx) |
| |
| |
| def LogPageHandler(request, job_id): |
| job = JobInfo(int(job_id)) |
| |
| ctx = MakeDefaultContext({'job_id': job_id, 'log_lines': job.GetLog()}) |
| |
| return render_to_response('job_log.html', ctx) |
| |
| |
| def JobGroupPageHandler(request, job_group_id): |
| group = JobGroupInfo(int(job_group_id)) |
| |
| ctx = MakeDefaultContext({ |
| 'group_id': job_group_id, |
| 'attributes': group.GetAttributes(), |
| 'job_list': group.GetJobList(), |
| 'reports': group.GetReportList() |
| }) |
| |
| return render_to_response('job_group.html', ctx) |
| |
| |
| def JobGroupFilesPageHandler(request, job_group_id, path): |
| group = JobGroupInfo(int(job_group_id)) |
| |
| return static.serve(request, |
| path, |
| document_root=group.GetHomeDirectory(), |
| show_indexes=True) |
| |
| |
| class FilterJobGroupsForm(forms.Form): |
| label = forms.ChoiceField(label='Filter by label:', required=False) |
| |
| |
| def JobGroupListPageHandler(request): |
| groups = JobGroupListInfo() |
| group_list = groups.GetList() |
| |
| field = FilterJobGroupsForm.base_fields['label'] |
| field.choices = [('*', '--- no filtering ---')] |
| field.choices.extend([(label, label) for label in groups.GetLabelList()]) |
| |
| if request.method == 'POST': |
| form = FilterJobGroupsForm(request.POST) |
| |
| if form.is_valid(): |
| label = form.cleaned_data['label'] |
| |
| if label != '*': |
| group_list = [group for group in group_list if group['label'] == label] |
| else: |
| form = FilterJobGroupsForm({'initial': '*'}) |
| |
| ctx = MakeDefaultContext({'filter': form, 'groups': group_list}) |
| |
| return render_to_response('job_group_list.html', ctx) |
| |
| |
| def MachineListPageHandler(request): |
| machine_list = pickle.loads(GetServerConnection().GetMachineList()) |
| |
| return render_to_response('machine_list.html', |
| MakeDefaultContext({'machines': machine_list})) |
| |
| |
| def DefaultPageHandler(request): |
| return HttpResponseRedirect('/job-group') |