blob: 68b65894a99d73e42343998ae035573bbf7d1aea [file] [log] [blame]
# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import datetime
import json
import os
import re
from chromite.lib import cros_logging as logging
from google.appengine.api import datastore_errors
from google.appengine.ext import db
from google.appengine.api import users
import webapp2
import jinja2
import model
# Could replace this with a function if there is ever any reason
# to spread entries over multiple datastores. Consistency is only
# gauranteed within a datastore, but access should be limited to
# about 1 per second. That should not be a problem for us.
DATASTORE_KEY = db.Key.from_path('Stats', 'default')
JINJA_ENVIRONMENT = jinja2.Environment(
class MainPage(webapp2.RequestHandler):
"""Provide interface for interacting with DB."""
# Regex to peel SQL-like SELECT off front, if present, grabbing SELECT args.
# Example: "SELECT foo,bar WHERE blah blah"
# ==> group(1)="foo,bar", group(2)="WHERE blah blah"
# Example: "SELECT foo , bar"
# ==> group(1)="foo , bar", group(2)=""
# Example: "WHERE blah blah"
# ==> No match
QUERY_SELECT_PREFIX_RE = re.compile(r'^\s*SELECT\s+'
r'([^\s,]+(?:\s*,\s*[^\s,]+)*)' # Group 1
r'(?:$|\s+)(.*)', # Group 2
# Regex to determine if WHERE is present, and capture everything after it.
# Example: "WHERE foo=bar ORDER BY whatever"
# ==> group(1)="foo=bar ORDER BY whatever"
# Example: "ORDER BY whatever"
# ==> No match
QUERY_WHERE_PREFIX_RE = re.compile(r'^WHERE\s+(.+)$',
# Regex to discover ORDER BY columns in order to highlight them in results.
QUERY_ORDER_RE = re.compile(r'ORDER\s+BY\s+(\S+)', re.IGNORECASE)
# Regex to discover LIMIT value in query.
QUERY_LIMIT_RE = re.compile(r'LIMIT\s+(\d+)', re.IGNORECASE)
# Regex for separating tokens by commas, allowing spaces on either side.
COMMA_RE = re.compile(r'\s*,\s*')
# Default columns to show in results table if no SELECT given.
DEFAULT_COLUMNS = ['end_date', 'cmd_line', 'run_time', 'board',
# All possible columns in Statistics model.
ALL_COLUMNS = sorted(
# Provide example queries in interface as a form of documentation.
("ORDER BY end_date,run_time"
" LIMIT 30"),
("WHERE username='mtennant'"
" ORDER BY end_date DESC"
" LIMIT 30"),
("SELECT end_datetime,cmd_base,cmd_args,run_time,package_count"
" WHERE board='amd64-generic'"
" ORDER BY end_datetime"
" LIMIT 30"),
("SELECT end_date,cmd_base,run_time,board,package_count"
" WHERE end_date=DATE('2012-03-28')"
" ORDER BY run_time"
" LIMIT 30"),
("SELECT end_date,cmd_base,cmd_args,run_time,username"
" WHERE run_time>20"
" LIMIT 30"),
def get(self):
"""Support GET to stats page."""
# Note that authorization is required to access this page, which
# is controlled in app.yaml and on appspot admin page.
orig_query = self.request.get('query')
logging.debug('Received raw query %r', orig_query)
# If no LIMIT was provided, default to a LIMIT of 30 for sanity.
if not
orig_query += ' LIMIT 30'
query = orig_query
# Peel off "SELECT" clause from front of query. GCL does not support SELECT
# filtering, but we will support it right here to select/filter columns.
query, columns = self._RemoveSelectFromQuery(query)
if query == orig_query and columns == self.DEFAULT_COLUMNS:
# This means there was no SELECT in query. That is equivalent to
# SELECT of default columns, so show that to user.
orig_query = 'SELECT %s %s' % (','.join(columns), orig_query)
# All queries should have the "ancestor" WHERE clause in them, but that
# need not be exposed to interface. Insert the clause intelligently.
query = self._AdjustWhereInQuery(query)
stat_entries = []
error_msg = None
stat_entries = model.Statistics.gql(query, DATASTORE_KEY)
except datastore_errors.BadQueryError as ex:
error_msg = '<p>%s.</p><p>Actual GCL query used: "%s"</p>' % (ex, query)
if self.request.get('format') == 'json':
# Write output in the JSON format.
d = self._ResultsToDictionary(stat_entries, columns)
class CustomEncoder(json.JSONEncoder):
"""Handles non-serializable classes by converting them to strings."""
def default(self, obj):
if (isinstance(obj, datetime.datetime) or
isinstance(obj, or
isinstance(obj, datetime.time)):
return obj.isoformat()
return json.JSONEncoder.default(self, obj)
self.response.content_type = 'application/json'
self.response.write(json.dumps(d, cls=CustomEncoder))
# Write output to the HTML page.
results_table = self._PrepareResultsTable(stat_entries, columns)
template_values = {
'error_msg': error_msg,
'gcl_query': query,
'user_query': orig_query,
'user_email': users.get_current_user(),
'results_table': results_table,
'column_list': self.ALL_COLUMNS,
'example_queries': self.EXAMPLE_QUERIES,
template = JINJA_ENVIRONMENT.get_template('index.html')
def _RemoveSelectFromQuery(self, query):
"""Remove SELECT clause from |query|, return tuple (new_query, columns)."""
match =
if match:
# A SELECT clause is present. Remove it but save requested columns.
columns = self.COMMA_RE.split(
query =
if columns == ['*']:
columns = self.ALL_COLUMNS
logging.debug('Columns selected for viewing: %s', ', '.join(columns))
return query, columns
logging.debug('Using default columns for viewing: %s',
', '.join(self.DEFAULT_COLUMNS))
return query, self.DEFAULT_COLUMNS
def _AdjustWhereInQuery(self, query):
"""Insert WHERE ANCESTOR into |query| and return."""
match =
if match:
return 'WHERE ANCESTOR IS :1 AND %s' %
return 'WHERE ANCESTOR IS :1 %s' % query
def _PrepareResultsTable(self, stat_entries, columns):
"""Prepare table for |stat_entries| using only |columns|."""
# One header blank for row numbers, then each column name.
table = [[c for c in [''] + columns]]
# Prepare list of table rows, one for each stat entry.
for stat_ix, stat_entry in enumerate(stat_entries):
row = [stat_ix + 1]
row += [getattr(stat_entry, col) for col in columns]
return table
def _ResultsToDictionary(self, stat_entries, columns):
"""Converts |stat_entries| to a dictionary with |columns| as keys.
stat_entries: A list of GqlQuery objects.
columns: A list of keys to use.
A dictionary with |columns| as keys.
stats_dict = dict()
keys = [c for c in columns]
for stat_ix, stat_entry in enumerate(stat_entries):
stats_dict[stat_ix] = dict(
(col, getattr(stat_entry, col)) for col in columns)
return stats_dict
class PostPage(webapp2.RequestHandler):
"""Provides interface for uploading command stats to database."""
def post(self):
"""Support POST of command stats."""'Stats POST received at %r', self.request.uri)
new_stat = model.Statistics(parent=DATASTORE_KEY)
# Check each supported DB property to see if it has a value set
# in the POST request.
for prop in
# Skip properties with auto_now or auto_now_add enabled.
model_prop = getattr(model.Statistics, prop)
if ((hasattr(model_prop, 'auto_now_add') and model_prop.auto_now_add) or
(hasattr(model_prop, 'auto_now') and model_prop.auto_now)):
# Note that using hasattr with self.request does not work at all.
# It (almost) always says the attribute is not present, when getattr
# does actually return a value. Also note that self.request.get is
# not returning None as the default value if no explicit default value
# is provided, contrary to the spec for dict.get.
value = self.request.get(prop, self.NO_VALUE)
if value is not self.NO_VALUE:
# String properties must be 500 characters or less (GQL requirement).
if isinstance(model_prop, db.StringProperty) and len(value) > 500:
logging.debug(' String property %r too long. Cutting off at 500'
' characters.', prop)
value = value[:500]
# Integer properties require casting
if isinstance(model_prop, db.IntegerProperty):
value = int(value)
logging.debug(' Stats POST property %r ==> %r', prop, value)
setattr(new_stat, prop, value)
# Use automatically set end_datetime prop to set end_date and end_time.
new_stat.end_time = new_stat.end_datetime.time()
new_stat.end_date =
# Save to model.