| # Copyright 2015 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """The datastore models for graph data. |
| |
| The Chromium project uses Buildbot to run its performance tests, and the |
| structure of the data for the Performance Dashboard reflects this. Metadata |
| about tests are structured in a hierarchy of Master, Bot, and Test entities. |
| Master and Bot entities represent Buildbot masters and builders respectively, |
| and Test entities represent groups of results, or individual data series. |
| |
| For example, entities might be structured as follows: |
| |
| Master: ChromiumPerf |
| Bot: win7 |
| Test: page_cycler.moz |
| Test: times |
| Test: page_load_time |
| Test: page_load_time_ref |
| Test: www.amazon.com |
| Test: www.bing.com |
| Test: commit_charge |
| Test: ref |
| Test: www.amazon.com |
| Test: www.bing.com |
| |
| The graph data points are represented by Row entities. Each Row entity contains |
| a revision and value, which are its X and Y values on a graph, and any other |
| metadata associated with an individual performance test result. |
| |
| The keys of the Row entities for a particular data series are start with a |
| TestContainer key, instead of a Test key. This way, the Row entities for each |
| data series are in a different "entity group". This allows a faster rate of |
| putting data in the datastore for many series at once. |
| |
| For example, Row entities are organized like this: |
| |
| TestContainer: ChromiumPerf/win7/page_cycler.moz/times/page_load_time |
| Row: revision 12345, value 2.5 |
| Row: revision 12346, value 2.0 |
| Row: revision 12347, value 2.1 |
| TestContainer: ChromiumPerf/win7/page_cycler.moz/times/page_load_time_ref |
| Row: revision 12345, value 2.4 |
| Row: revision 12346, value 2.0 |
| Row: revision 12347, value 2.2 |
| TestContainer: ChromiumPerf/win7/page_cycler.moz/commit_charge |
| Row: revision 12345, value 10 |
| Row: revision 12346, value 12 |
| Row: revision 12347, value 11 |
| |
| |
| IMPORTANT: If you add new kinds to this file, you must also add them to the |
| Daily Backup url in cron.yaml in order for them to be properly backed up. |
| See: https://developers.google.com/appengine/articles/scheduled_backups |
| """ |
| |
| import logging |
| |
| from google.appengine.ext import ndb |
| |
| from dashboard import layered_cache |
| from dashboard import utils |
| from dashboard.models import anomaly |
| from dashboard.models import anomaly_config |
| from dashboard.models import internal_only_model |
| from dashboard.models import sheriff as sheriff_module |
| from dashboard.models import stoppage_alert as stoppage_alert_module |
| |
| # Maximum level of nested tests. |
| MAX_TEST_ANCESTORS = 10 |
| |
| # Keys to the datastore-based cache. See stored_object. |
| LIST_TESTS_SUBTEST_CACHE_KEY = 'list_tests_get_tests_new_%s_%s_%s' |
| |
| _MAX_STRING_LENGTH = 500 |
| |
| |
| class Master(internal_only_model.InternalOnlyModel): |
| """Information about the Buildbot master. |
| |
| Masters are keyed by name, e.g. 'ChromiumGPU' or 'ChromiumPerf'. |
| All Bot entities that are Buildbot slaves of one master are children of one |
| Master entity in the datastore. |
| """ |
| # Master has no properties; the name of the master is the ID. |
| |
| |
| class Bot(internal_only_model.InternalOnlyModel): |
| """Information about a Buildbot slave that runs perf tests. |
| |
| Bots are keyed by name, e.g. 'xp-release-dual-core'. A Bot entity contains |
| information about whether the tests are only viewable to internal users, and |
| each bot has a parent that is a Master entity. A Bot is be the ancestor of |
| the Test entities that run on it. |
| """ |
| internal_only = ndb.BooleanProperty(default=False, indexed=True) |
| |
| |
| class Test(internal_only_model.CreateHookInternalOnlyModel): |
| """A Test entity is a node in a hierarchy of tests. |
| |
| A Test entity can represent a specific series of results which will be |
| plotted on a graph, or it can represent a group of such series of results, or |
| both. A Test entity that the property has_rows set to True corresponds to a |
| trace on a graph, and the parent Test for a group of such tests corresponds to |
| a graph with several traces. A parent Test for that test would correspond to a |
| group of related graphs. Top-level Tests (also known as test suites) are |
| parented by a Bot. |
| |
| Tests are keyed by name, and they also contain other metadata such as |
| description and units. |
| |
| NOTE: If you remove any properties from Test, they should be added to the |
| TEST_EXCLUDE_PROPERTIES list in migrate_test_names.py. |
| """ |
| internal_only = ndb.BooleanProperty(default=False, indexed=True) |
| |
| # Sheriff rotation for this test. Rotations are specified by regular |
| # expressions that can be edited at /edit_sheriffs. |
| sheriff = ndb.KeyProperty(kind=sheriff_module.Sheriff, indexed=True) |
| |
| # There is a default anomaly threshold config (in anomaly.py), and it can |
| # be overridden for a group of tests by using /edit_sheriffs. |
| overridden_anomaly_config = ndb.KeyProperty( |
| kind=anomaly_config.AnomalyConfig, indexed=True) |
| |
| # Keep track of what direction is an improvement for this graph so we can |
| # filter out alerts on regressions. |
| improvement_direction = ndb.IntegerProperty( |
| default=anomaly.UNKNOWN, |
| choices=[ |
| anomaly.UP, |
| anomaly.DOWN, |
| anomaly.UNKNOWN |
| ], |
| indexed=False |
| ) |
| |
| # Units of the child Rows of this Test, or None if there are no child Rows. |
| units = ndb.StringProperty(indexed=False) |
| |
| # The last alerted revision is used to avoid duplicate alerts. |
| last_alerted_revision = ndb.IntegerProperty(indexed=False) |
| |
| # Whether or not the test has child rows. Set by hook on Row class put. |
| has_rows = ndb.BooleanProperty(default=False, indexed=True) |
| |
| # If there is a currently a StoppageAlert that indicates that data hasn't |
| # been received for some time, then will be set. Otherwise, it is None. |
| stoppage_alert = ndb.KeyProperty( |
| kind=stoppage_alert_module.StoppageAlert, indexed=True) |
| |
| # A test is marked "deprecated" if no new points have been received for |
| # a long time; these tests should usually not be listed. |
| deprecated = ndb.BooleanProperty(default=False, indexed=True) |
| |
| # For top-level test entities, this is a list of sub-tests that are checked |
| # for alerts (i.e. they have a sheriff). For other tests, this is empty. |
| monitored = ndb.KeyProperty(repeated=True, indexed=True) |
| |
| # Description of what the test measures. |
| description = ndb.TextProperty(indexed=True) |
| |
| # Source code location of the test. Optional. |
| code = ndb.StringProperty(indexed=False, repeated=True) |
| |
| # Command to run the test. Optional. |
| command_line = ndb.StringProperty(indexed=False) |
| |
| # Computed properties are treated like member variables, so they have |
| # lowercase names, even though they look like methods to pylint. |
| # pylint: disable=invalid-name |
| |
| @ndb.ComputedProperty |
| def bot(self): # pylint: disable=invalid-name |
| """Immediate parent Bot entity, or None if this is not a test suite.""" |
| parent = self.key.parent() |
| if parent.kind() == 'Bot': |
| return parent |
| return None |
| |
| @ndb.ComputedProperty |
| def parent_test(self): # pylint: disable=invalid-name |
| """Immediate parent Test entity, or None if this is a test suite.""" |
| parent = self.key.parent() |
| if parent.kind() == 'Test': |
| return parent |
| return None |
| |
| @property |
| def test_path(self): |
| """Slash-separated list of key parts, 'master/bot/suite/chart/...'.""" |
| return utils.TestPath(self.key) |
| |
| @ndb.ComputedProperty |
| def master_name(self): |
| return self.key.pairs()[0][1] |
| |
| @ndb.ComputedProperty |
| def bot_name(self): |
| return self.key.pairs()[1][1] |
| |
| @ndb.ComputedProperty |
| def suite_name(self): |
| return self.key.pairs()[2][1] |
| |
| @ndb.ComputedProperty |
| def test_part1_name(self): |
| pairs = self.key.pairs() |
| if len(pairs) < 4: |
| return '' |
| return self.key.pairs()[3][1] |
| |
| @ndb.ComputedProperty |
| def test_part2_name(self): |
| pairs = self.key.pairs() |
| if len(pairs) < 5: |
| return '' |
| return self.key.pairs()[4][1] |
| |
| @ndb.ComputedProperty |
| def test_part3_name(self): |
| pairs = self.key.pairs() |
| if len(pairs) < 6: |
| return '' |
| return self.key.pairs()[5][1] |
| |
| @ndb.ComputedProperty |
| def test_part4_name(self): |
| pairs = self.key.pairs() |
| if len(pairs) < 7: |
| return '' |
| return self.key.pairs()[6][0] |
| |
| @classmethod |
| def _GetMasterBotSuite(cls, key): |
| while key and key.parent(): |
| if key.parent().kind() == 'Bot': |
| if not key.parent().parent(): |
| return None |
| return (key.parent().parent().string_id(), |
| key.parent().string_id(), |
| key.string_id()) |
| key = key.parent() |
| return None |
| |
| def __init__(self, *args, **kwargs): |
| # Indexed StringProperty has a maximum length. If this length is exceeded, |
| # then an error will be thrown in ndb.Model.__init__. |
| # Truncate the "description" property if necessary. |
| description = kwargs.get('description') or '' |
| kwargs['description'] = description[:_MAX_STRING_LENGTH] |
| super(Test, self).__init__(*args, **kwargs) |
| |
| def _pre_put_hook(self): |
| """This method is called before a Test is put into the datastore. |
| |
| Here, we check the sheriffs and anomaly configs to make sure they are |
| current. We also update the monitored list of the test suite. |
| """ |
| # Set the sheriff to the first sheriff (alphabetically by sheriff name) |
| # that has a test pattern that matches this test. |
| self.sheriff = None |
| for sheriff_entity in sheriff_module.Sheriff.query().fetch(): |
| for pattern in sheriff_entity.patterns: |
| if utils.TestMatchesPattern(self, pattern): |
| self.sheriff = sheriff_entity.key |
| if self.sheriff: |
| break |
| |
| # If this Test is monitored, add it to the monitored list of its test suite. |
| # A test is be monitored iff it has a sheriff, and monitored tests are |
| # tracked in the monitored list of a test suite Test entity. |
| test_suite = ndb.Key(*self.key.flat()[:6]).get() |
| if self.sheriff: |
| if test_suite and self.key not in test_suite.monitored: |
| test_suite.monitored.append(self.key) |
| test_suite.put() |
| elif test_suite and self.key in test_suite.monitored: |
| test_suite.monitored.remove(self.key) |
| test_suite.put() |
| |
| # Set the anomaly threshold config to the first one that has a test pattern |
| # that matches this test, if there is one. Anomaly configs are sorted by |
| # name, so that a config with a name that comes earlier lexicographically |
| # is considered higher-priority. |
| self.overridden_anomaly_config = None |
| anomaly_configs = anomaly_config.AnomalyConfig.query().fetch() |
| anomaly_configs.sort(key=lambda config: config.key.string_id()) |
| for anomaly_config_entity in anomaly_configs: |
| for pattern in anomaly_config_entity.patterns: |
| if utils.TestMatchesPattern(self, pattern): |
| self.overridden_anomaly_config = anomaly_config_entity.key |
| if self.overridden_anomaly_config: |
| break |
| |
| def CreateCallback(self): |
| """Called when the entity is first saved.""" |
| if self.key.parent().kind() != 'Bot': |
| layered_cache.Delete( |
| LIST_TESTS_SUBTEST_CACHE_KEY % self._GetMasterBotSuite(self.key)) |
| |
| @classmethod |
| # pylint: disable=unused-argument |
| def _pre_delete_hook(cls, key): |
| if key.parent() and key.parent().kind() != 'Bot': |
| layered_cache.Delete( |
| LIST_TESTS_SUBTEST_CACHE_KEY % Test._GetMasterBotSuite(key)) |
| |
| |
| class LastAddedRevision(ndb.Model): |
| """Represents the last added revision for a test path. |
| |
| The reason this property is separated from Test entity is to avoid contention |
| issues (Frequent update of entity within the same group). This property is |
| updated very frequent in /add_point. |
| """ |
| revision = ndb.IntegerProperty(indexed=False) |
| |
| |
| class Row(internal_only_model.InternalOnlyModel, ndb.Expando): |
| """A Row represents one data point. |
| |
| A Row has a revision and a value, which are the X and Y values, respectively. |
| Each Row belongs to one Test, along with all of the other Row entities that |
| it is plotted with. Rows are keyed by revision. |
| |
| In addition to the properties defined below, Row entities may also have other |
| properties which specify additional supplemental data. These are called |
| "supplemental columns", and should have the following prefixes: |
| d_: A data point, such as d_1st_run or d_50th_percentile. FloatProperty. |
| r_: Revision such as r_webkit or r_v8. StringProperty, limited to 25 |
| characters, '0-9' and '.'. |
| a_: Annotation such as a_chrome_bugid or a_gasp_anomaly. StringProperty. |
| """ |
| # Don't index by default (only explicitly indexed properties are indexed). |
| _default_indexed = False |
| internal_only = ndb.BooleanProperty(default=False, indexed=True) |
| |
| # The parent_test is the key of the Test entity that this Row belongs to. |
| @ndb.ComputedProperty |
| def parent_test(self): # pylint: disable=invalid-name |
| # The Test entity that a Row belongs to isn't actually its parent in the |
| # datastore. Rather, the parent key of each Row contains a test path, which |
| # contains the information necessary to get the actual Test key. |
| return utils.TestKey(self.key.parent().string_id()) |
| |
| # Points in each graph are sorted by "revision". This is usually a Chromium |
| # SVN version number, but it might also be any other integer, as long as |
| # newer points have higher numbers. |
| @ndb.ComputedProperty |
| def revision(self): # pylint: disable=invalid-name |
| return self.key.integer_id() |
| |
| # The time the revision was added to the dashboard is tracked in order |
| # to too many points from being added in a short period of time, which would |
| # indicate an error or malicious code. |
| timestamp = ndb.DateTimeProperty(auto_now_add=True, indexed=True) |
| |
| # The Y-value at this point. |
| value = ndb.FloatProperty(indexed=True) |
| |
| # The standard deviation at this point. Optional. |
| error = ndb.FloatProperty(indexed=False) |
| |
| def _pre_put_hook(self): |
| """Sets the has_rows property of the parent test before putting this Row. |
| |
| This isn't atomic because the parent_test put() and Row put() don't happen |
| in the same transaction. But in practice it shouldn't be an issue because |
| the parent test will get more points as the test runs. |
| """ |
| parent_test = self.parent_test.get() |
| |
| # If the Test pointed to by parent_test is not valid, that indicates |
| # that a Test entity was not properly created in add_point. |
| if not parent_test: |
| parent_key = self.key.parent() |
| logging.warning('Row put without valid Test. Parent key: %s', parent_key) |
| return |
| |
| if not parent_test.has_rows: |
| parent_test.has_rows = True |
| parent_test.put() |