# Copyright 2015 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import base64
import json
import unittest

import httplib2
import mock
import webapp2
import webtest

from google.appengine.ext import ndb

from dashboard import namespaced_stored_object
from dashboard import rietveld_service
from dashboard import start_try_job
from dashboard import stored_object
from dashboard import testing_common
from dashboard.models import bug_data
from dashboard.models import graph_data
from dashboard.models import try_job


# Below is a series of test strings which may contain long lines.
# pylint: disable=line-too-long
_EXPECTED_BISECT_CONFIG_DIFF = """config = {
-  'command': '',
-  'good_revision': '',
-  'bad_revision': '',
-  'metric': '',
-  'repeat_count':'',
-  'max_time_minutes': '',
+  "bad_revision": "215828",
+  "bisect_mode": "mean",
+  "bug_id": "12345",
+  "builder_type": "",
+  "command": "python tools/perf/run_benchmark -v --browser=release --output-format=buildbot --also-run-disabled-tests dromaeo.jslibstylejquery",
+  "good_revision": "215806",
+  "max_time_minutes": "20",
+  "metric": "jslib/jslib",
+  "repeat_count": "20",
+  "target_arch": "ia32"
 }
"""

_EXPECTED_BISECT_CONFIG_DIFF_FOR_INTERNAL_TEST = """config = {
-  'command': '',
-  'good_revision': '',
-  'bad_revision': '',
-  'metric': '',
-  'repeat_count':'',
-  'max_time_minutes': '',
+  "bad_revision": "f14a8f733cce874d5d66e8e6b86e75bbac240b0e",
+  "bisect_mode": "mean",
+  "bug_id": "12345",
+  "builder_type": "",
+  "command": "tools/perf/run_benchmark -v --browser=android-chrome --also-run-disabled-tests start_with_url.cold.startup_pages",
+  "good_revision": "d82ccc77c8a86ce9893a8035fb55aca666f044c8",
+  "max_time_minutes": "20",
+  "metric": "foreground_tab_request_start/foreground_tab_request_start",
+  "repeat_count": "20",
+  "target_arch": "ia32"
 }
"""

_EXPECTED_BISECT_CONFIG_DIFF_WITH_ARCHIVE = """config = {
-  'command': '',
-  'good_revision': '',
-  'bad_revision': '',
-  'metric': '',
-  'repeat_count':'',
-  'max_time_minutes': '',
+  "bad_revision": "215828",
+  "bisect_mode": "mean",
+  "bug_id": "12345",
+  "builder_type": "perf",
+  "command": "tools/perf/run_benchmark -v --browser=release --output-format=buildbot --also-run-disabled-tests dromaeo.jslibstylejquery",
+  "good_revision": "215806",
+  "max_time_minutes": "20",
+  "metric": "jslib/jslib",
+  "repeat_count": "20",
+  "target_arch": "ia32"
 }
"""

_EXPECTED_PERF_CONFIG_DIFF = """config = {
-  'command': '',
-  'metric': '',
-  'repeat_count': '',
-  'max_time_minutes': '',
+  "bad_revision": "215828",
+  "command": "tools/perf/run_benchmark -v --browser=release --output-format=buildbot --also-run-disabled-tests dromaeo.jslibstylejquery",
+  "good_revision": "215806",
+  "max_time_minutes": "60",
+  "repeat_count": "1"
 }
"""

_FAKE_XSRF_TOKEN = '1234567890'

_ISSUE_CREATED_RESPONSE = """Issue created. https://test-rietveld.appspot.com/33001
1
1001 filename
"""

_BISECT_CONFIG_CONTENTS = """# Copyright (c) 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

\"\"\"Config file for Run Performance Test Bisect Tool

This script is intended for use by anyone that wants to run a remote bisection
on a range of revisions to look for a performance regression. Modify the config
below and add the revision range, performance command, and metric. You can then
run a git try <bot>.

Changes to this file should never be submitted.

Args:
  'command': This is the full command line to pass to the
      bisect-perf-regression.py script in order to execute the test.
  'good_revision': An svn or git revision where the metric hadn't regressed yet.
  'bad_revision': An svn or git revision sometime after the metric had
      regressed.
  'metric': The name of the metric to parse out from the results of the
      performance test. You can retrieve the metric by looking at the stdio of
      the performance test. Look for lines of the format:

      RESULT <graph>: <trace>= <value> <units>

      The metric name is "<graph>/<trace>".
  'repeat_count': The number of times to repeat the performance test.
  'max_time_minutes': The script will attempt to run the performance test
      "repeat_count" times, unless it exceeds "max_time_minutes".

Sample config:

config = {
  'command': './out/Release/performance_ui_tests' +
      ' --gtest_filter=PageCyclerTest.Intl1File',
  'good_revision': '179755',
  'bad_revision': '179782',
  'metric': 'times/t',
  'repeat_count': '20',
  'max_time_minutes': '20',
}

On Windows:
  - If you're calling a python script you will need to add "python" to
the command:

config = {
  'command': 'python tools/perf/run_benchmark -v --browser=release kraken',
  'good_revision': '185319',
  'bad_revision': '185364',
  'metric': 'Total/Total',
  'repeat_count': '20',
  'max_time_minutes': '20',
}


On ChromeOS:
  - Script accepts either ChromeOS versions, or unix timestamps as revisions.
  - You don't need to specify --identity and --remote, they will be added to
    the command using the bot's BISECT_CROS_IP and BISECT_CROS_BOARD values.

config = {
  'command': './tools/perf/run_benchmark -v '\
      '--browser=cros-chrome-guest '\
      'dromaeo tools/perf/page_sets/dromaeo/jslibstylejquery.json',
  'good_revision': '4086.0.0',
  'bad_revision': '4087.0.0',
  'metric': 'jslib/jslib',
  'repeat_count': '20',
  'max_time_minutes': '20',
}

\"\"\"

config = {
  'command': '',
  'good_revision': '',
  'bad_revision': '',
  'metric': '',
  'repeat_count':'',
  'max_time_minutes': '',
}

# Workaround git try issue, see crbug.com/257689"""

_PERF_CONFIG_CONTENTS = """# Copyright 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

\"\"\"Config file for Run Performance Test Bot

This script is intended for use by anyone that wants to run a remote performance
test. Modify the config below and add the command to run the performance test,
the metric you're interested in, and repeat/discard parameters. You can then
run a git try <bot>.

Changes to this file should never be submitted.

Args:
  'command': This is the full command line to pass to the
      bisect-perf-regression.py script in order to execute the test.
  'metric': The name of the metric to parse out from the results of the
      performance test. You can retrieve the metric by looking at the stdio of
      the performance test. Look for lines of the format:

      RESULT <graph>: <trace>= <value> <units>

      The metric name is "<graph>/<trace>".
  'repeat_count': The number of times to repeat the performance test.
  'max_time_minutes': The script will attempt to run the performance test
      "repeat_count" times, unless it exceeds "max_time_minutes".

Sample config:

config = {
  'command': './tools/perf/run_benchmark --browser=release smoothness.key_mobile_sites',
  'metric': 'mean_frame_time/mean_frame_time',
  'repeat_count': '20',
  'max_time_minutes': '20',
}

On Windows:
  - If you're calling a python script you will need to add "python" to
the command:

config = {
  'command': 'python tools/perf/run_benchmark -v --browser=release smoothness.key_mobile_sites',
  'metric': 'mean_frame_time/mean_frame_time',
  'repeat_count': '20',
  'max_time_minutes': '20',
}


On ChromeOS:
  - Script accepts either ChromeOS versions, or unix timestamps as revisions.
  - You don't need to specify --identity and --remote, they will be added to
    the command using the bot's BISECT_CROS_IP and BISECT_CROS_BOARD values.

config = {
  'command': './tools/perf/run_benchmark -v '\
      '--browser=cros-chrome-guest '\
      'smoothness.key_mobile_sites',
  'metric': 'mean_frame_time/mean_frame_time',
  'repeat_count': '20',
  'max_time_minutes': '20',
}

\"\"\"

config = {
  'command': '',
  'metric': '',
  'repeat_count': '',
  'max_time_minutes': '',
}

# Workaround git try issue, see crbug.com/257689"""
# pylint: enable=line-too-long

# These globals are set in tests and checked in _MockMakeRequest.
_EXPECTED_CONFIG_DIFF = None
_TEST_EXPECTED_BOT = None
_TEST_EXPECTED_CONFIG_CONTENTS = None


def _MockFetch(url=None):
  if start_try_job._BISECT_CONFIG_PATH in url:
    return testing_common.FakeResponseObject(
        200, base64.encodestring(_BISECT_CONFIG_CONTENTS))
  elif start_try_job._PERF_CONFIG_PATH in url:
    return testing_common.FakeResponseObject(
        200, base64.encodestring(_PERF_CONFIG_CONTENTS))


def _MockFailedFetch(url=None):  # pylint: disable=unused-argument
    return testing_common.FakeResponseObject(404, {})


def _MockMakeRequest(path, *args, **kwargs):  # pylint: disable=unused-argument
  """Mocks out a request, returning a canned response."""
  if path.endswith('xsrf_token'):
    assert kwargs['headers']['X-Requesting-XSRF-Token'] == 1
    return (httplib2.Response({'status': '200'}), _FAKE_XSRF_TOKEN)
  if path == 'upload':
    assert kwargs['method'] == 'POST'
    assert _EXPECTED_CONFIG_DIFF in kwargs['body'], (
        '%s\nnot in\n%s\n' % (_EXPECTED_CONFIG_DIFF, kwargs['body']))
    return (httplib2.Response({'status': '200'}), _ISSUE_CREATED_RESPONSE)
  if path == '33001/upload_content/1/1001':
    assert kwargs['method'] == 'POST'
    assert _TEST_EXPECTED_CONFIG_CONTENTS in kwargs['body']
    return (httplib2.Response({'status': '200'}), 'Dummy content')
  if path == '33001/upload_complete/1':
    assert kwargs['method'] == 'POST'
    return (httplib2.Response({'status': '200'}), 'Dummy content')
  if path == '33001/try/1':
    assert _TEST_EXPECTED_BOT in kwargs['body']
    return (httplib2.Response({'status': '200'}), 'Dummy content')
  assert False, 'Invalid url %s requested!' % path


class StartBisectTest(testing_common.TestCase):

  def setUp(self):
    super(StartBisectTest, self).setUp()
    stored_object.Set(
        start_try_job._TESTER_DIRECTOR_MAP_KEY,
        {
            'linux_perf_tester': 'linux_perf_bisector',
            'win64_nv_tester': 'linux_perf_bisector',
        })
    app = webapp2.WSGIApplication(
        [('/start_try_job', start_try_job.StartBisectHandler)])
    self.testapp = webtest.TestApp(app)
    namespaced_stored_object.Set(
        start_try_job._BISECT_BOT_MAP_KEY,
        {
            'ChromiumPerf': [
                ('nexus4', 'android_nexus4_perf_bisect'),
                ('nexus7', 'android_nexus7_perf_bisect'),
                ('win8', 'win_8_perf_bisect'),
                ('xp', 'win_xp_perf_bisect'),
                ('android', 'android_nexus7_perf_bisect'),
                ('mac', 'mac_perf_bisect'),
                ('win', 'win_perf_bisect'),
                ('linux', 'linux_perf_bisect'),
                ('', 'linux_perf_bisect'),
            ],
        })
    namespaced_stored_object.Set(
        start_try_job._BUILDER_TYPES_KEY,
        {'ChromiumPerf': 'perf', 'OtherMaster': 'foo'})
    namespaced_stored_object.Set(
        start_try_job._BOT_BROWSER_MAP_KEY,
        [
            ['android', 'android-chromium'],
            ['winx64', 'release_x64'],
            ['win_x64', 'release_x64'],
            ['', 'release'],
        ])
    testing_common.SetSheriffDomains(['chromium.org'])
    # Add fake Rietveld auth info.
    rietveld_config = rietveld_service.RietveldConfig(
        id='default_rietveld_config',
        client_email='sullivan@chromium.org',
        service_account_key='Fake Account Key',
        server_url='https://test-rietveld.appspot.com')
    rietveld_config.put()

  def testPost_InvalidUser_ShowsErrorMessage(self):
    self.SetCurrentUser('foo@yahoo.com')
    response = self.testapp.post('/start_try_job', {
        'test_path': 'ChromiumPerf/win7/morejs/times/page_load_time',
        'step': 'prefill-info',
    })
    self.assertEqual(
        {'error': 'User "foo@yahoo.com" not authorized.'},
        json.loads(response.body))

  def testPost_PrefillInfoStep(self):
    self.SetCurrentUser('foo@chromium.org')
    testing_common.AddTests(
        ['ChromiumPerf'],
        ['win7',
         'android-nexus7',
         'chromium-rel-win8-dual',
         'chromium-rel-xp-single'], {
             'page_cycler.morejs': {
                 'times': {
                     'page_load_time': {},
                     'page_load_time_ref': {},
                     'blog.chromium.org': {},
                     'dev.chromium.org': {},
                     'test.blogspot.com': {},
                     'http___test.com_': {}
                 },
                 'vm_final_size_renderer': {
                     'ref': {},
                     'vm_final_size_renderer_extcs1': {}
                 },
             },
             'blink_perf': {
                 'Animation_balls': {}
             }
         }
    )
    tests = graph_data.Test.query().fetch()
    for test in tests:
      name = test.key.string_id()
      if name in ('times', 'page_cycler.morejs', 'blink_perf'):
        continue
      test.has_rows = True
    ndb.put_multi(tests)

    response = self.testapp.post('/start_try_job', {
        'test_path': ('ChromiumPerf/win7/page_cycler.morejs/'
                      'times/page_load_time'),
        'step': 'prefill-info',
    })
    info = json.loads(response.body)
    self.assertEqual('win_perf_bisect', info['bisect_bot'])
    self.assertEqual('foo@chromium.org', info['email'])
    self.assertEqual('page_cycler.morejs', info['suite'])
    self.assertEqual('times/page_load_time', info['default_metric'])
    self.assertEqual('ChromiumPerf', info['master'])
    self.assertFalse(info['internal_only'])
    self.assertTrue(info['use_archive'])
    self.assertEqual(
        [
            'android_nexus4_perf_bisect',
            'android_nexus7_perf_bisect',
            'linux_perf_bisect',
            'mac_perf_bisect',
            'win_8_perf_bisect',
            'win_perf_bisect',
            'win_xp_perf_bisect',
        ], info['all_bots'])
    self.assertEqual(
        [
            'times/blog.chromium.org',
            'times/dev.chromium.org',
            'times/http___test.com_',
            'times/page_load_time',
            'times/test.blogspot.com'
        ],
        info['all_metrics'])

    response = self.testapp.post('/start_try_job', {
        'test_path': ('ChromiumPerf/win7/page_cycler.morejs/'
                      'vm_final_size_renderer'),
        'step': 'prefill-info',
    })
    info = json.loads(response.body)
    self.assertEqual(
        ['vm_final_size_renderer/vm_final_size_renderer',
         'vm_final_size_renderer/vm_final_size_renderer_extcs1'],
        info['all_metrics'])

    response = self.testapp.post('/start_try_job', {
        'test_path': 'ChromiumPerf/win7/blink_perf/Animation_balls',
        'step': 'prefill-info',
    })
    info = json.loads(response.body)
    self.assertEqual('Animation_balls/Animation_balls', info['default_metric'])

    response = self.testapp.post('/start_try_job', {
        'test_path': 'ChromiumPerf/android-nexus7/blink_perf/Animation_balls',
        'step': 'prefill-info',
    })
    info = json.loads(response.body)
    self.assertEqual('android_nexus7_perf_bisect', info['bisect_bot'])

    response = self.testapp.post('/start_try_job', {
        'test_path': ('ChromiumPerf/chromium-rel-win8-dual/'
                      'blink_perf/Animation_balls'),
        'step': 'prefill-info',
    })
    info = json.loads(response.body)
    self.assertEqual('win_8_perf_bisect', info['bisect_bot'])

    response = self.testapp.post('/start_try_job', {
        'test_path': ('ChromiumPerf/chromium-rel-xp-single/'
                      'blink_perf/Animation_balls'),
        'step': 'prefill-info',
    })
    info = json.loads(response.body)
    self.assertEqual('win_xp_perf_bisect', info['bisect_bot'])

  def _TestGetBisectConfig(self, parameters, expected_config_dict):
    """Helper method to test get-config requests."""
    response = start_try_job.GetBisectConfig(**parameters)
    self.assertEqual(expected_config_dict, response)

  def testGetConfig_EmptyUseArchiveParameter_GivesEmptyBuilderType(self):
    self._TestGetBisectConfig(
        {
            'bisect_bot': 'linux_perf_bisect',
            'master_name': 'ChromiumPerf',
            'suite': 'page_cycler.moz',
            'metric': 'times/page_load_time',
            'good_revision': '265549',
            'bad_revision': '265556',
            'repeat_count': '15',
            'max_time_minutes': '8',
            'bug_id': '-1',
            'use_archive': '',
        },
        {
            'command': ('tools/perf/run_benchmark -v '
                        '--browser=release --output-format=buildbot '
                        '--also-run-disabled-tests '
                        'page_cycler.moz'),
            'good_revision': '265549',
            'bad_revision': '265556',
            'metric': 'times/page_load_time',
            'repeat_count': '15',
            'max_time_minutes': '8',
            'bug_id': '-1',
            'builder_type': '',
            'target_arch': 'ia32',
            'bisect_mode': 'mean',
        })

  def testGetConfig_UseBuildbucket_ChangesTelemetryOutputFormat(self):
    self._TestGetBisectConfig(
        {
            'bisect_bot': 'linux_perf_tester',
            'master_name': 'ChromiumPerf',
            'suite': 'page_cycler.moz',
            'metric': 'times/page_load_time',
            'good_revision': '265549',
            'bad_revision': '265556',
            'repeat_count': '15',
            'max_time_minutes': '8',
            'bug_id': '-1',
            'use_archive': 'true',
            'use_buildbucket': True,
        },
        {
            'command': ('src/tools/perf/run_benchmark -v '
                        '--browser=release --output-format=chartjson '
                        '--also-run-disabled-tests '
                        'page_cycler.moz'),
            'good_revision': '265549',
            'bad_revision': '265556',
            'metric': 'times/page_load_time',
            'repeat_count': '15',
            'max_time_minutes': '8',
            'bug_id': '-1',
            'builder_type': 'perf',
            'target_arch': 'ia32',
            'bisect_mode': 'mean',
            'recipe_tester_name': 'linux_perf_tester',
        })

  def testGetConfig_NonEmptyUseArchiveParameter_GivesNonEmptyBuilderType(self):
    # Any non-empty value for use_archive means that archives should be used.
    # Even if value of use_archive is "false", archives will still be used!
    self._TestGetBisectConfig(
        {
            'bisect_bot': 'linux_perf_bisect',
            'master_name': 'ChromiumPerf',
            'suite': 'page_cycler.moz',
            'metric': 'times/page_load_time',
            'good_revision': '265549',
            'bad_revision': '265556',
            'repeat_count': '15',
            'max_time_minutes': '8',
            'bug_id': '-1',
            'use_archive': '',
        },
        {
            'command': ('tools/perf/run_benchmark -v '
                        '--browser=release --output-format=buildbot '
                        '--also-run-disabled-tests '
                        'page_cycler.moz'),
            'good_revision': '265549',
            'bad_revision': '265556',
            'metric': 'times/page_load_time',
            'repeat_count': '15',
            'max_time_minutes': '8',
            'bug_id': '-1',
            'builder_type': '',
            'target_arch': 'ia32',
            'bisect_mode': 'mean',
        })

  def testGetConfig_TelemetryTest(self):
    self._TestGetBisectConfig(
        {
            'bisect_bot': 'win_perf_bisect',
            'master_name': 'ChromiumPerf',
            'suite': 'page_cycler.morejs',
            'metric': 'times/page_load_time',
            'good_revision': '12345',
            'bad_revision': '23456',
            'repeat_count': '15',
            'max_time_minutes': '8',
            'bug_id': '-1',
        },
        {
            'command': ('python tools/perf/run_benchmark -v '
                        '--browser=release --output-format=buildbot '
                        '--also-run-disabled-tests '
                        'page_cycler.morejs'),
            'good_revision': '12345',
            'bad_revision': '23456',
            'metric': 'times/page_load_time',
            'repeat_count': '15',
            'max_time_minutes': '8',
            'bug_id': '-1',
            'builder_type': '',
            'target_arch': 'ia32',
            'bisect_mode': 'mean',
        })

  def testGetConfig_BisectModeSetToReturnCode(self):
    self._TestGetBisectConfig(
        {
            'bisect_bot': 'linux_perf_bisect',
            'master_name': 'ChromiumPerf',
            'suite': 'page_cycler.moz',
            'metric': 'times/page_load_time',
            'good_revision': '265549',
            'bad_revision': '265556',
            'repeat_count': '15',
            'max_time_minutes': '8',
            'bug_id': '-1',
            'use_archive': '',
            'bisect_mode': 'return_code',
        },
        {
            'command': ('tools/perf/run_benchmark -v '
                        '--browser=release --output-format=buildbot '
                        '--also-run-disabled-tests '
                        'page_cycler.moz'),
            'good_revision': '265549',
            'bad_revision': '265556',
            'metric': 'times/page_load_time',
            'repeat_count': '15',
            'max_time_minutes': '8',
            'bug_id': '-1',
            'builder_type': '',
            'target_arch': 'ia32',
            'bisect_mode': 'return_code',
        })

  def _TestGetConfigCommand(self, expected_command, **params_to_override):
    """Helper method to test the command returned for a get-config request."""
    parameters = dict(
        {
            'bisect_bot': 'linux_perf_bisect',
            'suite': 'page_cycler.moz',
            'master_name': 'ChromiumPerf',
            'metric': 'times/page_load_time',
            'good_revision': '265549',
            'bad_revision': '265556',
            'repeat_count': '15',
            'max_time_minutes': '8',
            'bug_id': '-1',
            'use_archive': '',
            'use_buildbucket': False,
        }, **params_to_override)
    response = start_try_job.GetBisectConfig(**parameters)
    self.assertEqual(expected_command, response.get('command'))

  def testGetConfig_AndroidTelemetryTest(self):
    self._TestGetConfigCommand(
        ('tools/perf/run_benchmark -v '
         '--browser=android-chromium --output-format=buildbot '
         '--also-run-disabled-tests '
         'page_cycler.morejs'),
        bisect_bot='android_nexus7_perf_bisect',
        suite='page_cycler.morejs')

  def testGetConfig_CCPerftests(self):
    self._TestGetConfigCommand(
        ('./out/Release/cc_perftests '
         '--test-launcher-print-test-stdio=always'),
        bisect_bot='linux_perf_bisect',
        suite='cc_perftests')

  def testGetConfig_AndroidCCPerftests(self):
    self._TestGetConfigCommand(
        'build/android/test_runner.py gtest --release -s cc_perftests',
        bisect_bot='android_nexus7_perf_bisect',
        suite='cc_perftests')

  def testGetConfig_IdbPerf(self):
    self._TestGetConfigCommand(
        (r'.\out\Release\performance_ui_tests.exe '
         '--gtest_filter=IndexedDBTest.Perf'),
        bisect_bot='win_perf_bisect',
        suite='idb_perf')

  def testGetConfig_PerformanceBrowserTests(self):
    self._TestGetConfigCommand(
        ('./out/Release/performance_browser_tests '
         '--test-launcher-print-test-stdio=always '
         '--enable-gpu'),
        bisect_bot='linux_perf_bisect',
        suite='performance_browser_tests')

  def testGuessBisectBot_FetchesNameFromBisectBotMap(self):
    namespaced_stored_object.Set(
        start_try_job._BISECT_BOT_MAP_KEY,
        {'OtherMaster': [('foo', 'super_foo_bisect_bot')]})
    self.assertEqual(
        'super_foo_bisect_bot',
        start_try_job.GuessBisectBot('OtherMaster', 'foo'))

  def testGuessBisectBot_PlatformNotFound_UsesFallback(self):
    namespaced_stored_object.Set(
        start_try_job._BISECT_BOT_MAP_KEY,
        {'OtherMaster': [('foo', 'super_foo_bisect_bot')]})
    self.assertEqual(
        'linux_perf_bisect',
        start_try_job.GuessBisectBot('OtherMaster', 'bar'))

  def testGuessBisectBot_TreatsMasterNameAsPrefix(self):
    namespaced_stored_object.Set(
        start_try_job._BISECT_BOT_MAP_KEY,
        {'OtherMaster': [('foo', 'super_foo_bisect_bot')]})
    self.assertEqual(
        'super_foo_bisect_bot',
        start_try_job.GuessBisectBot('OtherMasterFyi', 'foo'))

  @mock.patch.object(start_try_job.buildbucket_service, 'PutJob',
                     mock.MagicMock(return_value='1234567'))
  def testPerformBuildbucketBisect(self):
    self.SetCurrentUser('foo@chromium.org')

    bug_data.Bug(id=12345).put()

    query_parameters = {
        'bisect_bot': 'linux_perf_tester',
        'suite': 'dromaeo.jslibstylejquery',
        'metric': 'jslib/jslib',
        'good_revision': '215806',
        'bad_revision': '215828',
        'repeat_count': '20',
        'max_time_minutes': '20',
        'bug_id': 12345,
        'use_archive': '',
        'step': 'perform-bisect',
    }
    response = self.testapp.post('/start_try_job', query_parameters)
    response_dict = json.loads(response.body)
    self.assertEqual(response_dict['issue_id'], '1234567')
    self.assertIn('1234567', response_dict['issue_url'])
    job_entities = try_job.TryJob.query(
        try_job.TryJob.buildbucket_job_id == '1234567').fetch()
    self.assertEqual(1, len(job_entities))
    self.assertTrue(job_entities[0].use_buildbucket)

  def testPerformBisect_InvalidConfig_ReturnsError(self):
    bisect_job = try_job.TryJob(
        bot='foo',
        config='config = {}',
        master_name='ChromiumPerf',
        internal_only=False,
        job_type='bisect',
        use_buildbucket=True)
    self.assertEqual(
        {'error': 'No "recipe_tester_name" given.'},
        start_try_job.PerformBisect(bisect_job))

  @mock.patch(
      'google.appengine.api.urlfetch.fetch',
      mock.MagicMock(side_effect=_MockFetch))
  @mock.patch.object(
      start_try_job.rietveld_service.RietveldService, 'MakeRequest',
      mock.MagicMock(side_effect=_MockMakeRequest))
  def testPerformBisect(self):
    self.SetCurrentUser('foo@chromium.org')

    # Create bug.
    bug_data.Bug(id=12345).put()

    query_parameters = {
        'bisect_bot': 'win_perf_bisect',
        'suite': 'dromaeo.jslibstylejquery',
        'metric': 'jslib/jslib',
        'good_revision': '215806',
        'bad_revision': '215828',
        'repeat_count': '20',
        'max_time_minutes': '20',
        'bug_id': 12345,
        'use_archive': '',
        'step': 'perform-bisect',
    }
    global _EXPECTED_CONFIG_DIFF
    global _TEST_EXPECTED_BOT
    global _TEST_EXPECTED_CONFIG_CONTENTS
    _EXPECTED_CONFIG_DIFF = _EXPECTED_BISECT_CONFIG_DIFF
    _TEST_EXPECTED_BOT = 'win_perf_bisect'
    _TEST_EXPECTED_CONFIG_CONTENTS = _BISECT_CONFIG_CONTENTS
    response = self.testapp.post('/start_try_job', query_parameters)
    self.assertEqual(
        json.dumps({'issue_id': '33001',
                    'issue_url': 'https://test-rietveld.appspot.com/33001'}),
        response.body)

  @mock.patch(
      'google.appengine.api.urlfetch.fetch',
      mock.MagicMock(side_effect=_MockFetch))
  @mock.patch.object(
      start_try_job.rietveld_service.RietveldService, 'MakeRequest',
      mock.MagicMock(side_effect=_MockMakeRequest))
  def testPerformPerfTry(self):
    self.SetCurrentUser('foo@chromium.org')

    query_parameters = {
        'bisect_bot': 'linux_perf_bisect',
        'suite': 'dromaeo.jslibstylejquery',
        'good_revision': '215806',
        'bad_revision': '215828',
        'step': 'perform-perf-try',
        'rerun_option': '',
    }
    global _EXPECTED_CONFIG_DIFF
    global _TEST_EXPECTED_CONFIG_CONTENTS
    global _TEST_EXPECTED_BOT
    _EXPECTED_CONFIG_DIFF = _EXPECTED_PERF_CONFIG_DIFF
    _TEST_EXPECTED_CONFIG_CONTENTS = _PERF_CONFIG_CONTENTS
    _TEST_EXPECTED_BOT = 'linux_perf_bisect'
    response = self.testapp.post('/start_try_job', query_parameters)
    self.assertEqual(json.dumps({'issue_id': '33001'}), response.body)

  @mock.patch(
      'google.appengine.api.urlfetch.fetch',
      mock.MagicMock(side_effect=_MockFailedFetch))
  @mock.patch.object(
      start_try_job.rietveld_service.RietveldService, 'MakeRequest',
      mock.MagicMock(side_effect=_MockMakeRequest))
  def testPerformBisectStep_DeleteJobOnFailedBisect(self):
    self.SetCurrentUser('foo@chromium.org')
    # Fake Rietveld auth info
    cfg = rietveld_service.RietveldConfig(
        id='default_rietveld_config',
        client_email='sullivan@chromium.org',
        service_account_key='Fake Account Key',
        server_url='https://test-rietveld.appspot.com/')
    cfg.put()

    query_parameters = {
        'bisect_bot': 'linux_perf_bisect',
        'suite': 'dromaeo.jslibstylejquery',
        'good_revision': '215806',
        'bad_revision': '215828',
        'rerun_option': '',
    }
    global _EXPECTED_CONFIG_DIFF
    global _TEST_EXPECTED_CONFIG_CONTENTS
    global _TEST_EXPECTED_BOT
    _EXPECTED_CONFIG_DIFF = _EXPECTED_PERF_CONFIG_DIFF
    _TEST_EXPECTED_CONFIG_CONTENTS = _PERF_CONFIG_CONTENTS
    _TEST_EXPECTED_BOT = 'linux_perf_bisect'

    query_parameters['step'] = 'perform-bisect'
    self.testapp.post('/start_try_job', query_parameters)
    try_jobs = try_job.TryJob.query().fetch()
    self.assertEqual(0, len(try_jobs))

  @mock.patch(
      'google.appengine.api.urlfetch.fetch',
      mock.MagicMock(side_effect=_MockFailedFetch))
  @mock.patch.object(
      start_try_job.rietveld_service.RietveldService, 'MakeRequest',
      mock.MagicMock(side_effect=_MockMakeRequest))
  def testPerformPerfTryStep_DeleteJobOnFailedBisect(self):
    self.SetCurrentUser('foo@chromium.org')
    # Fake Rietveld auth info
    cfg = rietveld_service.RietveldConfig(
        id='default_rietveld_config',
        client_email='sullivan@chromium.org',
        service_account_key='Fake Account Key',
        server_url='https://test-rietveld.appspot.com/')
    cfg.put()

    query_parameters = {
        'bisect_bot': 'linux_perf_bisect',
        'suite': 'dromaeo.jslibstylejquery',
        'good_revision': '215806',
        'bad_revision': '215828',
        'rerun_option': '',
    }
    global _EXPECTED_CONFIG_DIFF
    global _TEST_EXPECTED_CONFIG_CONTENTS
    global _TEST_EXPECTED_BOT
    _EXPECTED_CONFIG_DIFF = _EXPECTED_PERF_CONFIG_DIFF
    _TEST_EXPECTED_CONFIG_CONTENTS = _PERF_CONFIG_CONTENTS
    _TEST_EXPECTED_BOT = 'linux_perf_bisect'

    query_parameters['step'] = 'perform-perf-try'
    self.testapp.post('/start_try_job', query_parameters)
    try_jobs = try_job.TryJob.query().fetch()
    self.assertEqual(0, len(try_jobs))

  @mock.patch(
      'google.appengine.api.urlfetch.fetch',
      mock.MagicMock(side_effect=_MockFetch))
  @mock.patch(
      'google.appengine.api.app_identity.get_default_version_hostname',
      mock.MagicMock(return_value='my-dashboard.appspot.com'))
  @mock.patch.object(start_try_job.buildbucket_service, 'PutJob',
                     mock.MagicMock(return_value='1234567'))
  def testPerformBisectWithArchive(self):
    self.SetCurrentUser('foo@chromium.org')

    # Create bug.
    bug_data.Bug(id=12345).put()

    query_parameters = {
        'bisect_bot': 'linux_perf_tester',
        'suite': 'dromaeo.jslibstylejquery',
        'metric': 'jslib/jslib',
        'good_revision': '215806',
        'bad_revision': '215828',
        'repeat_count': '20',
        'max_time_minutes': '20',
        'bug_id': 12345,
        'use_archive': 'true',
        'bisect_mode': 'mean',
        'step': 'perform-bisect',
    }
    response = self.testapp.post('/start_try_job', query_parameters)
    self.assertEqual(
        json.dumps({'issue_id': '1234567',
                    'issue_url': ('https://my-dashboard.appspot.com'
                                  '/buildbucket_job_status/1234567')}),
        response.body)

  def testGetBisectconfig_UseArchive(self):
    self._TestGetBisectConfig(
        {
            'bisect_bot': 'win_perf_bisect',
            'master_name': 'ChromiumPerf',
            'suite': 'page_cycler.morejs',
            'metric': 'times/page_load_time',
            'good_revision': '12345',
            'bad_revision': '23456',
            'repeat_count': '15',
            'max_time_minutes': '8',
            'bug_id': '-1',
            'use_archive': 'true',
        },
        {
            'command': ('python tools/perf/run_benchmark -v '
                        '--browser=release --output-format=buildbot '
                        '--also-run-disabled-tests '
                        'page_cycler.morejs'),
            'good_revision': '12345',
            'bad_revision': '23456',
            'metric': 'times/page_load_time',
            'repeat_count': '15',
            'max_time_minutes': '8',
            'bug_id': '-1',
            'builder_type': 'perf',
            'target_arch': 'ia32',
            'bisect_mode': 'mean',
        })

  def testGetBisectConfig_WithTargetArch(self):
    self._TestGetBisectConfig(
        {
            'bisect_bot': 'win_x64_perf_bisect',
            'master_name': 'ChromiumPerf',
            'suite': 'page_cycler.moz',
            'metric': 'times/page_load_time',
            'good_revision': '265549',
            'bad_revision': '265556',
            'repeat_count': '15',
            'max_time_minutes': '8',
            'bug_id': '-1',
            'use_archive': ''
        },
        {
            'command': ('python tools/perf/run_benchmark -v '
                        '--browser=release_x64 --output-format=buildbot '
                        '--also-run-disabled-tests '
                        'page_cycler.moz'),
            'good_revision': '265549',
            'bad_revision': '265556',
            'metric': 'times/page_load_time',
            'repeat_count': '15',
            'max_time_minutes': '8',
            'bug_id': '-1',
            'builder_type': '',
            'target_arch': 'x64',
            'bisect_mode': 'mean',
        })

  def testGetConfig_UseBuildbucket_AndroidTelemetryTest(self):
    self._TestGetConfigCommand(
        ('src/tools/perf/run_benchmark -v '
         '--browser=android-chromium --output-format=chartjson '
         '--also-run-disabled-tests '
         'page_cycler.morejs'),
        bisect_bot='android_nexus7_perf_bisect',
        suite='page_cycler.morejs',
        use_buildbucket=True)

  def testGetConfig_UseBuildbucket_CCPerftests(self):
    self._TestGetConfigCommand(
        ('./src/out/Release/cc_perftests '
         '--test-launcher-print-test-stdio=always'),
        bisect_bot='linux_perf_bisect',
        suite='cc_perftests',
        use_buildbucket=True)

  def testGetConfig_UseBuildbucket_AndroidCCPerftests(self):
    self._TestGetConfigCommand(
        'src/build/android/test_runner.py gtest --release -s cc_perftests',
        bisect_bot='android_nexus7_perf_bisect',
        suite='cc_perftests',
        use_buildbucket=True)

  def testGetConfig_UseBuildbucket_IdbPerf(self):
    self._TestGetConfigCommand(
        ('.\src\out\Release\performance_ui_tests.exe '
         '--gtest_filter=IndexedDBTest.Perf'),
        bisect_bot='win_perf_bisect',
        suite='idb_perf',
        use_buildbucket=True)

  def testGetConfig_UseBuildbucket_PerformanceBrowserTests(self):
    self._TestGetConfigCommand(
        ('./src/out/Release/performance_browser_tests '
         '--test-launcher-print-test-stdio=always '
         '--enable-gpu'),
        bisect_bot='linux_perf_bisect',
        suite='performance_browser_tests',
        use_buildbucket=True)

  def testGetConfig_X64Bot_UsesX64ReleaseDirectory(self):
    self._TestGetConfigCommand(
        ('.\\src\\out\\Release_x64\\performance_browser_tests.exe '
         '--test-launcher-print-test-stdio=always '
         '--enable-gpu'),
        bisect_bot='winx64nvidia_perf_bisect',
        suite='performance_browser_tests',
        use_buildbucket=True)

  def testGuessMetric_SummaryMetricWithNoTIRLabel(self):
    testing_common.AddTests(
        ['M'], ['b'],
        {'benchmark': {'chart': {'page': {}}}})
    self.assertEqual(
        'chart/chart',
        start_try_job.GuessMetric('M/b/benchmark/chart'))

  def testGuessMetric_SummaryMetricWithTIRLabel(self):
    testing_common.AddTests(
        ['M'], ['b'],
        {'benchmark': {'chart': {'tir_label': {'page': {}}}}})
    self.assertEqual(
        'tir_label-chart/tir_label-chart',
        start_try_job.GuessMetric('M/b/benchmark/chart/tir_label'))


class RewriteMetricNameTests(testing_common.TestCase):

  def testRewriteMetricWithoutInteractionRecord(self):
    self.assertEqual(
        'old/skool',
        start_try_job._RewriteMetricName('old/skool'))

  def testRewriteMetricWithInteractionRecord(self):
    self.assertEqual(
        'interaction-chart/trace',
        start_try_job._RewriteMetricName('chart/interaction/trace'))


if __name__ == '__main__':
  unittest.main()