| # Copyright 2013 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| # |
| # This file lists the pyauto tests that run as a part of the functional test |
| # suite. |
| # |
| # Tests can be enabled on a per-platform basis. Tests not listed here will |
| # not be run. |
| # |
| # Within each platform, tests are specified in up to 3 groups: (1) enabled |
| # tests; (2) tests that are permanently disabled because they do not apply |
| # to the given platform and hence should not be run; and (3) disabled tests |
| # that need to be investigated/fixed. Tests are listed alphabetically within |
| # each group. |
| # |
| # Test names can be specified in any of the following ways: |
| # 1. as a module, in which case all tests in that module will be run |
| # example: 'test_basic' |
| # 2. or as a test case, in which case all tests in that test case will be run |
| # example: 'test_basic.SimpleTest' |
| # 3. or as an individual test |
| # example: 'test_basic.SimpleTest.testCanOpenGoogle' |
| # |
| # Tests beginning with '-' will be excluded. This can be used to enforce |
| # exclusions for a particular platform. |
| # Names beginning with '@' will be treated as suite names and will be |
| # recursively expanded. |
| |
| { |
| # This suite gets run on 'Google Chrome' builds. |
| 'FULL': { |
| 'all': [ |
| 'about_plugins_ui.AboutPluginsUITest', |
| 'about_plugins_ui.ChromeAboutPluginsUITest', |
| 'apptest', |
| 'autofill', |
| 'codesign', |
| 'crash_reporter', |
| 'execute_javascript', |
| 'extensions', |
| 'fullscreen_mouselock', |
| 'gtalk.test_basic', |
| 'infobars', |
| 'media.media_basic_playback', |
| 'multiprofile', |
| 'nacl_sdk', |
| 'ntp', |
| 'omnibox', |
| 'passwords', |
| 'prefs', |
| 'prefs_ui', |
| 'pyauto_webdriver', |
| 'search_engines', |
| 'special_tabs', |
| 'test_pyauto', |
| |
| # =========================== |
| # Permanently-disabled tests. |
| # =========================== |
| |
| # ================================================== |
| # Disabled tests that need to be investigated/fixed. |
| # ================================================== |
| # crbug.com/145006 |
| '-about_plugins_ui.ChromeAboutPluginsUITest.testEnableAndDisableFlashPlugin', |
| # crbug.com/100365 |
| '-autofill.AutofillTest.testDisplayLineItemForEntriesWithNoCCNum', |
| # crbug.com/171828 |
| '-autofill.AutofillTest.testNoDuplicatePhoneNumsInPrefs', |
| # The source is behind. Waiting for dev to automate the update. |
| # crbug.com/109160 |
| '-execute_javascript.ExecuteJavascriptTest.testExecuteJavascriptInExtension', |
| # crbug.com/123396 |
| '-fullscreen_mouselock.FullscreenMouselockTest.testPatternsForFSAndML', |
| # crbug.com/132665 |
| '-fullscreen_mouselock.FullscreenMouselockTest.testTabFSExitWhenNavBackToPrevPage', |
| '-fullscreen_mouselock.FullscreenMouselockTest.testTabFSExitWhenNavToNewPage', |
| '-fullscreen_mouselock.FullscreenMouselockTest.testMLExitWhenNavBackToPrevPage', |
| '-fullscreen_mouselock.FullscreenMouselockTest.testMLExitWhenNavToNewPage', |
| # crbug.com/140460 |
| '-fullscreen_mouselock.FullscreenMouselockTest.testNoTabFSExitWhenJSExitMouseLock', |
| # crosbug.com/136875 |
| '-fullscreen_mouselock.FullscreenMouselockTest.testTabFSDoesNotExitForAnchorLinks', |
| # crbug.com/179263 |
| '-gtalk.test_basic.BasicTest.testCurrentVersion', |
| '-gtalk.test_basic.BasicTest.testRCVersion', |
| # crbug.com/165796 |
| '-infobars.OneClickInfobarTest', |
| # crbug.com/131874 |
| '-infobars.OneClickInfobarTest.testNoOneClickInfobarAfterCancel', |
| # crbug.com/133315 |
| '-infobars.OneClickInfobarTest.testDisplayOneClickInfobarAfterDismiss', |
| # Mysteriously broken? |
| # crbug.com/138857 |
| '-multiprofile.MultiprofileTest.testMakeSearchEngineDefaultInMultiprofile', |
| # crbug.com/179268 |
| '-ntp.NTPTest.testCannotUninstallWebStore', |
| '-ntp.NTPTest.testGetAppsInNewProfile', |
| '-ntp.NTPTest.testGetAppsWhenInstallApp', |
| '-ntp.NTPTest.testGetAppsWhenInstallNonApps', |
| '-ntp.NTPTest.testUninstallApp', |
| # crbug.com/143308 |
| '-omnibox.OmniboxLiveTest.testGoogleSearch', |
| # crbug.com/71715 |
| '-omnibox.OmniboxTest.testHistoryResult', |
| # crbug.com/123019 |
| '-omnibox.OmniboxTest.testAutoCompleteForNonAsciiSearch', |
| # crbug.com/137041 |
| '-omnibox.OmniboxTest.testDifferentTypesOfResults', |
| # crbug.com/162341 |
| '-prefs.PrefsTest.testAllowSelectedGeoTracking', |
| '-prefs.PrefsTest.testDismissedInfobarSavesNoEntry', |
| '-prefs.PrefsTest.testGeolocationBlockedWhenTrackingDenied', |
| '-prefs.PrefsTest.testGeolocationPref', |
| # crbug.com/85600 |
| '-prefs.PrefsTest.testNavigationStateOnSessionRestore', |
| # crbug.com/157271 |
| '-prefs_ui.BasicSettingsUITest.testCancelStartupURLSetting', |
| '-prefs_ui.BasicSettingsUITest.testSetStartupPages', |
| '-prefs_ui.BasicSettingsUITest.testUseCurrentPagesForStartup', |
| # crbug.com/162341 |
| '-prefs_ui.PrefsUITest.testBehaviorValueCorrectlyDisplayed', |
| # crbug.com/112051 |
| '-prefs_ui.PrefsUITest.testChangeExceptionBehaviorUI', |
| # crbug.com/157271 |
| '-prefs_ui.PrefsUITest.testDeleteExceptionUI', |
| # crbug.com/151973 |
| '-prefs_ui.PrefsUITest.testInitialLineEntryInIncognitoUI', |
| # crbug.com/162341 |
| '-prefs_ui.PrefsUITest.testLocationSettingOptionsUI', |
| '-prefs_ui.PrefsUITest.testNoInitialLineEntryInUI', |
| # crbug.com/132285 |
| '-special_tabs.SpecialTabsTest.testSpecialURLTabs', |
| # crbug.com/168081 |
| '-ntp.NTPTest.testCloseOneTab', |
| ], |
| |
| 'win': [ |
| 'gpu', |
| # =========================== |
| # Permanently-disabled tests. |
| # =========================== |
| |
| # ================================================== |
| # Disabled tests that need to be investigated/fixed. |
| # ================================================== |
| # crbug.com/105948 |
| '-autofill.AutofillTest.testPostalCodeAndStateLabelsBasedOnCountry', |
| # crbug.com/111289 |
| '-extensions.ExtensionsTest.testAllowAccessFileURLs', |
| # crbug.com/113090 |
| '-extensions.ExtensionsTest.testAllowIncognitoExtension', |
| # crbug.com/171490 |
| '-passwords.PasswordTest.testPasswdInfoNotStoredWhenAutocompleteOff', |
| # crbug.com/117569 |
| '-passwords.PasswordTest.testSavedPasswordInTabsAndWindows', |
| # crbug.com/98526 |
| '-pyauto_webdriver.PyAutoWebDriverTest.testCanConnectToRestartedBrowser', |
| '-pyauto_webdriver.PyAutoWebDriverTest.testTypeIntoTextBox', |
| ], |
| |
| 'mac': [ |
| # =========================== |
| # Permanently-disabled tests. |
| # =========================== |
| |
| # ================================================== |
| # Disabled tests that need to be investigated/fixed. |
| # ================================================== |
| # Keychain popups make autofill/password tests difficult: crbug.com/49378 |
| '-prefs_ui.PrefsUITest.testSetPasswordAndDelete', |
| # codesign tests should run *after* signing. crbug.com/50481 |
| '-codesign', |
| # crbug.com/124922 |
| '-fullscreen_mouselock.FullscreenMouselockTest.testMouseLockExitWhenBrowserLoseFocus', |
| # crbug.com/125989 |
| '-fullscreen_mouselock.FullscreenMouselockTest.testMouseLockExitWhenAlertDialogShow', |
| # Fails on chrome-mac-10_7-qa only: crbug.com/124886 |
| '-fullscreen_mouselock.FullscreenMouselockTest.testPrefsForFullscreenExit', |
| '-fullscreen_mouselock.FullscreenMouselockTest.testNoMouseLockWhenCancelFS', |
| # crbug.com/121484 |
| '-multiprofile.MultiprofileTest.test20NewProfiles', |
| '-ntp.NTPTest.testLaunchAppNewWindow', # crbug.com/79812 |
| # crbug.com/70437 |
| '-omnibox.OmniboxTest.testHistoryResult', |
| # crbug.com/91617 |
| '-omnibox.OmniboxTest.testContentHistory', |
| # Keychain popups make autofill/password tests difficult: crbug.com/49378 |
| '-passwords', |
| # crbug.com/69619 |
| '-search_engines.SearchEnginesTest.testDiscoverSearchEngine', |
| # crbug.com/98526 |
| '-pyauto_webdriver.PyAutoWebDriverTest.testCanConnectToRestartedBrowser', |
| '-pyauto_webdriver.PyAutoWebDriverTest.testTypeIntoTextBox', |
| ], |
| |
| 'linux': [ |
| 'test_clean_exit', |
| |
| # =========================== |
| # Permanently-disabled tests. |
| # =========================== |
| # System password manager obstructs password automation. |
| '-passwords', |
| |
| # ================================================== |
| # Disabled tests that need to be investigated/fixed. |
| # ================================================== |
| # crbug.com/111289 |
| '-extensions.ExtensionsTest.testAllowAccessFileURLs', |
| # crbug.com/91033 |
| '-omnibox.OmniboxTest.testOmniboxSearchHistory', |
| ], |
| |
| 'chromeos': [ |
| 'chromeos_basic', |
| 'chromeos_browser', |
| 'chromeos_crosh', |
| 'chromeos_file_browser', |
| 'chromeos_power', |
| 'chromeos_prefs', |
| 'chromeos_security', |
| 'chromeos_time', |
| 'doc_viewing', |
| 'secure_shell', |
| 'youtube', |
| |
| # =========================== |
| # Permanently-disabled tests. |
| # =========================== |
| # No codesign verification on ChromeOS. |
| '-codesign', |
| # Sync is already signed in with the login account. |
| # So one-click infobar tests do tno apply to chromeos. |
| '-infobars.OneClickInfobarTest', |
| # Multi-profile doesn't apply to chromeos yet. |
| '-multiprofile', |
| '-ntp.NTPTest.testDifferentProfileNotAppearInMostVisited', |
| # No NaCl support on ChromeOS. |
| '-nacl_sdk', |
| |
| # ================================================== |
| # Disabled tests that need to be investigated/fixed. |
| # ================================================== |
| # crbug.com/132337 |
| '-autofill.AutofillTest.testTabOrderForEditAddress', |
| # crosbug.com/19556 |
| '-extensions.ExtensionsTest.testAllowAccessFileURLs', |
| '-extensions.ExtensionsTest.testAllowIncognitoExtension', |
| '-extensions.ExtensionsTest.testDisableEnableExtension', |
| # crbug.com/134593 |
| '-gtalk.test_basic.BasicTest.testCurrentVersion', |
| '-gtalk.test_basic.BasicTest.testRCVersion', |
| # crosbug.com/24496 |
| '-infobars.InfobarTest.testPluginCrashForMultiTabs', |
| # crbug.com/109035 |
| '-infobars.InfobarTest.testPluginCrashInfobar', |
| # crosbug.com/14256 |
| '-ntp.NTPTest.testLaunchAppFullScreen', |
| # Content history broken in omnibox. crosbug.com/14416 |
| '-omnibox.OmniboxTest.testContentHistory', |
| # crbug.com/91033 |
| '-omnibox.OmniboxTest.testOmniboxSearchHistory', |
| # crosbug.com/19760 |
| '-passwords.PasswordTest.testClearFetchedCredForNewUserName', |
| '-passwords.PasswordTest.testSavedPasswordInTabsAndWindows', |
| # onunload popups get created in the same window on chromeos |
| # Session restore not working with PyAuto. crosbug.com/12648 |
| '-prefs.PrefsTest.testNavigationStateOnSessionRestore', |
| '-prefs.PrefsTest.testSessionRestoreURLs', |
| '-prefs.PrefsTest.testSessionRestore', |
| # Deal with i18n chars. crosbug.com/12639 |
| '-omnibox.OmniboxTest.testCrazyFilenames', |
| # crosbug.com/20025 |
| '-chromeos_browser.ChromeosBrowserTest.testFullScreen', |
| # Chrome driver does not work in Chrome OS. |
| # crosbug.com/19556 |
| '-prefs_ui', |
| '-prefs.PrefsTest.testGeolocationBlockedWhenTrackingDenied', |
| '-prefs.PrefsTest.testGeolocationPref', |
| '-pyauto_webdriver', |
| ], |
| }, |
| |
| # Performance tests. |
| 'PERFORMANCE': { |
| 'all': [ |
| 'perf', |
| |
| # ================================================== |
| # Disabled tests that need to be investigated/fixed. |
| # ================================================== |
| '-perf.HTML5BenchmarkTest', # crbug.com/134476 |
| '-perf.LiveWebappLoadTest.testNewTabGmail', # crbug.com/136554 |
| '-perf.ScrollTest.testGmailScroll', # crbug.com/136554 |
| '-perf.WebGLTest.testWebGLField', # crbug.com/132797 |
| |
| # =========================== |
| # Permanently-disabled tests. |
| # =========================== |
| # Invoked outside of the pyauto_tests framework. |
| '-perf.BenchmarkPerfTest.testSpaceport', |
| '-perf.PopularSitesScrollTest.test2012Q3', |
| ], |
| 'win': [ |
| # ================================================== |
| # Disabled tests that need to be investigated/fixed. |
| # ================================================== |
| '-perf.GPUPerfTest', # Fails. Discuss with prachij@. |
| '-perf.PageCyclerNetSimTest', # Dependence missing: crbug.com/132559 |
| '-perf.LiveGamePerfTest', # Requires linux /proc/stat. |
| '-perf.YoutubePerfTest', # AttributeError: AssertPlayingState. |
| |
| # =========================== |
| # Permanently-disabled tests. |
| # =========================== |
| '-perf.MemoryTest', # Designed only for ChromeOS. |
| '-perf.NetflixPerfTest', # Designed only for ChromeOS. |
| ], |
| 'mac': [ |
| # ================================================== |
| # Disabled tests that need to be investigated/fixed. |
| # ================================================== |
| '-perf.GPUPerfTest', # Fails. Discuss with prachij@. |
| '-perf.PageCyclerNetSimTest', # Dependence missing: crbug.com/132559 |
| '-perf.LiveGamePerfTest', # Requires linux /proc/stat. |
| '-perf.YoutubePerfTest', # AttributeError: AssertPlayingState. |
| |
| # =========================== |
| # Permanently-disabled tests. |
| # =========================== |
| '-perf.MemoryTest', # Designed only for ChromeOS. |
| '-perf.NetflixPerfTest', # Designed only for ChromeOS. |
| ], |
| 'linux': [ |
| # ================================================== |
| # Disabled tests that need to be investigated/fixed. |
| # ================================================== |
| '-perf.GPUPerfTest', # Fails. Discuss with prachij@. |
| '-perf.PageCyclerNetSimTest', # Dependence missing: crbug.com/132559 |
| '-perf.WebGLTest', # May not render WebGL; need to verify on QA bot. |
| '-perf.YoutubePerfTest', # AttributeError: AssertPlayingState. |
| |
| # =========================== |
| # Permanently-disabled tests. |
| # =========================== |
| '-perf.MemoryTest', # Designed only for ChromeOS. |
| '-perf.NetflixPerfTest', # Designed only for ChromeOS. |
| ], |
| 'chromeos': [ |
| # ================================================== |
| # Disabled tests that need to be investigated/fixed. |
| # ================================================== |
| '-perf.NetflixPerfTest', # crosbug.com/32320. |
| '-perf.PageCyclerNetSimTest', # Investigate Web Page Replay on ChromeOS. |
| '-perf.YoutubePerfTest', # crbug.com/233706 |
| |
| # =========================== |
| # Permanently-disabled tests. |
| # =========================== |
| '-perf.GPUPerfTest', # Designed only for Chrome desktop. |
| ], |
| }, |
| |
| 'PERFORMANCE_LAB': { |
| 'all': [ |
| 'perf.BenchmarkPerfTest.testV8BenchmarkSuite', |
| 'perf.FlashTest.testFlashGaming', |
| 'perf.ScrollTest.testGooglePlusScroll', |
| 'perf.TabPerfTest.test20Tabs', |
| 'perf.WebGLTest.testWebGLSpaceRocks', |
| ], |
| }, |
| |
| 'EMPTY': { |
| }, |
| |
| # ChromeOS flash tests. |
| 'CHROMEOS_FLASH': { |
| 'chromeos': [ |
| 'flash', |
| ], |
| }, |
| |
| # ChromeOS volume tests. |
| 'CHROMEOS_VOLUME': { |
| 'chromeos': [ |
| 'chromeos_volume', |
| ], |
| }, |
| |
| # ChromeOS Accessibility tests. |
| 'CHROMEOS_ACCESSIBILITY': { |
| 'chromeos': [ |
| 'chromeos_accessibility', |
| ], |
| }, |
| |
| # PGO performance suite. |
| 'PGO': { |
| 'chromeos': [ |
| 'perf.BenchmarkPerfTest.testV8BenchmarkSuite', |
| 'perf.PageCyclerTest', |
| 'perf.ScrollTest.testGooglePlusScroll', |
| ], |
| }, |
| |
| # Subset of ChromeOS performance tests for seaboard. |
| 'CHROMEOS_PERF_SEABOARD': { |
| 'chromeos': [ |
| 'perf.ScrollTest.testBlankPageScroll', |
| 'perf.ScrollTest.testGooglePlusScroll', |
| 'perf.ScrollTest.testTextScroll', |
| ], |
| }, |
| |
| # HTML5 media performance tests. |
| 'AV_PERF': { |
| 'linux': [ |
| 'media.audio_latency_perf', |
| 'media.audio_playback_perf', |
| 'media.media_constrained_network_perf', |
| 'media.media_scrub_perf', |
| 'media.media_seek_perf', |
| 'media.media_stat_perf', |
| 'media.mixed_audio_latency_perf', |
| ], |
| 'win': [ |
| 'media.audio_latency_perf', |
| 'media.audio_playback_perf', |
| 'media.media_constrained_network_perf', |
| 'media.media_scrub_perf', |
| 'media.media_seek_perf', |
| 'media.media_stat_perf', |
| 'media.mixed_audio_latency_perf', |
| ], |
| }, |
| |
| # Trace event tests. |
| 'TRACING': { |
| 'all': [ |
| 'tracing.tracing_smoke_test', |
| ], |
| }, |
| |
| # Chromoting tests. |
| 'CHROMOTING': { |
| 'all': [ |
| 'chromoting.auth', |
| 'chromoting.it2me_basic', |
| 'chromoting.me2me_connect', |
| 'chromoting.me2me_enable', |
| ], |
| 'linux': [ |
| # TODO(yihongg): Me2me test against Linux is not working yet. |
| '-chromoting.me2me_connect', |
| '-chromoting.me2me_enable', |
| ], |
| 'chromeos': [ |
| # ChromeOS doesn't yet support the chromoting host. |
| '-chromoting.auth', |
| '-chromoting.it2me_basic', |
| '-chromoting.me2me_connect', |
| '-chromoting.me2me_enable', |
| ], |
| }, |
| |
| # Pyauto functional tests running on coverage bots. |
| 'CODE_COVERAGE': { |
| 'all': [ |
| '@FULL', |
| ], |
| 'linux': [ |
| # These tests fail on coverage bots. Disabling for now. |
| '-test_clean_exit', |
| '-about_plugins_ui', |
| '-autofill', |
| '-pyauto_webdriver', |
| '-extensions', |
| '-flash', |
| '-fullscreen_mouselock', |
| '-gtalk.test_basic', |
| '-infobars', |
| '-multiprofile', |
| '-prefs', |
| '-prefs_ui', |
| '-apptest', |
| '-plugins', |
| '-omnibox', |
| '-special_tabs', |
| '-ntp.NTPTest.testLaunchAppFullScreen', |
| '-media.media_basic_playback.MediaBasicPlaybackTest.testBasicPlaybackMatrix', |
| '-search_engines.SearchEnginesTest.testDiscoverSearchEngine', |
| '-ntp.NTPTest.testUninstallApp', |
| '-ntp.NTPTest.testLaunchAppWithDefaultSettings', |
| '-ntp.NTPTest.testLaunchAppRegularTab', |
| '-ntp.NTPTest.testLaunchAppPinnedTab', |
| '-ntp.NTPTest.testGetAppsWhenInstallApp', |
| '-ntp.NTPTest.testLaunchAppNewWindow', |
| ], |
| }, |
| } |
