| /* |
| * Copyright (C) 2014 The Android Open Source Project |
| * |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| package com.android.cts.tradefed.testtype; |
| |
| import com.android.cts.tradefed.build.StubCtsBuildHelper; |
| import com.android.cts.tradefed.UnitTests; |
| import com.android.cts.util.AbiUtils; |
| import com.android.ddmlib.IShellOutputReceiver; |
| import com.android.ddmlib.testrunner.ITestRunListener; |
| import com.android.ddmlib.testrunner.TestIdentifier; |
| import com.android.tradefed.device.DeviceNotAvailableException; |
| import com.android.tradefed.device.ITestDevice; |
| import com.android.tradefed.result.ITestInvocationListener; |
| import com.android.tradefed.testtype.IAbi; |
| |
| import junit.framework.TestCase; |
| |
| import org.easymock.EasyMock; |
| import org.easymock.IAnswer; |
| |
| import java.io.File; |
| import java.util.ArrayList; |
| import java.util.Collections; |
| import java.util.Collection; |
| import java.util.HashMap; |
| import java.util.List; |
| import java.util.Map; |
| |
| /** |
| * Unit tests for {@link DeqpTestRunner}. |
| */ |
| public class DeqpTestRunnerTest extends TestCase { |
| private static final String NAME = "dEQP-GLES3"; |
| private static final String ID = AbiUtils.createId(UnitTests.ABI.getName(), NAME); |
| private static final String CASE_LIST_FILE_NAME = "/sdcard/dEQP-TestCaseList.txt"; |
| private static final String LOG_FILE_NAME = "/sdcard/TestLog.qpa"; |
| private static final String INSTRUMENTATION_NAME = |
| "com.drawelements.deqp/com.drawelements.deqp.testercore.DeqpInstrumentation"; |
| private static final String QUERY_INSTRUMENTATION_NAME = |
| "com.drawelements.deqp/com.drawelements.deqp.platformutil.DeqpPlatformCapabilityQueryInstrumentation"; |
| private static final String DEQP_ONDEVICE_APK = "com.drawelements.deqp.apk"; |
| private static final String DEQP_ONDEVICE_PKG = "com.drawelements.deqp"; |
| private static final String ONLY_LANDSCAPE_FEATURES = |
| "feature:"+DeqpTestRunner.FEATURE_LANDSCAPE; |
| private static final String ALL_FEATURES = |
| ONLY_LANDSCAPE_FEATURES + "\nfeature:"+DeqpTestRunner.FEATURE_PORTRAIT; |
| private static List<Map<String,String>> DEFAULT_INSTANCE_ARGS; |
| |
| static { |
| DEFAULT_INSTANCE_ARGS = new ArrayList<>(1); |
| DEFAULT_INSTANCE_ARGS.add(new HashMap<String,String>()); |
| DEFAULT_INSTANCE_ARGS.iterator().next().put("glconfig", "rgba8888d24s8"); |
| DEFAULT_INSTANCE_ARGS.iterator().next().put("rotation", "unspecified"); |
| DEFAULT_INSTANCE_ARGS.iterator().next().put("surfacetype", "window"); |
| } |
| |
| /** |
| * {@inheritDoc} |
| */ |
| @Override |
| protected void setUp() throws Exception { |
| super.setUp(); |
| } |
| |
| /** |
| * Test version of OpenGL ES. |
| */ |
| private void testGlesVersion(int requiredMajorVersion, int requiredMinorVersion, int majorVersion, int minorVersion) throws Exception { |
| final TestIdentifier testId = new TestIdentifier("dEQP-GLES" |
| + Integer.toString(requiredMajorVersion) + Integer.toString(requiredMinorVersion) |
| + ".info", "version"); |
| |
| final String testPath = "dEQP-GLES" |
| + Integer.toString(requiredMajorVersion) + Integer.toString(requiredMinorVersion) |
| +".info.version"; |
| |
| final String testTrie = "{dEQP-GLES" |
| + Integer.toString(requiredMajorVersion) + Integer.toString(requiredMinorVersion) |
| + "{info{version}}}"; |
| |
| final String resultCode = "Pass"; |
| |
| /* MultiLineReceiver expects "\r\n" line ending. */ |
| final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=" + resultCode + "\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Detail" + resultCode + "\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_CODE: 0\r\n"; |
| |
| ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); |
| ITestInvocationListener mockListener |
| = EasyMock.createStrictMock(ITestInvocationListener.class); |
| Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>(); |
| tests.add(testId); |
| |
| Map<TestIdentifier, List<Map<String, String>>> instance = new HashMap<>(); |
| instance.put(testId, DEFAULT_INSTANCE_ARGS); |
| |
| DeqpTestRunner deqpTest = new DeqpTestRunner(NAME, |
| "dEQP-GLES" + Integer.toString(requiredMajorVersion) |
| + (requiredMinorVersion > 0 ? Integer.toString(requiredMinorVersion) : ""), |
| tests, instance); |
| deqpTest.setAbi(UnitTests.ABI); |
| |
| int version = (majorVersion << 16) | minorVersion; |
| EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) |
| .andReturn(Integer.toString(version)).atLeastOnce(); |
| |
| if (majorVersion > requiredMajorVersion |
| || (majorVersion == requiredMajorVersion && minorVersion >= requiredMinorVersion)) { |
| |
| EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))) |
| .andReturn("").once(); |
| EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(), |
| EasyMock.eq(true), |
| EasyMock.eq(AbiUtils.createAbiFlag(UnitTests.ABI.getName())))) |
| .andReturn(null).once(); |
| |
| expectRenderConfigQuery(mockDevice, requiredMajorVersion, requiredMinorVersion); |
| |
| EasyMock.expect(mockDevice.executeShellCommand( |
| EasyMock.eq("rm " + CASE_LIST_FILE_NAME))).andReturn("").once(); |
| |
| EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("rm " + LOG_FILE_NAME))) |
| .andReturn("").once(); |
| |
| EasyMock.expect(mockDevice.pushString(testTrie + "\n", CASE_LIST_FILE_NAME)) |
| .andReturn(true).once(); |
| |
| String command = String.format( |
| "am instrument %s -w -e deqpLogFileName \"%s\" -e deqpCmdLine \"" |
| + "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " |
| + "--deqp-screen-rotation=unspecified " |
| + "--deqp-surface-type=window " |
| + "--deqp-log-images=disable\" " |
| + "-e deqpLogData \"%s\" %s", |
| AbiUtils.createAbiFlag(UnitTests.ABI.getName()), LOG_FILE_NAME, |
| CASE_LIST_FILE_NAME, false, INSTRUMENTATION_NAME); |
| |
| mockDevice.executeShellCommand(EasyMock.eq(command), |
| EasyMock.<IShellOutputReceiver>notNull()); |
| |
| EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() { |
| @Override |
| public Object answer() { |
| IShellOutputReceiver receiver |
| = (IShellOutputReceiver)EasyMock.getCurrentArguments()[1]; |
| |
| receiver.addOutput(output.getBytes(), 0, output.length()); |
| receiver.flush(); |
| |
| return null; |
| } |
| }); |
| |
| EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))) |
| .andReturn("").once(); |
| } |
| |
| mockListener.testRunStarted(ID, 1); |
| EasyMock.expectLastCall().once(); |
| |
| mockListener.testStarted(EasyMock.eq(testId)); |
| EasyMock.expectLastCall().once(); |
| |
| mockListener.testEnded(EasyMock.eq(testId), EasyMock.<Map<String, String>>notNull()); |
| EasyMock.expectLastCall().once(); |
| |
| mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull()); |
| EasyMock.expectLastCall().once(); |
| |
| EasyMock.replay(mockDevice); |
| EasyMock.replay(mockListener); |
| |
| deqpTest.setDevice(mockDevice); |
| deqpTest.setBuildHelper(new StubCtsBuildHelper()); |
| deqpTest.run(mockListener); |
| |
| EasyMock.verify(mockListener); |
| EasyMock.verify(mockDevice); |
| } |
| |
| private void expectRenderConfigQuery(ITestDevice mockDevice, int majorVersion, int minorVersion) |
| throws Exception { |
| expectRenderConfigQuery(mockDevice, String.format("--deqp-gl-config-name=rgba8888d24s8 " |
| + "--deqp-screen-rotation=unspecified " |
| + "--deqp-surface-type=window " |
| + "--deqp-gl-major-version=%d " |
| + "--deqp-gl-minor-version=%d", majorVersion, minorVersion)); |
| } |
| |
| private void expectRenderConfigQuery(ITestDevice mockDevice, String commandLine) |
| throws Exception { |
| expectRenderConfigQueryAndReturn(mockDevice, commandLine, "Yes"); |
| } |
| |
| private void expectRenderConfigQueryAndReturn(ITestDevice mockDevice, String commandLine, |
| String output) throws Exception { |
| final String queryOutput = "INSTRUMENTATION_RESULT: Supported=" + output + "\r\n" |
| + "INSTRUMENTATION_CODE: 0\r\n"; |
| final String command = String.format( |
| "am instrument %s -w -e deqpQueryType renderConfigSupported -e deqpCmdLine " |
| + "\"%s\" %s", |
| AbiUtils.createAbiFlag(UnitTests.ABI.getName()), commandLine, QUERY_INSTRUMENTATION_NAME); |
| |
| mockDevice.executeShellCommand(EasyMock.eq(command), |
| EasyMock.<IShellOutputReceiver>notNull()); |
| |
| EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() { |
| @Override |
| public Object answer() { |
| IShellOutputReceiver receiver |
| = (IShellOutputReceiver)EasyMock.getCurrentArguments()[1]; |
| |
| receiver.addOutput(queryOutput.getBytes(), 0, queryOutput.length()); |
| receiver.flush(); |
| |
| return null; |
| } |
| }); |
| } |
| |
| /** |
| * Test that result code produces correctly pass or fail. |
| */ |
| private void testResultCode(final String resultCode, boolean pass) throws Exception { |
| final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.info", "version"); |
| final String testPath = "dEQP-GLES3.info.version"; |
| final String testTrie = "{dEQP-GLES3{info{version}}}"; |
| |
| /* MultiLineReceiver expects "\r\n" line ending. */ |
| final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=" + resultCode + "\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Detail" + resultCode + "\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_CODE: 0\r\n"; |
| |
| ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); |
| ITestInvocationListener mockListener |
| = EasyMock.createStrictMock(ITestInvocationListener.class); |
| Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>(); |
| tests.add(testId); |
| |
| Map<TestIdentifier, List<Map<String, String>>> instance = new HashMap<>(); |
| instance.put(testId, DEFAULT_INSTANCE_ARGS); |
| |
| DeqpTestRunner deqpTest = new DeqpTestRunner(NAME, NAME, tests, instance); |
| deqpTest.setAbi(UnitTests.ABI); |
| |
| int version = 3 << 16; |
| EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) |
| .andReturn(Integer.toString(version)).atLeastOnce(); |
| |
| EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("") |
| .once(); |
| |
| EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(), |
| EasyMock.eq(true), EasyMock.eq(AbiUtils.createAbiFlag(UnitTests.ABI.getName())))) |
| .andReturn(null).once(); |
| |
| expectRenderConfigQuery(mockDevice, 3, 0); |
| |
| EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("rm " + CASE_LIST_FILE_NAME))) |
| .andReturn("").once(); |
| |
| EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("rm " + LOG_FILE_NAME))) |
| .andReturn("").once(); |
| |
| EasyMock.expect(mockDevice.pushString(testTrie + "\n", CASE_LIST_FILE_NAME)).andReturn(true) |
| .once(); |
| |
| String command = String.format( |
| "am instrument %s -w -e deqpLogFileName \"%s\" -e deqpCmdLine \"" |
| + "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " |
| + "--deqp-screen-rotation=unspecified " |
| + "--deqp-surface-type=window " |
| + "--deqp-log-images=disable\" " |
| + "-e deqpLogData \"%s\" %s", |
| AbiUtils.createAbiFlag(UnitTests.ABI.getName()), LOG_FILE_NAME, |
| CASE_LIST_FILE_NAME, false, INSTRUMENTATION_NAME); |
| |
| mockDevice.executeShellCommand(EasyMock.eq(command), |
| EasyMock.<IShellOutputReceiver>notNull()); |
| |
| EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() { |
| @Override |
| public Object answer() { |
| IShellOutputReceiver receiver |
| = (IShellOutputReceiver)EasyMock.getCurrentArguments()[1]; |
| |
| receiver.addOutput(output.getBytes(), 0, output.length()); |
| receiver.flush(); |
| |
| return null; |
| } |
| }); |
| |
| mockListener.testRunStarted(ID, 1); |
| EasyMock.expectLastCall().once(); |
| |
| mockListener.testStarted(EasyMock.eq(testId)); |
| EasyMock.expectLastCall().once(); |
| |
| if (!pass) { |
| mockListener.testFailed(testId, |
| "=== with config {glformat=rgba8888d24s8,rotation=unspecified,surfacetype=window} ===\n" |
| + resultCode + ": Detail" + resultCode); |
| |
| EasyMock.expectLastCall().once(); |
| } |
| |
| mockListener.testEnded(EasyMock.eq(testId), EasyMock.<Map<String, String>>notNull()); |
| EasyMock.expectLastCall().once(); |
| |
| mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull()); |
| EasyMock.expectLastCall().once(); |
| |
| EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("") |
| .once(); |
| |
| EasyMock.replay(mockDevice); |
| EasyMock.replay(mockListener); |
| |
| deqpTest.setDevice(mockDevice); |
| deqpTest.setBuildHelper(new StubCtsBuildHelper()); |
| deqpTest.run(mockListener); |
| |
| EasyMock.verify(mockListener); |
| EasyMock.verify(mockDevice); |
| } |
| |
| /** |
| * Test running multiple test cases. |
| */ |
| public void testRun_multipleTests() throws Exception { |
| /* MultiLineReceiver expects "\r\n" line ending. */ |
| final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.vendor\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.renderer\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.version\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.shading_language_version\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.extensions\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.render_target\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_CODE: 0\r\n"; |
| |
| final TestIdentifier[] testIds = { |
| new TestIdentifier("dEQP-GLES3.info", "vendor"), |
| new TestIdentifier("dEQP-GLES3.info", "renderer"), |
| new TestIdentifier("dEQP-GLES3.info", "version"), |
| new TestIdentifier("dEQP-GLES3.info", "shading_language_version"), |
| new TestIdentifier("dEQP-GLES3.info", "extensions"), |
| new TestIdentifier("dEQP-GLES3.info", "render_target") |
| }; |
| |
| final String[] testPaths = { |
| "dEQP-GLES3.info.vendor", |
| "dEQP-GLES3.info.renderer", |
| "dEQP-GLES3.info.version", |
| "dEQP-GLES3.info.shading_language_version", |
| "dEQP-GLES3.info.extensions", |
| "dEQP-GLES3.info.render_target" |
| }; |
| |
| final String testTrie |
| = "{dEQP-GLES3{info{vendor,renderer,version,shading_language_version,extensions,render_target}}}"; |
| |
| ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); |
| ITestInvocationListener mockListener |
| = EasyMock.createStrictMock(ITestInvocationListener.class); |
| Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>(); |
| Map<TestIdentifier, List<Map<String, String>>> instances = new HashMap<>(); |
| |
| for (TestIdentifier id : testIds) { |
| tests.add(id); |
| instances.put(id, DEFAULT_INSTANCE_ARGS); |
| } |
| |
| DeqpTestRunner deqpTest = new DeqpTestRunner(NAME, NAME, tests, instances); |
| deqpTest.setAbi(UnitTests.ABI); |
| |
| int version = 3 << 16; |
| EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) |
| .andReturn(Integer.toString(version)).atLeastOnce(); |
| |
| EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("") |
| .once(); |
| EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(), |
| EasyMock.eq(true), EasyMock.eq(AbiUtils.createAbiFlag(UnitTests.ABI.getName())))) |
| .andReturn(null).once(); |
| |
| expectRenderConfigQuery(mockDevice, 3, 0); |
| |
| EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("rm " + CASE_LIST_FILE_NAME))) |
| .andReturn("").once(); |
| |
| EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("rm " + LOG_FILE_NAME))) |
| .andReturn("").once(); |
| |
| EasyMock.expect(mockDevice.pushString(testTrie + "\n", CASE_LIST_FILE_NAME)) |
| .andReturn(true).once(); |
| |
| String command = String.format( |
| "am instrument %s -w -e deqpLogFileName \"%s\" -e deqpCmdLine \"" |
| + "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " |
| + "--deqp-screen-rotation=unspecified " |
| + "--deqp-surface-type=window " |
| + "--deqp-log-images=disable\" " |
| + "-e deqpLogData \"%s\" %s", |
| AbiUtils.createAbiFlag(UnitTests.ABI.getName()), LOG_FILE_NAME, |
| CASE_LIST_FILE_NAME, false, INSTRUMENTATION_NAME); |
| |
| mockDevice.executeShellCommand(EasyMock.eq(command), |
| EasyMock.<IShellOutputReceiver>notNull()); |
| |
| EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() { |
| @Override |
| public Object answer() { |
| IShellOutputReceiver receiver |
| = (IShellOutputReceiver)EasyMock.getCurrentArguments()[1]; |
| |
| receiver.addOutput(output.getBytes(), 0, output.length()); |
| receiver.flush(); |
| |
| return null; |
| } |
| }); |
| |
| mockListener.testRunStarted(ID, testPaths.length); |
| EasyMock.expectLastCall().once(); |
| |
| for (int i = 0; i < testPaths.length; i++) { |
| mockListener.testStarted(EasyMock.eq(testIds[i])); |
| EasyMock.expectLastCall().once(); |
| |
| mockListener.testEnded(EasyMock.eq(testIds[i]), |
| EasyMock.<Map<String, String>>notNull()); |
| |
| EasyMock.expectLastCall().once(); |
| } |
| |
| mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull()); |
| EasyMock.expectLastCall().once(); |
| |
| EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("") |
| .once(); |
| |
| EasyMock.replay(mockDevice); |
| EasyMock.replay(mockListener); |
| |
| deqpTest.setDevice(mockDevice); |
| deqpTest.setBuildHelper(new StubCtsBuildHelper()); |
| deqpTest.run(mockListener); |
| |
| EasyMock.verify(mockListener); |
| EasyMock.verify(mockDevice); |
| } |
| |
| /** |
| * Test that test are left unexecuted if pm list query fails |
| */ |
| public void testRun_queryPmListFailure() |
| throws Exception { |
| final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.orientation", "test"); |
| |
| ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); |
| ITestInvocationListener mockListener |
| = EasyMock.createStrictMock(ITestInvocationListener.class); |
| Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>(); |
| tests.add(testId); |
| |
| Map<TestIdentifier, List<Map<String, String>>> instance = new HashMap<>(); |
| instance.put(testId, new ArrayList<Map<String,String>>(1)); |
| instance.get(testId).add(new HashMap<String,String>()); |
| instance.get(testId).iterator().next().put("glconfig", "rgba8888d24s8"); |
| instance.get(testId).iterator().next().put("rotation", "90"); |
| instance.get(testId).iterator().next().put("surfacetype", "window"); |
| |
| DeqpTestRunner deqpTest = new DeqpTestRunner(NAME, NAME, tests, instance); |
| deqpTest.setAbi(UnitTests.ABI); |
| deqpTest.setDevice(mockDevice); |
| deqpTest.setBuildHelper(new StubCtsBuildHelper()); |
| |
| int version = 3 << 16; |
| EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) |
| .andReturn(Integer.toString(version)).atLeastOnce(); |
| |
| EasyMock.expect(mockDevice.executeShellCommand("pm list features")) |
| .andReturn("not a valid format"); |
| |
| EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))). |
| andReturn("").once(); |
| |
| EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(), |
| EasyMock.eq(true), |
| EasyMock.eq(AbiUtils.createAbiFlag(UnitTests.ABI.getName())))).andReturn(null) |
| .once(); |
| |
| EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))) |
| .andReturn("").once(); |
| |
| mockListener.testRunStarted(ID, 1); |
| EasyMock.expectLastCall().once(); |
| |
| mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull()); |
| EasyMock.expectLastCall().once(); |
| |
| EasyMock.replay(mockDevice); |
| EasyMock.replay(mockListener); |
| deqpTest.run(mockListener); |
| EasyMock.verify(mockListener); |
| EasyMock.verify(mockDevice); |
| } |
| |
| /** |
| * Test that test are left unexecuted if renderablity query fails |
| */ |
| public void testRun_queryRenderabilityFailure() |
| throws Exception { |
| final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.orientation", "test"); |
| |
| ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); |
| ITestInvocationListener mockListener |
| = EasyMock.createStrictMock(ITestInvocationListener.class); |
| Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>(); |
| tests.add(testId); |
| |
| Map<TestIdentifier, List<Map<String, String>>> instance = new HashMap<>(); |
| instance.put(testId, new ArrayList<Map<String,String>>(1)); |
| instance.get(testId).add(new HashMap<String,String>()); |
| instance.get(testId).iterator().next().put("glconfig", "rgba8888d24s8"); |
| instance.get(testId).iterator().next().put("rotation", "unspecified"); |
| instance.get(testId).iterator().next().put("surfacetype", "window"); |
| |
| DeqpTestRunner deqpTest = new DeqpTestRunner(NAME, NAME, tests, instance); |
| deqpTest.setAbi(UnitTests.ABI); |
| deqpTest.setDevice(mockDevice); |
| deqpTest.setBuildHelper(new StubCtsBuildHelper()); |
| |
| int version = 3 << 16; |
| EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) |
| .andReturn(Integer.toString(version)).atLeastOnce(); |
| |
| EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))). |
| andReturn("").once(); |
| |
| EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(), |
| EasyMock.eq(true), |
| EasyMock.eq(AbiUtils.createAbiFlag(UnitTests.ABI.getName())))).andReturn(null) |
| .once(); |
| |
| expectRenderConfigQueryAndReturn(mockDevice, |
| "--deqp-gl-config-name=rgba8888d24s8 " |
| + "--deqp-screen-rotation=unspecified " |
| + "--deqp-surface-type=window " |
| + "--deqp-gl-major-version=3 " |
| + "--deqp-gl-minor-version=0", "Maybe?"); |
| |
| EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))) |
| .andReturn("").once(); |
| |
| mockListener.testRunStarted(ID, 1); |
| EasyMock.expectLastCall().once(); |
| |
| mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull()); |
| EasyMock.expectLastCall().once(); |
| |
| EasyMock.replay(mockDevice); |
| EasyMock.replay(mockListener); |
| deqpTest.run(mockListener); |
| EasyMock.verify(mockListener); |
| EasyMock.verify(mockDevice); |
| } |
| |
| /** |
| * Test that orientation is supplied to runner correctly |
| */ |
| private void testOrientation(final String rotation, final String featureString) |
| throws Exception { |
| final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.orientation", "test"); |
| final String testPath = "dEQP-GLES3.orientation.test"; |
| final String testTrie = "{dEQP-GLES3{orientation{test}}}"; |
| final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_CODE: 0\r\n"; |
| |
| ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); |
| ITestInvocationListener mockListener |
| = EasyMock.createStrictMock(ITestInvocationListener.class); |
| Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>(); |
| tests.add(testId); |
| |
| Map<TestIdentifier, List<Map<String, String>>> instance = new HashMap<>(); |
| instance.put(testId, new ArrayList<Map<String,String>>(1)); |
| instance.get(testId).add(new HashMap<String,String>()); |
| instance.get(testId).iterator().next().put("glconfig", "rgba8888d24s8"); |
| instance.get(testId).iterator().next().put("rotation", rotation); |
| instance.get(testId).iterator().next().put("surfacetype", "window"); |
| |
| DeqpTestRunner deqpTest = new DeqpTestRunner(NAME, NAME, tests, instance); |
| deqpTest.setAbi(UnitTests.ABI); |
| deqpTest.setDevice(mockDevice); |
| deqpTest.setBuildHelper(new StubCtsBuildHelper()); |
| |
| int version = 3 << 16; |
| EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) |
| .andReturn(Integer.toString(version)).atLeastOnce(); |
| |
| if (!rotation.equals(DeqpTestRunner.BatchRunConfiguration.ROTATION_UNSPECIFIED)) { |
| EasyMock.expect(mockDevice.executeShellCommand("pm list features")) |
| .andReturn(featureString); |
| } |
| |
| final boolean isPortraitOrientation = |
| rotation.equals(DeqpTestRunner.BatchRunConfiguration.ROTATION_PORTRAIT) || |
| rotation.equals(DeqpTestRunner.BatchRunConfiguration.ROTATION_REVERSE_PORTRAIT); |
| final boolean isLandscapeOrientation = |
| rotation.equals(DeqpTestRunner.BatchRunConfiguration.ROTATION_LANDSCAPE) || |
| rotation.equals(DeqpTestRunner.BatchRunConfiguration.ROTATION_REVERSE_LANDSCAPE); |
| final boolean executable = |
| rotation.equals(DeqpTestRunner.BatchRunConfiguration.ROTATION_UNSPECIFIED) || |
| (isPortraitOrientation && |
| featureString.contains(DeqpTestRunner.FEATURE_PORTRAIT)) || |
| (isLandscapeOrientation && |
| featureString.contains(DeqpTestRunner.FEATURE_LANDSCAPE)); |
| |
| EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))). |
| andReturn("").once(); |
| |
| EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(), |
| EasyMock.eq(true), |
| EasyMock.eq(AbiUtils.createAbiFlag(UnitTests.ABI.getName())))).andReturn(null) |
| .once(); |
| |
| if (executable) { |
| expectRenderConfigQuery(mockDevice, String.format( |
| "--deqp-gl-config-name=rgba8888d24s8 --deqp-screen-rotation=%s " |
| + "--deqp-surface-type=window --deqp-gl-major-version=3 " |
| + "--deqp-gl-minor-version=0", rotation)); |
| |
| EasyMock.expect( |
| mockDevice.executeShellCommand(EasyMock.eq("rm " + CASE_LIST_FILE_NAME))) |
| .andReturn("").once(); |
| |
| EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("rm " + LOG_FILE_NAME))) |
| .andReturn("").once(); |
| |
| EasyMock.expect(mockDevice.pushString(testTrie + "\n", CASE_LIST_FILE_NAME)) |
| .andReturn(true).once(); |
| |
| String command = String.format( |
| "am instrument %s -w -e deqpLogFileName \"%s\" -e deqpCmdLine \"" |
| + "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " |
| + "--deqp-screen-rotation=%s " |
| + "--deqp-surface-type=window " |
| + "--deqp-log-images=disable\" " |
| + "-e deqpLogData \"%s\" %s", |
| AbiUtils.createAbiFlag(UnitTests.ABI.getName()), LOG_FILE_NAME, |
| CASE_LIST_FILE_NAME, rotation, false, INSTRUMENTATION_NAME); |
| |
| mockDevice.executeShellCommand(EasyMock.eq(command), |
| EasyMock.<IShellOutputReceiver>notNull()); |
| |
| EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() { |
| @Override |
| public Object answer() { |
| IShellOutputReceiver receiver |
| = (IShellOutputReceiver)EasyMock.getCurrentArguments()[1]; |
| |
| receiver.addOutput(output.getBytes(), 0, output.length()); |
| receiver.flush(); |
| |
| return null; |
| } |
| }); |
| } |
| |
| EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))) |
| .andReturn("").once(); |
| |
| mockListener.testRunStarted(ID, 1); |
| EasyMock.expectLastCall().once(); |
| |
| mockListener.testStarted(EasyMock.eq(testId)); |
| EasyMock.expectLastCall().once(); |
| |
| mockListener.testEnded(EasyMock.eq(testId), EasyMock.<Map<String, String>>notNull()); |
| EasyMock.expectLastCall().once(); |
| |
| mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull()); |
| EasyMock.expectLastCall().once(); |
| |
| EasyMock.replay(mockDevice); |
| EasyMock.replay(mockListener); |
| deqpTest.run(mockListener); |
| EasyMock.verify(mockListener); |
| EasyMock.verify(mockDevice); |
| } |
| |
| /** |
| * Test OpeGL ES3 tests on device with OpenGL ES2. |
| */ |
| public void testRun_require30DeviceVersion20() throws Exception { |
| testGlesVersion(3, 0, 2, 0); |
| } |
| |
| /** |
| * Test OpeGL ES3.1 tests on device with OpenGL ES2. |
| */ |
| public void testRun_require31DeviceVersion20() throws Exception { |
| testGlesVersion(3, 1, 2, 0); |
| } |
| |
| /** |
| * Test OpeGL ES3 tests on device with OpenGL ES3. |
| */ |
| public void testRun_require30DeviceVersion30() throws Exception { |
| testGlesVersion(3, 0, 3, 0); |
| } |
| |
| /** |
| * Test OpeGL ES3.1 tests on device with OpenGL ES3. |
| */ |
| public void testRun_require31DeviceVersion30() throws Exception { |
| testGlesVersion(3, 1, 3, 0); |
| } |
| |
| /** |
| * Test OpeGL ES3 tests on device with OpenGL ES3.1. |
| */ |
| public void testRun_require30DeviceVersion31() throws Exception { |
| testGlesVersion(3, 0, 3, 1); |
| } |
| |
| /** |
| * Test OpeGL ES3.1 tests on device with OpenGL ES3.1. |
| */ |
| public void testRun_require31DeviceVersion31() throws Exception { |
| testGlesVersion(3, 1, 3, 1); |
| } |
| |
| /** |
| * Test dEQP Pass result code. |
| */ |
| public void testRun_resultPass() throws Exception { |
| testResultCode("Pass", true); |
| } |
| |
| /** |
| * Test dEQP Fail result code. |
| */ |
| public void testRun_resultFail() throws Exception { |
| testResultCode("Fail", false); |
| } |
| |
| /** |
| * Test dEQP NotSupported result code. |
| */ |
| public void testRun_resultNotSupported() throws Exception { |
| testResultCode("NotSupported", true); |
| } |
| |
| /** |
| * Test dEQP QualityWarning result code. |
| */ |
| public void testRun_resultQualityWarning() throws Exception { |
| testResultCode("QualityWarning", true); |
| } |
| |
| /** |
| * Test dEQP CompatibilityWarning result code. |
| */ |
| public void testRun_resultCompatibilityWarning() throws Exception { |
| testResultCode("CompatibilityWarning", true); |
| } |
| |
| /** |
| * Test dEQP ResourceError result code. |
| */ |
| public void testRun_resultResourceError() throws Exception { |
| testResultCode("ResourceError", false); |
| } |
| |
| /** |
| * Test dEQP InternalError result code. |
| */ |
| public void testRun_resultInternalError() throws Exception { |
| testResultCode("InternalError", false); |
| } |
| |
| /** |
| * Test dEQP Crash result code. |
| */ |
| public void testRun_resultCrash() throws Exception { |
| testResultCode("Crash", false); |
| } |
| |
| /** |
| * Test dEQP Timeout result code. |
| */ |
| public void testRun_resultTimeout() throws Exception { |
| testResultCode("Timeout", false); |
| } |
| /** |
| * Test dEQP Orientation |
| */ |
| public void testRun_orientationLandscape() throws Exception { |
| testOrientation("90", ALL_FEATURES); |
| } |
| |
| /** |
| * Test dEQP Orientation |
| */ |
| public void testRun_orientationPortrait() throws Exception { |
| testOrientation("0", ALL_FEATURES); |
| } |
| |
| /** |
| * Test dEQP Orientation |
| */ |
| public void testRun_orientationReverseLandscape() throws Exception { |
| testOrientation("270", ALL_FEATURES); |
| } |
| |
| /** |
| * Test dEQP Orientation |
| */ |
| public void testRun_orientationReversePortrait() throws Exception { |
| testOrientation("180", ALL_FEATURES); |
| } |
| |
| /** |
| * Test dEQP Orientation |
| */ |
| public void testRun_orientationUnspecified() throws Exception { |
| testOrientation("unspecified", ALL_FEATURES); |
| } |
| |
| /** |
| * Test dEQP Orientation with limited features |
| */ |
| public void testRun_orientationUnspecifiedLimitedFeatures() throws Exception { |
| testOrientation("unspecified", ONLY_LANDSCAPE_FEATURES); |
| } |
| |
| /** |
| * Test dEQP Orientation with limited features |
| */ |
| public void testRun_orientationLandscapeLimitedFeatures() throws Exception { |
| testOrientation("90", ONLY_LANDSCAPE_FEATURES); |
| } |
| |
| /** |
| * Test dEQP Orientation with limited features |
| */ |
| public void testRun_orientationPortraitLimitedFeatures() throws Exception { |
| testOrientation("0", ONLY_LANDSCAPE_FEATURES); |
| } |
| |
| /** |
| * Test dEQP unsupported pixel format |
| */ |
| public void testRun_unsupportedPixelFormat() throws Exception { |
| final String pixelFormat = "rgba5658d16m4"; |
| final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.pixelformat", "test"); |
| final String testPath = "dEQP-GLES3.pixelformat.test"; |
| final String testTrie = "{dEQP-GLES3{pixelformat{test}}}"; |
| final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_CODE: 0\r\n"; |
| |
| ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); |
| ITestInvocationListener mockListener |
| = EasyMock.createStrictMock(ITestInvocationListener.class); |
| Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>(); |
| tests.add(testId); |
| |
| Map<TestIdentifier, List<Map<String, String>>> instance = new HashMap<>(); |
| instance.put(testId, new ArrayList<Map<String,String>>(1)); |
| instance.get(testId).add(new HashMap<String,String>()); |
| instance.get(testId).iterator().next().put("glconfig", pixelFormat); |
| instance.get(testId).iterator().next().put("rotation", "unspecified"); |
| instance.get(testId).iterator().next().put("surfacetype", "window"); |
| |
| DeqpTestRunner deqpTest = new DeqpTestRunner(NAME, NAME, tests, instance); |
| deqpTest.setAbi(UnitTests.ABI); |
| deqpTest.setDevice(mockDevice); |
| deqpTest.setBuildHelper(new StubCtsBuildHelper()); |
| |
| int version = 3 << 16; |
| EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) |
| .andReturn(Integer.toString(version)).atLeastOnce(); |
| |
| EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))). |
| andReturn("").once(); |
| |
| EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(), |
| EasyMock.eq(true), |
| EasyMock.eq(AbiUtils.createAbiFlag(UnitTests.ABI.getName())))).andReturn(null) |
| .once(); |
| |
| expectRenderConfigQueryAndReturn(mockDevice, String.format( |
| "--deqp-gl-config-name=%s --deqp-screen-rotation=unspecified " |
| + "--deqp-surface-type=window " |
| + "--deqp-gl-major-version=3 " |
| + "--deqp-gl-minor-version=0", pixelFormat), "No"); |
| |
| EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))) |
| .andReturn("").once(); |
| |
| mockListener.testRunStarted(ID, 1); |
| EasyMock.expectLastCall().once(); |
| |
| mockListener.testStarted(EasyMock.eq(testId)); |
| EasyMock.expectLastCall().once(); |
| |
| mockListener.testEnded(EasyMock.eq(testId), EasyMock.<Map<String, String>>notNull()); |
| EasyMock.expectLastCall().once(); |
| |
| mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull()); |
| EasyMock.expectLastCall().once(); |
| |
| EasyMock.replay(mockDevice); |
| EasyMock.replay(mockListener); |
| deqpTest.run(mockListener); |
| EasyMock.verify(mockListener); |
| EasyMock.verify(mockDevice); |
| } |
| |
| /** |
| * Test dEQP with multiple instances |
| */ |
| public void testRun_multipleInstances() throws Exception { |
| final String instrumentationAnswerConfigA = |
| "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.instances.passall\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.instances.failone\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.instances.crashtwo\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n"; // early eof |
| final String instrumentationAnswerConfigB = |
| "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.instances.passall\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.instances.crashtwo\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-TerminateTestCase-Reason=Magic\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=TerminateTestCase\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.instances.skipone\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_CODE: 0\r\n"; |
| final String instrumentationAnswerConfigC = |
| "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.instances.failone\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Fail\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Fail\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.instances.crashtwo\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_CODE: 0\r\n"; |
| |
| final TestIdentifier[] testIds = { |
| new TestIdentifier("dEQP-GLES3.instances", "passall"), |
| new TestIdentifier("dEQP-GLES3.instances", "failone"), |
| new TestIdentifier("dEQP-GLES3.instances", "crashtwo"), |
| new TestIdentifier("dEQP-GLES3.instances", "skipone"), |
| }; |
| |
| final String[] testPaths = { |
| "dEQP-GLES3.instances.passall", |
| "dEQP-GLES3.instances.failone", |
| "dEQP-GLES3.instances.crashtwo", |
| "dEQP-GLES3.instances.skipone", |
| }; |
| |
| Map<String,String> supportedConfigA = new HashMap<>(); |
| supportedConfigA.put("glconfig", "rgba8888d24s8"); |
| supportedConfigA.put("rotation", "unspecified"); |
| supportedConfigA.put("surfacetype", "window"); |
| |
| Map<String,String> supportedConfigB = new HashMap<>(); |
| supportedConfigB.put("glconfig", "rgba8888d24s8"); |
| supportedConfigB.put("rotation", "90"); |
| supportedConfigB.put("surfacetype", "window"); |
| |
| Map<String,String> supportedConfigC = new HashMap<>(); |
| supportedConfigC.put("glconfig", "rgba8888d24s8"); |
| supportedConfigC.put("rotation", "180"); |
| supportedConfigC.put("surfacetype", "window"); |
| |
| Map<String,String> unsupportedConfig = new HashMap<>(); |
| unsupportedConfig.put("glconfig", "rgb565d16s0"); |
| unsupportedConfig.put("rotation", "unspecified"); |
| unsupportedConfig.put("surfacetype", "window"); |
| |
| Map<TestIdentifier, List<Map<String, String>>> instances = new HashMap<>(); |
| |
| // pass all |
| instances.put(testIds[0], new ArrayList<Map<String,String>>()); |
| instances.get(testIds[0]).add(supportedConfigA); |
| instances.get(testIds[0]).add(supportedConfigB); |
| |
| // fail one |
| instances.put(testIds[1], new ArrayList<Map<String,String>>()); |
| instances.get(testIds[1]).add(supportedConfigA); |
| instances.get(testIds[1]).add(supportedConfigC); |
| |
| // crash two |
| instances.put(testIds[2], new ArrayList<Map<String,String>>()); |
| instances.get(testIds[2]).add(supportedConfigA); |
| instances.get(testIds[2]).add(supportedConfigC); |
| instances.get(testIds[2]).add(supportedConfigB); |
| |
| // skip one |
| instances.put(testIds[3], new ArrayList<Map<String,String>>()); |
| instances.get(testIds[3]).add(supportedConfigB); |
| instances.get(testIds[3]).add(unsupportedConfig); |
| |
| Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>(); |
| for (TestIdentifier id : testIds) { |
| tests.add(id); |
| } |
| |
| ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); |
| ITestInvocationListener mockListener |
| = EasyMock.createStrictMock(ITestInvocationListener.class); |
| |
| DeqpTestRunner deqpTest = new DeqpTestRunner(NAME, NAME, tests, instances); |
| deqpTest.setAbi(UnitTests.ABI); |
| deqpTest.setDevice(mockDevice); |
| deqpTest.setBuildHelper(new StubCtsBuildHelper()); |
| |
| int version = 3 << 16; |
| EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) |
| .andReturn(Integer.toString(version)).atLeastOnce(); |
| EasyMock.expect(mockDevice.executeShellCommand("pm list features")).andReturn(ALL_FEATURES) |
| .anyTimes(); |
| |
| EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))). |
| andReturn("").once(); |
| |
| EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(), |
| EasyMock.eq(true), |
| EasyMock.eq(AbiUtils.createAbiFlag(UnitTests.ABI.getName())))).andReturn(null) |
| .once(); |
| |
| // query config A |
| expectRenderConfigQueryAndReturn(mockDevice, "--deqp-gl-config-name=rgba8888d24s8 " |
| + "--deqp-screen-rotation=unspecified " |
| + "--deqp-surface-type=window " |
| + "--deqp-gl-major-version=3 " |
| + "--deqp-gl-minor-version=0", "Yes"); |
| |
| // run config A |
| runInstrumentationLineAndAnswer(mockDevice, |
| "{dEQP-GLES3{instances{passall,failone,crashtwo}}}", |
| "--deqp-caselist-file=" + CASE_LIST_FILE_NAME |
| + " --deqp-gl-config-name=rgba8888d24s8 " |
| + "--deqp-screen-rotation=unspecified " |
| + "--deqp-surface-type=window " |
| + "--deqp-log-images=disable", instrumentationAnswerConfigA); |
| |
| // query for config B |
| expectRenderConfigQueryAndReturn(mockDevice, "--deqp-gl-config-name=rgba8888d24s8 " |
| + "--deqp-screen-rotation=90 " |
| + "--deqp-surface-type=window " |
| + "--deqp-gl-major-version=3 " |
| + "--deqp-gl-minor-version=0", "Yes"); |
| |
| // run for config B |
| runInstrumentationLineAndAnswer(mockDevice, |
| "{dEQP-GLES3{instances{passall,crashtwo,skipone}}}", |
| "--deqp-caselist-file=" + CASE_LIST_FILE_NAME |
| + " --deqp-gl-config-name=rgba8888d24s8 " |
| + "--deqp-screen-rotation=90 " |
| + "--deqp-surface-type=window " |
| + "--deqp-log-images=disable", instrumentationAnswerConfigB); |
| |
| // query for config C |
| expectRenderConfigQueryAndReturn(mockDevice, "--deqp-gl-config-name=rgba8888d24s8 " |
| + "--deqp-screen-rotation=180 " |
| + "--deqp-surface-type=window " |
| + "--deqp-gl-major-version=3 " |
| + "--deqp-gl-minor-version=0", "Yes"); |
| |
| // run for config C |
| runInstrumentationLineAndAnswer(mockDevice, |
| "{dEQP-GLES3{instances{failone,crashtwo}}}", |
| "--deqp-caselist-file=" + CASE_LIST_FILE_NAME |
| + " --deqp-gl-config-name=rgba8888d24s8 " |
| + "--deqp-screen-rotation=180 " |
| + "--deqp-surface-type=window " |
| + "--deqp-log-images=disable" , instrumentationAnswerConfigC); |
| |
| // query for unsupported config |
| expectRenderConfigQueryAndReturn(mockDevice, "--deqp-gl-config-name=rgb565d16s0 " |
| + "--deqp-screen-rotation=unspecified " |
| + "--deqp-surface-type=window " |
| + "--deqp-gl-major-version=3 " |
| + "--deqp-gl-minor-version=0", "No"); |
| |
| EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))) |
| .andReturn("").once(); |
| |
| mockListener.testRunStarted(ID, 4); |
| EasyMock.expectLastCall().once(); |
| |
| // pass all |
| mockListener.testStarted(EasyMock.eq(testIds[0])); |
| EasyMock.expectLastCall().once(); |
| |
| mockListener.testEnded(EasyMock.eq(testIds[0]), EasyMock.<Map<String, String>>notNull()); |
| EasyMock.expectLastCall().once(); |
| |
| // fail one |
| mockListener.testStarted(EasyMock.eq(testIds[1])); |
| EasyMock.expectLastCall().once(); |
| |
| mockListener.testFailed(testIds[1], |
| "=== with config {glformat=rgba8888d24s8,rotation=180,surfacetype=window} ===\n" |
| + "Fail: Fail"); |
| EasyMock.expectLastCall().once(); |
| |
| mockListener.testEnded(EasyMock.eq(testIds[1]), EasyMock.<Map<String, String>>notNull()); |
| EasyMock.expectLastCall().once(); |
| |
| // crash two |
| mockListener.testStarted(EasyMock.eq(testIds[2])); |
| EasyMock.expectLastCall().once(); |
| |
| mockListener.testFailed(testIds[2], |
| "=== with config {glformat=rgba8888d24s8,rotation=unspecified,surfacetype=window} ===\n" |
| + "Crash: Incomplete test log\n" |
| + "=== with config {glformat=rgba8888d24s8,rotation=90,surfacetype=window} ===\n" |
| + "Terminated: Magic"); |
| EasyMock.expectLastCall().once(); |
| |
| mockListener.testEnded(EasyMock.eq(testIds[2]), EasyMock.<Map<String, String>>notNull()); |
| EasyMock.expectLastCall().once(); |
| |
| // skip one |
| mockListener.testStarted(EasyMock.eq(testIds[3])); |
| EasyMock.expectLastCall().once(); |
| |
| mockListener.testEnded(EasyMock.eq(testIds[3]), EasyMock.<Map<String, String>>notNull()); |
| EasyMock.expectLastCall().once(); |
| |
| mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull()); |
| EasyMock.expectLastCall().once(); |
| |
| EasyMock.replay(mockDevice); |
| EasyMock.replay(mockListener); |
| deqpTest.run(mockListener); |
| EasyMock.verify(mockListener); |
| EasyMock.verify(mockDevice); |
| } |
| |
| /** |
| * Test dEQP with runner if device is lost during one of multiple instances. |
| */ |
| public void testRun_multipleInstancesLossOfDeviceMidInstance() throws Exception { |
| final String instrumentationAnswerFine = |
| "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.loss.instance\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_CODE: 0\r\n"; |
| final String instrumentationAnswerCrash = |
| "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" |
| + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.loss.instance\r\n" |
| + "INSTRUMENTATION_STATUS_CODE: 0\r\n"; // early <EOF> |
| |
| final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.loss", "instance"); |
| final String testPath = "dEQP-GLES3.loss.instance"; |
| |
| Map<String,String> supportedConfigA = new HashMap<>(); |
| supportedConfigA.put("glconfig", "rgba8888d24s8"); |
| supportedConfigA.put("rotation", "unspecified"); |
| supportedConfigA.put("surfacetype", "window"); |
| |
| Map<String,String> supportedConfigB = new HashMap<>(); |
| supportedConfigB.put("glconfig", "rgba8888d24s8"); |
| supportedConfigB.put("rotation", "90"); |
| supportedConfigB.put("surfacetype", "window"); |
| |
| Map<String,String> supportedConfigC = new HashMap<>(); |
| supportedConfigC.put("glconfig", "rgba8888d24s8"); |
| supportedConfigC.put("rotation", "180"); |
| supportedConfigC.put("surfacetype", "window"); |
| |
| Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>(); |
| tests.add(testId); |
| |
| Map<TestIdentifier, List<Map<String, String>>> instance = new HashMap<>(); |
| instance.put(testId, new ArrayList<Map<String,String>>()); |
| instance.get(testId).add(supportedConfigA); |
| instance.get(testId).add(supportedConfigB); |
| instance.get(testId).add(supportedConfigC); |
| |
| ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); |
| ITestInvocationListener mockListener |
| = EasyMock.createStrictMock(ITestInvocationListener.class); |
| |
| DeqpTestRunner deqpTest = new DeqpTestRunner(NAME, NAME, tests, instance); |
| deqpTest.setAbi(UnitTests.ABI); |
| deqpTest.setDevice(mockDevice); |
| deqpTest.setBuildHelper(new StubCtsBuildHelper()); |
| |
| int version = 3 << 16; |
| EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) |
| .andReturn(Integer.toString(version)).atLeastOnce(); |
| EasyMock.expect(mockDevice.executeShellCommand("pm list features")).andReturn(ALL_FEATURES) |
| .anyTimes(); |
| |
| EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))). |
| andReturn("").once(); |
| |
| EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(), |
| EasyMock.eq(true), |
| EasyMock.eq(AbiUtils.createAbiFlag(UnitTests.ABI.getName())))).andReturn(null) |
| .once(); |
| |
| // query config A |
| expectRenderConfigQueryAndReturn(mockDevice, "--deqp-gl-config-name=rgba8888d24s8 " |
| + "--deqp-screen-rotation=unspecified " |
| + "--deqp-surface-type=window " |
| + "--deqp-gl-major-version=3 " |
| + "--deqp-gl-minor-version=0", "Yes"); |
| |
| // run config A |
| runInstrumentationLineAndAnswer(mockDevice, |
| "{dEQP-GLES3{loss{instance}}}", |
| "--deqp-caselist-file=" + CASE_LIST_FILE_NAME |
| + " --deqp-gl-config-name=rgba8888d24s8 " |
| + "--deqp-screen-rotation=unspecified " |
| + "--deqp-surface-type=window " |
| + "--deqp-log-images=disable", instrumentationAnswerFine); |
| |
| // query config B |
| expectRenderConfigQueryAndReturn(mockDevice, "--deqp-gl-config-name=rgba8888d24s8 " |
| + "--deqp-screen-rotation=90 " |
| + "--deqp-surface-type=window " |
| + "--deqp-gl-major-version=3 " |
| + "--deqp-gl-minor-version=0", "Yes"); |
| |
| // run config B |
| EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("rm " + CASE_LIST_FILE_NAME))) |
| .andReturn("").once(); |
| |
| EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("rm " + LOG_FILE_NAME))) |
| .andReturn("").once(); |
| |
| EasyMock.expect(mockDevice.pushString("{dEQP-GLES3{loss{instance}}}\n", CASE_LIST_FILE_NAME)) |
| .andReturn(true).once(); |
| |
| String command = String.format( |
| "am instrument %s -w -e deqpLogFileName \"%s\" -e deqpCmdLine \"" |
| + "--deqp-caselist-file=%s" |
| + " --deqp-gl-config-name=rgba8888d24s8 " |
| + "--deqp-screen-rotation=90 " |
| + "--deqp-surface-type=window " |
| + "--deqp-log-images=disable\" " |
| + "-e deqpLogData \"%s\" %s", |
| AbiUtils.createAbiFlag(UnitTests.ABI.getName()), LOG_FILE_NAME, |
| CASE_LIST_FILE_NAME, false, INSTRUMENTATION_NAME); |
| |
| mockDevice.executeShellCommand(EasyMock.eq(command), |
| EasyMock.<IShellOutputReceiver>notNull()); |
| |
| EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() { |
| @Override |
| public Object answer() throws DeviceNotAvailableException { |
| IShellOutputReceiver receiver |
| = (IShellOutputReceiver)EasyMock.getCurrentArguments()[1]; |
| |
| receiver.addOutput(instrumentationAnswerCrash.getBytes(), 0, |
| instrumentationAnswerCrash.length()); |
| throw new DeviceNotAvailableException(); |
| } |
| }); |
| |
| mockListener.testRunStarted(ID, 1); |
| EasyMock.expectLastCall().once(); |
| |
| mockListener.testStarted(EasyMock.eq(testId)); |
| EasyMock.expectLastCall().once(); |
| |
| mockListener.testFailed(testId, |
| "=== with config {glformat=rgba8888d24s8,rotation=90,surfacetype=window} ===\n" |
| + "Crash: Device lost"); |
| EasyMock.expectLastCall().once(); |
| |
| mockListener.testEnded(EasyMock.eq(testId), EasyMock.<Map<String, String>>notNull()); |
| EasyMock.expectLastCall().once(); |
| |
| EasyMock.replay(mockDevice); |
| EasyMock.replay(mockListener); |
| |
| try { |
| deqpTest.run(mockListener); |
| fail("did not get DeviceNotAvailableException"); |
| } catch (DeviceNotAvailableException ex) { |
| // expected |
| } |
| |
| EasyMock.verify(mockListener); |
| EasyMock.verify(mockDevice); |
| } |
| |
| private void runInstrumentationLineAndAnswer(ITestDevice mockDevice, final String testTrie, |
| final String cmd, final String output) throws Exception { |
| EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("rm " + CASE_LIST_FILE_NAME))) |
| .andReturn("").once(); |
| |
| EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("rm " + LOG_FILE_NAME))) |
| .andReturn("").once(); |
| |
| EasyMock.expect(mockDevice.pushString(testTrie + "\n", CASE_LIST_FILE_NAME)) |
| .andReturn(true).once(); |
| |
| String command = String.format( |
| "am instrument %s -w -e deqpLogFileName \"%s\" -e deqpCmdLine \"%s\" " |
| + "-e deqpLogData \"%s\" %s", |
| AbiUtils.createAbiFlag(UnitTests.ABI.getName()), LOG_FILE_NAME, |
| cmd, false, INSTRUMENTATION_NAME); |
| |
| mockDevice.executeShellCommand(EasyMock.eq(command), |
| EasyMock.<IShellOutputReceiver>notNull()); |
| |
| EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() { |
| @Override |
| public Object answer() { |
| IShellOutputReceiver receiver |
| = (IShellOutputReceiver)EasyMock.getCurrentArguments()[1]; |
| |
| receiver.addOutput(output.getBytes(), 0, output.length()); |
| receiver.flush(); |
| |
| return null; |
| } |
| }); |
| } |
| } |
