/*
 * Copyright (C) 2017 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.android.compatibility.common.tradefed.presubmit;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;

import com.android.compatibility.SuiteInfo;
import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper;
import com.android.compatibility.common.tradefed.result.ResultReporter;
import com.android.compatibility.common.tradefed.testtype.CompatibilityTest;
import com.android.compatibility.common.tradefed.testtype.IModuleDef;
import com.android.compatibility.common.util.IInvocationResult;
import com.android.compatibility.common.util.TestStatus;
import com.android.ddmlib.testrunner.TestIdentifier;
import com.android.tradefed.build.IBuildInfo;
import com.android.tradefed.config.ConfigurationFactory;
import com.android.tradefed.config.OptionSetter;
import com.android.tradefed.device.DeviceNotAvailableException;
import com.android.tradefed.device.ITestDevice;
import com.android.tradefed.invoker.IInvocationContext;
import com.android.tradefed.invoker.InvocationContext;
import com.android.tradefed.invoker.ShardListener;
import com.android.tradefed.result.ITestInvocationListener;
import com.android.tradefed.result.ResultForwarder;
import com.android.tradefed.testtype.IBuildReceiver;
import com.android.tradefed.testtype.IDeviceTest;
import com.android.tradefed.testtype.IInvocationContextReceiver;
import com.android.tradefed.testtype.IRemoteTest;
import com.android.tradefed.util.AbiUtils;
import com.android.tradefed.util.FileUtil;

import org.easymock.EasyMock;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;

/**
 * Integration tests between {@link CompatibilityTest} and {@link ResultReporter} to ensure proper
 * flow of run and results.
 */
@RunWith(JUnit4.class)
public class IntegrationTest {

    private static final String CONFIG =
            "<configuration description=\"Auto Generated File\">\n" +
            "<option name=\"config-descriptor:metadata\" key=\"component\" value=\"%s\" />\n" +
            "<test class=\"com.android.compatibility.common.tradefed.testtype.%s\">\n" +
            "    <option name=\"report-test\" value=\"%s\" />\n" +
            "    <option name=\"run-complete\" value=\"%s\" />\n" +
            "    <option name=\"test-fail\" value=\"%s\" />\n" +
            "    <option name=\"internal-retry\" value=\"%s\" />\n" +
            "</test>\n" +
            "</configuration>";
    private static final String FILENAME = "%s.config";
    private static final String TEST_STUB = "TestStub"; // Test stub
    private static final String SIMPLE_TEST_STUB = "SimpleTestStub"; // Simple test stub
    private static final String TEST_STUB_SHARDABLE = "TestStubShardable";
    private static final String COMMAND_LINE = "run cts";

    private CompatibilityTest mTest;
    private ResultReporter mReporter;
    private ITestDevice mMockDevice;
    private IBuildInfo mMockBuildInfo;
    private IInvocationContext mContext;

    private File mRootDir;
    private File mAndroidFolder;
    private File mTestDir;
    private Map<String, String> mAttributes;

    @Before
    public void setUp() throws IOException {
        mAttributes = new HashMap<>();
        mTest = new CompatibilityTest() {
            @Override
            protected Set<String> getAbisForBuildTargetArch() {
                Set<String> abis = new HashSet<>();
                abis.add("arm64-v8a");
                abis.add("armeabi-v7a");
                return abis;
            }
        };
        mReporter = new ResultReporter();
        mMockDevice = EasyMock.createMock(ITestDevice.class);
        mMockBuildInfo = EasyMock.createMock(IBuildInfo.class);
        mTest.setBuild(mMockBuildInfo);
        mTest.setDevice(mMockDevice);
        mRootDir = FileUtil.createTempDir("fake-cts-root-dir");
        mAndroidFolder = FileUtil.createTempDir("android-", mRootDir);
        mTestDir = new File(mAndroidFolder, "testcases");
        mTestDir.mkdirs();
        String suiteName = mAndroidFolder.getName().split("-")[1];
        // Create fake build attributes
        mAttributes.put(CompatibilityBuildHelper.ROOT_DIR, mRootDir.getAbsolutePath());
        mAttributes.put(CompatibilityBuildHelper.SUITE_NAME, suiteName);
        mAttributes.put(CompatibilityBuildHelper.START_TIME_MS, "0");
        mAttributes.put(CompatibilityBuildHelper.SUITE_VERSION, "10");
        mAttributes.put(CompatibilityBuildHelper.SUITE_PLAN, "cts");
        mAttributes.put(CompatibilityBuildHelper.SUITE_BUILD, "good-build");
        mAttributes.put(CompatibilityBuildHelper.COMMAND_LINE_ARGS, COMMAND_LINE);

        // these attributes seems necessary for re-run, not for run
        mAttributes.put("cts:build_fingerprint", "fingerprint");
        mAttributes.put("cts:build_product", "product");
        mAttributes.put("cts:build_id", "bid");

        EasyMock.expect(mMockBuildInfo.getBuildAttributes()).andStubReturn(mAttributes);

        EasyMock.expect(mMockDevice.getSerialNumber()).andStubReturn("SERIAL");
        EasyMock.expect(mMockBuildInfo.getDeviceSerial()).andStubReturn("SERIAL");

        mContext = new InvocationContext();
        mContext.addAllocatedDevice("default", mMockDevice);
        mContext.addDeviceBuildInfo("default", mMockBuildInfo);
        mTest.setInvocationContext(mContext);
    }

    @After
    public void tearDown() {
        FileUtil.recursiveDelete(mRootDir);
    }

    /**
     * Create a CTS configuration with a fake tests to exercise all cases.
     *
     * @param testsDir The testcases/ dir where to put the module
     * @param name the name of the module.
     * @param moduleClass the fake test class to use.
     * @param reportTest True if the test report some tests
     * @param runComplete True if the test run is complete
     * @param doesOneTestFail True if one of the test is going to fail
     * @param internalRetry True if the test will retry the module itself once
     */
    private void createConfig(File testsDir, String name, String moduleClass, boolean reportTest,
            boolean runComplete, boolean doesOneTestFail, boolean internalRetry)
                    throws IOException {
        createConfig(testsDir, name, moduleClass, reportTest, runComplete, doesOneTestFail,
                internalRetry, "foo");

    }

    /**
     * Create a CTS configuration with a fake tests to exercise all cases.
     *
     * @param testsDir The testcases/ dir where to put the module
     * @param name the name of the module.
     * @param moduleClass the fake test class to use.
     * @param reportTest True if the test report some tests
     * @param runComplete True if the test run is complete
     * @param doesOneTestFail True if one of the test is going to fail
     * @param internalRetry True if the test will retry the module itself once
     * @param component the platform component name that the module can be categorized under
     */
    private void createConfig(File testsDir, String name, String moduleClass, boolean reportTest,
            boolean runComplete, boolean doesOneTestFail, boolean internalRetry, String component)
                    throws IOException {
        File config = new File(testsDir, String.format(FILENAME, name));
        FileUtil.deleteFile(config);
        if (!config.createNewFile()) {
            throw new IOException(String.format("Failed to create '%s'", config.getAbsolutePath()));
        }

        FileUtil.writeToFile(String.format(CONFIG, component, moduleClass, reportTest, runComplete,
                doesOneTestFail, internalRetry), config);
    }

    /**
     * Simple tests running in one module that should be marked complete.
     */
    @Test
    public void testSingleModuleRun() throws Exception {
        final String moduleName = "module_run";
        final String mAbi = "arm64-v8a";
        createConfig(mTestDir, moduleName, TEST_STUB, true, true, true, false);
        EasyMock.expect(mMockDevice.getProperty("ro.product.cpu.abilist")).andReturn(mAbi);

        mMockBuildInfo.addBuildAttribute(EasyMock.eq(CompatibilityBuildHelper.MODULE_IDS),
                EasyMock.eq(AbiUtils.createId(mAbi, moduleName)));
        EasyMock.expectLastCall();

        EasyMock.replay(mMockDevice, mMockBuildInfo);
        mReporter.invocationStarted(mContext);
        mTest.run(mReporter);
        mReporter.invocationEnded(500);
        EasyMock.verify(mMockDevice, mMockBuildInfo);
        IInvocationResult result = mReporter.getResult();
        assertEquals(2, result.countResults(TestStatus.PASS));
        assertEquals(1, result.countResults(TestStatus.FAIL));
        assertEquals(1, result.getModules().size());
        assertEquals(1, result.getModuleCompleteCount());
    }

    /**
     * Verify that result reporters test run ended callback can receive component name as configured
     * in module config metadata field.
     */
    @Test
    public void testSingleModuleRun_checkMetadata() throws Exception {
        final String moduleName = "AwsomeModule";
        final String mAbi = "arm64-v8a";
        final String component = "CriticalComponent";
        final List<String> receivedComponentsTestEnded = new ArrayList<>();
        final List<String> receivedModuleNameTestEnded = new ArrayList<>();
        final List<String> receivedAbiTestEnded = new ArrayList<>();
        final List<String> receivedComponentsTestRunEnded = new ArrayList<>();
        final List<String> receivedModuleNameTestRunEnded = new ArrayList<>();
        final List<String> receivedAbiTestRunEnded = new ArrayList<>();
        createConfig(mTestDir, moduleName, SIMPLE_TEST_STUB, true, true, true, false, component);
        EasyMock.expect(mMockDevice.getProperty("ro.product.cpu.abilist")).andReturn(mAbi);

        mMockBuildInfo.addBuildAttribute(EasyMock.eq(CompatibilityBuildHelper.MODULE_IDS),
                EasyMock.eq(AbiUtils.createId(mAbi, moduleName)));
        EasyMock.expectLastCall();

        EasyMock.replay(mMockDevice, mMockBuildInfo);
        ITestInvocationListener myListener = new ITestInvocationListener() {
            private IInvocationContext myContext;
            @Override
            public void invocationStarted(IInvocationContext context) {
                myContext = context;
            }
            @Override
            public void testRunEnded(long elapsedTimeMillis, Map<String, String> runMetrics) {
                receivedComponentsTestRunEnded.addAll(myContext.getModuleInvocationContext()
                        .getConfigurationDescriptor().getMetaData("component"));
                receivedModuleNameTestRunEnded.addAll(myContext.getModuleInvocationContext()
                        .getAttributes().get(IModuleDef.MODULE_NAME));
                receivedAbiTestRunEnded.addAll(myContext.getModuleInvocationContext()
                        .getAttributes().get(IModuleDef.MODULE_ABI));
            }
            @Override
            public void testEnded(TestIdentifier test, long endTime,
                    Map<String, String> testMetrics) {
                receivedComponentsTestEnded.addAll(myContext.getModuleInvocationContext()
                        .getConfigurationDescriptor().getMetaData("component"));
                receivedModuleNameTestEnded.addAll(myContext.getModuleInvocationContext()
                        .getAttributes().get(IModuleDef.MODULE_NAME));
                receivedAbiTestEnded.addAll(myContext.getModuleInvocationContext()
                        .getAttributes().get(IModuleDef.MODULE_ABI));
            }
        };
        myListener.invocationStarted(mContext);
        mTest.run(myListener);
        myListener.invocationEnded(500);
        EasyMock.verify(mMockDevice, mMockBuildInfo);
        // verify metadata was retrieved during testRunEnded callbacks
        assertEquals("[testRunEnded] wrong number of metadata collected",
                1, receivedComponentsTestRunEnded.size());
        assertEquals("[testRunEnded] wrong component metadata field received",
                component, receivedComponentsTestRunEnded.get(0));
        assertEquals("[testRunEnded] wrong number of module name collected",
                1, receivedModuleNameTestRunEnded.size());
        assertEquals(moduleName, receivedModuleNameTestRunEnded.get(0));
        assertEquals("[testEnded] wrong number of module abi collected",
                1, receivedAbiTestRunEnded.size());
        assertEquals(mAbi, receivedAbiTestRunEnded.get(0));
        // verify metadata was retrieved during testEnded callbacks
        assertEquals("[testEnded] wrong number of metadata collected",
                1, receivedComponentsTestEnded.size());
        assertEquals("[testEnded] wrong component metadata field received",
                component, receivedComponentsTestEnded.get(0));
        assertEquals("[testEnded] wrong number of module name collected",
                1, receivedModuleNameTestEnded.size());
        assertEquals(moduleName, receivedModuleNameTestEnded.get(0));
        assertEquals("[testEnded] wrong number of module abi collected",
                1, receivedAbiTestEnded.size());
        assertEquals(mAbi, receivedAbiTestEnded.get(0));
    }

    /**
     * Simple tests running in one module that run some tests but not all of them.
     */
    @Test
    public void testSingleModuleRun_incomplete() throws Exception {
        final String moduleName = "module_run_incomplete";
        final String mAbi = "arm64-v8a";
        createConfig(mTestDir, moduleName, TEST_STUB, true, false, true, false);
        EasyMock.expect(mMockDevice.getProperty("ro.product.cpu.abilist")).andReturn(mAbi);

        mMockBuildInfo.addBuildAttribute(EasyMock.eq(CompatibilityBuildHelper.MODULE_IDS),
                EasyMock.eq(AbiUtils.createId(mAbi, moduleName)));
        EasyMock.expectLastCall();

        EasyMock.replay(mMockDevice, mMockBuildInfo);
        mReporter.invocationStarted(mContext);
        mTest.run(mReporter);
        mReporter.invocationEnded(500);
        EasyMock.verify(mMockDevice, mMockBuildInfo);
        IInvocationResult result = mReporter.getResult();
        assertEquals(1, result.countResults(TestStatus.PASS));
        assertEquals(1, result.countResults(TestStatus.FAIL));
        // Module should not be seen as complete.
        assertEquals(1, result.getModules().size());
        assertEquals(0, result.getModuleCompleteCount());
    }

    /**
     * Simple tests running in one module that should be marked complete since it runs all its
     * tests after an internal retry (like InstrumentationTest).
     * FIXME: Fix the expectation of this test
     */
    @Test
    public void testSingleModuleRun_completeAfterInternalRetry() throws Exception {
        final String moduleName = "module_completeAfterRetry";
        final String mAbi = "arm64-v8a";
        createConfig(mTestDir, moduleName, TEST_STUB, true, true, true, true);
        EasyMock.expect(mMockDevice.getProperty("ro.product.cpu.abilist")).andReturn(mAbi);

        mMockBuildInfo.addBuildAttribute(EasyMock.eq(CompatibilityBuildHelper.MODULE_IDS),
                EasyMock.eq(AbiUtils.createId(mAbi, moduleName)));
        EasyMock.expectLastCall();

        EasyMock.replay(mMockDevice, mMockBuildInfo);
        mReporter.invocationStarted(mContext);
        mTest.run(mReporter);
        mReporter.invocationEnded(500);
        EasyMock.verify(mMockDevice, mMockBuildInfo);
        IInvocationResult result = mReporter.getResult();
        // FIXME: All tests should be marked as executed and not aggregating the count.
        assertEquals(2, result.countResults(TestStatus.PASS));
        assertEquals(1, result.countResults(TestStatus.FAIL));
        // FIXME: Module should be complete since all its test have run.
        assertEquals(1, result.getModules().size());
        assertEquals(0, result.getModuleCompleteCount());
    }

    /**
     * Simple tests running in one module that run some tests but not all of them, then we
     * attempt to run it again and they still didn't run.
     * FIXME: This test expectation needs to be fixed
     */
    @Test
    public void testSingleModuleRun_incomplete_rerun_incomplete() throws Exception {
        final String moduleName = "module_incomplete_rerun";
        final String mAbi = "arm64-v8a";
        createConfig(mTestDir, moduleName, TEST_STUB, true, false, true, false);
        EasyMock.expect(mMockDevice.getProperty("ro.product.cpu.abilist")).andReturn(mAbi);

        mMockBuildInfo.addBuildAttribute(EasyMock.eq(CompatibilityBuildHelper.MODULE_IDS),
                EasyMock.eq(AbiUtils.createId(mAbi, moduleName)));
        EasyMock.expectLastCall();

        // extra calls for retry
        EasyMock.expect(mMockDevice.getProperty("ro.build.fingerprint")).andReturn("fingerprint");
        EasyMock.expect(mMockDevice.getProperty("ro.product.cpu.abilist")).andReturn(mAbi);
        mMockBuildInfo.addBuildAttribute(EasyMock.eq(CompatibilityBuildHelper.MODULE_IDS),
                EasyMock.eq(AbiUtils.createId(mAbi, moduleName)));
        EasyMock.expectLastCall();
        mMockBuildInfo.addBuildAttribute(EasyMock.eq("retry_command_line_args"),
                EasyMock.eq(COMMAND_LINE));
        EasyMock.expectLastCall();

        EasyMock.replay(mMockDevice, mMockBuildInfo);
        mReporter.invocationStarted(mContext);
        mTest.run(mReporter);
        mReporter.invocationEnded(500);

        IInvocationResult result = mReporter.getResult();
        assertEquals(1, result.countResults(TestStatus.PASS));
        assertEquals(1, result.countResults(TestStatus.FAIL));
        // Module should not be seen as complete.
        assertEquals(0, result.getModuleCompleteCount());

        // We re-run it
        mReporter = new ResultReporter();
        mTest = new CompatibilityTest() {
            @Override
            protected Set<String> getAbisForBuildTargetArch() {
                Set<String> abis = new HashSet<>();
                abis.add("arm64-v8a");
                return abis;
            }
        };
        mTest.setDevice(mMockDevice);
        mTest.setBuild(mMockBuildInfo);
        mTest.setInvocationContext(mContext);
        OptionSetter setter = new OptionSetter(mTest, mReporter);
        setter.setOptionValue("retry", "0");

        mReporter.invocationStarted(mContext);
        mTest.run(mReporter);
        mReporter.invocationEnded(500);
        EasyMock.verify(mMockDevice, mMockBuildInfo);

        // Check retry results
        result = mReporter.getResult();
        // FIXME: We should only have 1 not_executed in the retry too. They should not aggregate
        // from one run to another.
        assertEquals(1, result.countResults(TestStatus.PASS));
        assertEquals(1, result.countResults(TestStatus.FAIL));
        // Module should not be seen as complete.
        assertEquals(1, result.getModules().size());
        assertEquals(0, result.getModuleCompleteCount());
    }

    /**
     * Simple tests running in one module that run some tests but not all of them, we then attempt
     * to retry run them and this time the not_executed is executed.
     */
    @Test
    public void testSingleModuleRun_incomplete_rerun_complete() throws Exception {
        final String moduleName = "module_incom_rerun_complete";
        final String mAbi = "arm64-v8a";
        createConfig(mTestDir, moduleName, TEST_STUB, true, false, true, false);
        EasyMock.expect(mMockDevice.getProperty("ro.product.cpu.abilist")).andReturn(mAbi);

        mMockBuildInfo.addBuildAttribute(EasyMock.eq(CompatibilityBuildHelper.MODULE_IDS),
                EasyMock.eq(AbiUtils.createId(mAbi, moduleName)));
        EasyMock.expectLastCall();

        // extra calls for retry
        EasyMock.expect(mMockDevice.getProperty("ro.build.fingerprint")).andReturn("fingerprint");
        EasyMock.expect(mMockDevice.getProperty("ro.product.cpu.abilist")).andReturn(mAbi);
        mMockBuildInfo.addBuildAttribute(EasyMock.eq(CompatibilityBuildHelper.MODULE_IDS),
                EasyMock.eq(AbiUtils.createId(mAbi, moduleName)));
        EasyMock.expectLastCall();
        mMockBuildInfo.addBuildAttribute(EasyMock.eq("retry_command_line_args"),
                EasyMock.eq(COMMAND_LINE));
        EasyMock.expectLastCall();

        EasyMock.replay(mMockDevice, mMockBuildInfo);
        mReporter.invocationStarted(mContext);
        mTest.run(mReporter);
        mReporter.invocationEnded(500);

        IInvocationResult result = mReporter.getResult();
        assertEquals(1, result.countResults(TestStatus.PASS));
        assertEquals(1, result.countResults(TestStatus.FAIL));
        // Module should not be seen as complete.
        assertEquals(0, result.getModuleCompleteCount());

        // We replace the config by one that runs all tests without failures.
        createConfig(mTestDir, moduleName, TEST_STUB, true, true, false, false);
        // Usually configs do not change during the same session so we clear the map to have
        // the new version of the config.
        ((ConfigurationFactory)ConfigurationFactory.getInstance()).clearMapConfig();

        // We re-run it
        mReporter = new ResultReporter();
        mTest = new CompatibilityTest() {
            @Override
            protected Set<String> getAbisForBuildTargetArch() {
                Set<String> abis = new HashSet<>();
                abis.add("arm64-v8a");
                return abis;
            }
        };
        mTest.setDevice(mMockDevice);
        mTest.setBuild(mMockBuildInfo);
        mTest.setInvocationContext(mContext);
        OptionSetter setter = new OptionSetter(mTest, mReporter);
        setter.setOptionValue("retry", "0");

        mReporter.invocationStarted(mContext);
        mTest.run(mReporter);
        mReporter.invocationEnded(500);
        EasyMock.verify(mMockDevice, mMockBuildInfo);

        // Check retry results
        result = mReporter.getResult();
        assertEquals(3, result.countResults(TestStatus.PASS));
        assertEquals(0, result.countResults(TestStatus.FAIL));
        // Module should be marked as complete after retry.
        assertEquals(1, result.getModules().size());
        assertEquals(1, result.getModuleCompleteCount());
    }

    // ***** Case for sharding interaction *****

    /**
     * Helper to create a shard listener with the original reporter as master.
     */
    private ITestInvocationListener getShardListener(ResultReporter masterReporter) {
        List<ITestInvocationListener> shardListeners = new ArrayList<ITestInvocationListener>();
        ShardListener origConfigListener = new ShardListener(masterReporter);
        ResultReporter reporterClone = (ResultReporter) masterReporter.clone();
        shardListeners.add(reporterClone);
        shardListeners.add(origConfigListener);
        ResultForwarder shard = new ResultForwarder(shardListeners);
        return shard;
    }

    /**
     * Helper Thread to run the IShardableTest.
     */
    private class ShardThread extends Thread {
        private IRemoteTest mShardTest;
        private ResultReporter mMasterReporter;
        private IBuildInfo mBuild;
        private ITestDevice mDevice;
        private IInvocationContext mShardContext;

        public ShardThread(IRemoteTest test, ResultReporter masterReporter, IBuildInfo build,
                ITestDevice device, IInvocationContext context) {
            mShardTest = test;
            mMasterReporter = masterReporter;
            mBuild = build;
            mDevice = device;
            mShardContext = context;
        }

        @Override
        public void run() {
            ITestInvocationListener listener = getShardListener(mMasterReporter);
            ((IBuildReceiver)mShardTest).setBuild(mBuild);
            ((IDeviceTest)mShardTest).setDevice(mDevice);
            ((IInvocationContextReceiver)mShardTest).setInvocationContext(mContext);
            listener.invocationStarted(mShardContext);
            try {
                mShardTest.run(listener);
            } catch (DeviceNotAvailableException e) {
                throw new RuntimeException(e);
            } finally {
                listener.invocationEnded(500);
            }
        }
    }

    /**
     * Simple tests running in one module that should be marked complete when each shard run a test
     * from the module. Each Module is going to run 1 pass 1 fail. 2 modules and 2 shards.
     * Using the {@link CompatibilityTest#split()}.
     */
    @Test
    public void testSingleModuleRun_sharded() throws Exception {
        final String moduleName = "module_sharded";
        Set<String> abis = AbiUtils.getAbisForArch(SuiteInfo.TARGET_ARCH);
        Iterator<String> ite = abis.iterator();
        final String abi1 = ite.next();
        final String abi2 = ite.next();
        createConfig(mTestDir, moduleName, TEST_STUB_SHARDABLE, true, true, true, false);
        EasyMock.expect(mMockDevice.getProperty("ro.product.cpu.abilist")).andReturn(
                String.format("%s,%s", abi1, abi2));
        mMockBuildInfo.addBuildAttribute(EasyMock.eq(CompatibilityBuildHelper.MODULE_IDS),
                EasyMock.anyObject());
        EasyMock.expectLastCall();

        EasyMock.replay(mMockDevice, mMockBuildInfo);

        OptionSetter setter = new OptionSetter(mTest);
        setter.setOptionValue("shards", "2");
        List<IRemoteTest> tests = (List<IRemoteTest>) mTest.split();
        // We expect 2 shards
        assertEquals(2, tests.size());

        List<ShardThread> threads = new ArrayList<>();
        // Run all shards
        for (IRemoteTest test : tests) {
            ShardThread st = new ShardThread(test, mReporter, mMockBuildInfo, mMockDevice,
                    mContext);
            threads.add(st);
            st.start();
        }
        for (ShardThread thread : threads) {
            thread.join(5000);
        }
        // Allow some time for ResultReport to finalize the results coming from the threads.
        boolean finalized = mReporter.waitForFinalized(2, TimeUnit.MINUTES);
        assertTrue(finalized);
        EasyMock.verify(mMockDevice, mMockBuildInfo);
        // Check aggregated results to make sure it's consistent.
        IInvocationResult result = mReporter.getResult();
        assertEquals(4, result.countResults(TestStatus.PASS));
        assertEquals(4, result.countResults(TestStatus.FAIL));
        assertEquals(2, result.getModules().size());
        assertEquals(2, result.getModuleCompleteCount());
    }

    /**
     * Simple tests running in one module that should be marked incomplete when shards do not
     * complete. Each shard is going to run 1 pass 1 fail 1 not_executed.
     * Using the {@link CompatibilityTest#split()}.
     */
    @Test
    public void testSingleModuleRun_sharded_incomplete() throws Exception {
        final String moduleName = "module_sharded_incomplete";
        Set<String> abis = AbiUtils.getAbisForArch(SuiteInfo.TARGET_ARCH);
        Iterator<String> ite = abis.iterator();
        final String abi1 = ite.next();
        final String abi2 = ite.next();
        createConfig(mTestDir, moduleName, TEST_STUB_SHARDABLE, true, false, true, false);
        EasyMock.expect(mMockDevice.getProperty("ro.product.cpu.abilist")).andReturn(
                String.format("%s,%s", abi1, abi2));
        mMockBuildInfo.addBuildAttribute(EasyMock.eq(CompatibilityBuildHelper.MODULE_IDS),
                EasyMock.anyObject());
        EasyMock.expectLastCall();

        EasyMock.replay(mMockDevice, mMockBuildInfo);
        OptionSetter setter = new OptionSetter(mTest);
        setter.setOptionValue("shards", "2");
        List<IRemoteTest> tests = (List<IRemoteTest>) mTest.split();
        // We expect 2 shards
        assertEquals(2, tests.size());

        List<ShardThread> threads = new ArrayList<>();
        // Run all shards
        for (IRemoteTest test : tests) {
            ShardThread st = new ShardThread(test, mReporter, mMockBuildInfo, mMockDevice,
                    mContext);
            threads.add(st);
            st.start();
        }
        for (ShardThread thread : threads) {
            thread.join(5000);
        }
        // Allow some time for ResultReport to finalize the results coming from the threads.
        boolean finalized = mReporter.waitForFinalized(2, TimeUnit.MINUTES);
        assertTrue(finalized);
        EasyMock.verify(mMockDevice, mMockBuildInfo);
        // Check aggregated results to make sure it's consistent.
        IInvocationResult result = mReporter.getResult();
        assertEquals(4, result.countResults(TestStatus.PASS));
        assertEquals(4, result.countResults(TestStatus.FAIL));
        assertEquals(2, result.getModules().size());
        assertEquals(0, result.getModuleCompleteCount());
    }

    /**
     * Simple tests running in one module that should be marked complete when each shard run a test
     * from the module.
     * We are going to run only one of the shard since IStrictShardable allows it.
     * Using the {@link CompatibilityTest#getTestShard(int, int)}.
     * FIXME: Fix expectation of this test.
     */
    @Test
    public void testSingleModuleRun_sharded_getTestShard() throws Exception {
        final String moduleName = "module_sharded_getTestShard";
        Set<String> abis = AbiUtils.getAbisForArch(SuiteInfo.TARGET_ARCH);
        Iterator<String> ite = abis.iterator();
        final String abi1 = ite.next();
        final String abi2 = ite.next();
        createConfig(mTestDir, moduleName, TEST_STUB_SHARDABLE, true, true, true, false);
        EasyMock.expect(mMockDevice.getProperty("ro.product.cpu.abilist")).andReturn(
                String.format("%s,%s", abi1, abi2));

        String expectedAdd = AbiUtils.createId(abi1, moduleName) + ","
                + AbiUtils.createId(abi2, moduleName);
        mMockBuildInfo.addBuildAttribute(EasyMock.eq(CompatibilityBuildHelper.MODULE_IDS),
                EasyMock.anyObject());
        EasyMock.expectLastCall();
        mAttributes.put(CompatibilityBuildHelper.MODULE_IDS, expectedAdd);

        EasyMock.replay(mMockDevice, mMockBuildInfo);

        List<IRemoteTest> tests = new ArrayList<>();
        tests.add(mTest.getTestShard(3, 0));
        // We are only running one of the shards since they should be independent.
        assertEquals(1, tests.size());

        ((IBuildReceiver)tests.get(0)).setBuild(mMockBuildInfo);
        ((IDeviceTest)tests.get(0)).setDevice(mMockDevice);
        ((IInvocationContextReceiver)tests.get(0)).setInvocationContext(mContext);
        mReporter.invocationStarted(mContext);
        try {
            tests.get(0).run(mReporter);
        } catch (DeviceNotAvailableException e) {
            throw new RuntimeException(e);
        } finally {
            mReporter.invocationEnded(500);
        }
        EasyMock.verify(mMockDevice, mMockBuildInfo);

        IInvocationResult result = mReporter.getResult();
        assertEquals(2, result.countResults(TestStatus.PASS));
        assertEquals(2, result.countResults(TestStatus.FAIL));
        // FIXME: Only one module should be expected since within the one shard requested to run
        // only one module existed.
        assertEquals(2, result.getModules().size());
        // FIXME: The module for the shard should be completed since all tests run.
        // TestRunHandler in this case create an expectation of 3 testRunStarted just because of
        // the number of shards.
        assertEquals(0, result.getModuleCompleteCount());
    }
}