common/host-side/tradefed/src/com/android/compatibility/common/tradefed/testtype/ModuleRepo.java - platform/cts - Git at Google

 /*
  * Copyright (C) 2015 The Android Open Source Project
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
  *      http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
 package com.android.compatibility.common.tradefed.testtype;

 import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper;
 import com.android.compatibility.common.tradefed.result.TestRunHandler;
 import com.android.compatibility.common.tradefed.util.LinearPartition;
 import com.android.compatibility.common.tradefed.util.UniqueModuleCountUtil;
 import com.android.compatibility.common.util.TestFilter;
 import com.android.ddmlib.Log.LogLevel;
 import com.android.tradefed.build.IBuildInfo;
 import com.android.tradefed.config.ConfigurationException;
 import com.android.tradefed.config.ConfigurationFactory;
 import com.android.tradefed.config.IConfiguration;
 import com.android.tradefed.config.IConfigurationFactory;
 import com.android.tradefed.log.LogUtil.CLog;
 import com.android.tradefed.testtype.IAbi;
 import com.android.tradefed.testtype.IBuildReceiver;
 import com.android.tradefed.testtype.IRemoteTest;
 import com.android.tradefed.testtype.IStrictShardableTest;
 import com.android.tradefed.testtype.ITestFileFilterReceiver;
 import com.android.tradefed.testtype.ITestFilterReceiver;
 import com.android.tradefed.util.AbiUtils;
 import com.android.tradefed.util.FileUtil;
 import com.android.tradefed.util.TimeUtil;

 import java.io.File;
 import java.io.FilenameFilter;
 import java.io.IOException;
 import java.io.PrintWriter;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;

 /**
  * Retrieves Compatibility test module definitions from the repository.
  */
 public class ModuleRepo implements IModuleRepo {

     private static final String CONFIG_EXT = ".config";
     private static final Map<String, Integer> ENDING_MODULES = new HashMap<>();
     static {
         ENDING_MODULES.put("CtsMonkeyTestCases", 1);
     }
     // Synchronization objects for Token Modules.
     private int mInitCount = 0;
     private Set<IModuleDef> mTokenModuleScheduled;
     private static Object lock = new Object();

     private int mTotalShards;
     private Integer mShardIndex;

     private Map<String, Set<String>> mDeviceTokens = new HashMap<>();
     private Map<String, Map<String, List<String>>> mTestArgs = new HashMap<>();
     private Map<String, Map<String, List<String>>> mModuleArgs = new HashMap<>();
     private boolean mIncludeAll;
     private Map<String, List<TestFilter>> mIncludeFilters = new HashMap<>();
     private Map<String, List<TestFilter>> mExcludeFilters = new HashMap<>();
     private IConfigurationFactory mConfigFactory = ConfigurationFactory.getInstance();

     private volatile boolean mInitialized = false;

     // Holds all the tests with tokens waiting to be run. Meaning the DUT must have a specific token.
     private List<IModuleDef> mTokenModules = new ArrayList<>();
     private List<IModuleDef> mNonTokenModules = new ArrayList<>();

     /**
      * {@inheritDoc}
      */
     @Override
     public int getNumberOfShards() {
         return mTotalShards;
     }

     /**
      * Returns the device tokens of this module repo. Exposed for testing.
      */
     protected Map<String, Set<String>> getDeviceTokens() {
         return mDeviceTokens;
     }

     /**
      * A {@link FilenameFilter} to find all modules in a directory who match the given pattern.
      */
     public static class NameFilter implements FilenameFilter {

         private String mPattern;

         public NameFilter(String pattern) {
             mPattern = pattern;
         }

         /**
          * {@inheritDoc}
          */
         @Override
         public boolean accept(File dir, String name) {
             return name.contains(mPattern) && name.endsWith(CONFIG_EXT);
         }
     }

     /**
      * {@inheritDoc}
      */
     @Override
     public List<IModuleDef> getNonTokenModules() {
         return mNonTokenModules;
     }

     /**
      * {@inheritDoc}
      */
     @Override
     public List<IModuleDef> getTokenModules() {
         return mTokenModules;
     }

     /**
      * {@inheritDoc}
      */
     @Override
     public String[] getModuleIds() {
         Set<String> moduleIdSet = new HashSet<>();
         for (IModuleDef moduleDef : mNonTokenModules) {
             moduleIdSet.add(moduleDef.getId());
         }
         for (IModuleDef moduleDef : mTokenModules) {
             moduleIdSet.add(moduleDef.getId());
         }
         return moduleIdSet.toArray(new String[moduleIdSet.size()]);
     }

     /**
      * {@inheritDoc}
      */
     @Override
     public boolean isInitialized() {
         return mInitialized;
     }

     /**
      * {@inheritDoc}
      */
     @Override
     public void initialize(int totalShards, Integer shardIndex, File testsDir, Set<IAbi> abis,
             List<String> deviceTokens, List<String> testArgs, List<String> moduleArgs,
             Set<String> includeFilters, Set<String> excludeFilters, IBuildInfo buildInfo) {
         CLog.d("Initializing ModuleRepo\nShards:%d\nTests Dir:%s\nABIs:%s\nDevice Tokens:%s\n" +
                 "Test Args:%s\nModule Args:%s\nIncludes:%s\nExcludes:%s",
                 totalShards, testsDir.getAbsolutePath(), abis, deviceTokens, testArgs, moduleArgs,
                 includeFilters, excludeFilters);
         mInitialized = true;
         mTotalShards = totalShards;
         mShardIndex = shardIndex;
         synchronized (lock) {
             if (mTokenModuleScheduled == null) {
                 mTokenModuleScheduled = new HashSet<>();
             }
         }

         for (String line : deviceTokens) {
             String[] parts = line.split(":");
             if (parts.length == 2) {
                 String key = parts[0];
                 String value = parts[1];
                 Set<String> list = mDeviceTokens.get(key);
                 if (list == null) {
                     list = new HashSet<>();
                     mDeviceTokens.put(key, list);
                 }
                 list.add(value);
             } else {
                 throw new IllegalArgumentException(
                         String.format("Could not parse device token: %s", line));
             }
         }
         putArgs(testArgs, mTestArgs);
         putArgs(moduleArgs, mModuleArgs);
         mIncludeAll = includeFilters.isEmpty();
         // Include all the inclusions
         addFilters(includeFilters, mIncludeFilters, abis);
         // Exclude all the exclusions
         addFilters(excludeFilters, mExcludeFilters, abis);

         File[] configFiles = testsDir.listFiles(new ConfigFilter());
         if (configFiles.length == 0) {
             throw new IllegalArgumentException(
                     String.format("No config files found in %s", testsDir.getAbsolutePath()));
         }
         Map<String, Integer> shardedTestCounts = new HashMap<>();
         for (File configFile : configFiles) {
             final String name = configFile.getName().replace(CONFIG_EXT, "");
             final String[] pathArg = new String[] { configFile.getAbsolutePath() };
             try {
                 // Invokes parser to process the test module config file
                 // Need to generate a different config for each ABI as we cannot guarantee the
                 // configs are idempotent. This however means we parse the same file multiple times
                 for (IAbi abi : abis) {
                     String id = AbiUtils.createId(abi.getName(), name);
                     if (!shouldRunModule(id)) {
                         // If the module should not run tests based on the state of filters,
                         // skip this name/abi combination.
                         continue;
                     }

                     IConfiguration config = mConfigFactory.createConfigurationFromArgs(pathArg);
                     Map<String, List<String>> args = new HashMap<>();
                     if (mModuleArgs.containsKey(name)) {
                         args.putAll(mModuleArgs.get(name));
                     }
                     if (mModuleArgs.containsKey(id)) {
                         args.putAll(mModuleArgs.get(id));
                     }
                     for (Entry<String, List<String>> entry : args.entrySet()) {
                         for (String entryValue : entry.getValue()) {
                             // Collection-type options can be injected with multiple values
                             String entryName = entry.getKey();
                             if (entryValue.contains(":")) {
                                 // entryValue is key-value pair
                                 String key = entryValue.split(":")[0];
                                 String value = entryValue.split(":")[1];
                                 config.injectOptionValue(entryName, key, value);
                             } else {
                                 // entryValue is just the argument value
                                 config.injectOptionValue(entryName, entryValue);
                             }
                         }
                     }

                     List<IRemoteTest> tests = config.getTests();
                     for (IRemoteTest test : tests) {
                         String className = test.getClass().getName();
                         Map<String, List<String>> testArgsMap = new HashMap<>();
                         if (mTestArgs.containsKey(className)) {
                             testArgsMap.putAll(mTestArgs.get(className));
                         }
                         for (Entry<String, List<String>> entry : testArgsMap.entrySet()) {
                             for (String entryValue : entry.getValue()) {
                                 String entryName = entry.getKey();
                                 if (entryValue.contains(":")) {
                                     // entryValue is key-value pair
                                     String key = entryValue.split(":")[0];
                                     String value = entryValue.split(":")[1];
                                     config.injectOptionValue(entryName, key, value);
                                 } else {
                                     // entryValue is just the argument value
                                     config.injectOptionValue(entryName, entryValue);
                                 }
                             }
                         }
                         addFiltersToTest(test, abi, name);
                     }
                     List<IRemoteTest> shardedTests = tests;
                     if (mTotalShards > 1) {
                          shardedTests = splitShardableTests(tests, buildInfo);
                     }
                     if (shardedTests.size() > 1) {
                         shardedTestCounts.put(id, shardedTests.size());
                     }
                     for (IRemoteTest test : shardedTests) {
                         addModuleDef(name, abi, test, pathArg);
                     }
                 }
             } catch (ConfigurationException e) {
                 throw new RuntimeException(String.format("error parsing config file: %s",
                         configFile.getName()), e);
             }
         }
         mExcludeFilters.clear();
         TestRunHandler.setTestRuns(new CompatibilityBuildHelper(buildInfo), shardedTestCounts);
     }

     private List<IRemoteTest> splitShardableTests(List<IRemoteTest> tests, IBuildInfo buildInfo) {
         ArrayList<IRemoteTest> shardedList = new ArrayList<>(tests.size());
         for (IRemoteTest test : tests) {
             if (test instanceof IBuildReceiver) {
                 ((IBuildReceiver)test).setBuild(buildInfo);
             }
             if (mShardIndex != null && test instanceof IStrictShardableTest) {
                 for (int i = 0; i < mTotalShards; i++) {
                     shardedList.add(((IStrictShardableTest)test).getTestShard(mTotalShards, i));
                 }
             } else {
                 shardedList.add(test);
             }
         }
         return shardedList;
     }

     private void addFilters(Set<String> stringFilters,
             Map<String, List<TestFilter>> filters, Set<IAbi> abis) {
         for (String filterString : stringFilters) {
             TestFilter filter = TestFilter.createFrom(filterString);
             String abi = filter.getAbi();
             if (abi == null) {
                 for (IAbi a : abis) {
                     addFilter(a.getName(), filter, filters);
                 }
             } else {
                 addFilter(abi, filter, filters);
             }
         }
     }

     private void addFilter(String abi, TestFilter filter,
             Map<String, List<TestFilter>> filters) {
         getFilter(filters, AbiUtils.createId(abi, filter.getName())).add(filter);
     }

     private List<TestFilter> getFilter(Map<String, List<TestFilter>> filters, String id) {
         List<TestFilter> fs = filters.get(id);
         if (fs == null) {
             fs = new ArrayList<>();
             filters.put(id, fs);
         }
         return fs;
     }

     private void addModuleDef(String name, IAbi abi, IRemoteTest test,
             String[] configPaths) throws ConfigurationException {
         // Invokes parser to process the test module config file
         IConfiguration config = mConfigFactory.createConfigurationFromArgs(configPaths);
         addModuleDef(new ModuleDef(name, abi, test, config.getTargetPreparers(),
                 config.getConfigurationDescription()));
     }

     private void addModuleDef(IModuleDef moduleDef) {
         Set<String> tokens = moduleDef.getTokens();
         if (tokens != null && !tokens.isEmpty()) {
             mTokenModules.add(moduleDef);
         } else {
             mNonTokenModules.add(moduleDef);
         }
     }

     private void addFiltersToTest(IRemoteTest test, IAbi abi, String name) {
         String moduleId = AbiUtils.createId(abi.getName(), name);
         if (!(test instanceof ITestFilterReceiver)) {
             throw new IllegalArgumentException(String.format(
                     "Test in module %s must implement ITestFilterReceiver.", moduleId));
         }
         List<TestFilter> mdIncludes = getFilter(mIncludeFilters, moduleId);
         List<TestFilter> mdExcludes = getFilter(mExcludeFilters, moduleId);
         if (!mdIncludes.isEmpty()) {
             addTestIncludes((ITestFilterReceiver) test, mdIncludes, name);
         }
         if (!mdExcludes.isEmpty()) {
             addTestExcludes((ITestFilterReceiver) test, mdExcludes, name);
         }
     }

     private boolean shouldRunModule(String moduleId) {
         List<TestFilter> mdIncludes = getFilter(mIncludeFilters, moduleId);
         List<TestFilter> mdExcludes = getFilter(mExcludeFilters, moduleId);
         // if including all modules or includes exist for this module, and there are not excludes
         // for the entire module, this module should be run.
         return (mIncludeAll || !mdIncludes.isEmpty()) && !containsModuleExclude(mdExcludes);
     }

     private void addTestIncludes(ITestFilterReceiver test, List<TestFilter> includes,
             String name) {
         if (test instanceof ITestFileFilterReceiver) {
             File includeFile = createFilterFile(name, ".include", includes);
             ((ITestFileFilterReceiver)test).setIncludeTestFile(includeFile);
         } else {
             // add test includes one at a time
             for (TestFilter include : includes) {
                 String filterTestName = include.getTest();
                 if (filterTestName != null) {
                     test.addIncludeFilter(filterTestName);
                 }
             }
         }
     }

     private void addTestExcludes(ITestFilterReceiver test, List<TestFilter> excludes,
             String name) {
         if (test instanceof ITestFileFilterReceiver) {
             File excludeFile = createFilterFile(name, ".exclude", excludes);
             ((ITestFileFilterReceiver)test).setExcludeTestFile(excludeFile);
         } else {
             // add test excludes one at a time
             for (TestFilter exclude : excludes) {
                 test.addExcludeFilter(exclude.getTest());
             }
         }
     }

     private File createFilterFile(String prefix, String suffix, List<TestFilter> filters) {
         File filterFile = null;
         PrintWriter out = null;
         try {
             filterFile = FileUtil.createTempFile(prefix, suffix);
             out = new PrintWriter(filterFile);
             for (TestFilter filter : filters) {
                 String filterTest = filter.getTest();
                 if (filterTest != null) {
                     out.println(filterTest);
                 }
             }
             out.flush();
         } catch (IOException e) {
             throw new RuntimeException("Failed to create filter file");
         } finally {
             if (out != null) {
                 out.close();
             }
         }
         filterFile.deleteOnExit();
         return filterFile;
     }

     /*
      * Returns true iff one or more test filters in excludes apply to the entire module.
      */
     private boolean containsModuleExclude(Collection<TestFilter> excludes) {
         for (TestFilter exclude : excludes) {
             if (exclude.getTest() == null) {
                 return true;
             }
         }
         return false;
     }

     /**
      * A {@link FilenameFilter} to find all the config files in a directory.
      */
     public static class ConfigFilter implements FilenameFilter {

         /**
          * {@inheritDoc}
          */
         @Override
         public boolean accept(File dir, String name) {
             CLog.d("%s/%s", dir.getAbsolutePath(), name);
             return name.endsWith(CONFIG_EXT);
         }
     }

     /**
      * {@inheritDoc}
      */
     @Override
     public LinkedList<IModuleDef> getModules(String serial, int shardIndex) {
         Collections.sort(mNonTokenModules, new ExecutionOrderComparator());
         List<IModuleDef> modules = getShard(mNonTokenModules, shardIndex, mTotalShards);
         if (modules == null) {
             modules = new LinkedList<IModuleDef>();
         }
         long estimatedTime = 0;
         for (IModuleDef def : modules) {
             estimatedTime += def.getRuntimeHint();
         }

         // FIXME: Token Modules are the only last part that is not deterministic.
         synchronized (lock) {
             // Get tokens from the device
             Set<String> tokens = mDeviceTokens.get(serial);
             if (tokens != null && !tokens.isEmpty()) {
                 // if it matches any of the token modules, add them
                 for (IModuleDef def : mTokenModules) {
                     if (!mTokenModuleScheduled.contains(def)) {
                         if (tokens.equals(def.getTokens())) {
                             modules.add(def);
                             CLog.d("Adding %s to scheduled token", def);
                             mTokenModuleScheduled.add(def);
                         }
                     }
                 }
             }
             // the last shard going through may add everything remaining.
             if (mInitCount == (mTotalShards - 1) &&
                     mTokenModuleScheduled.size() != mTokenModules.size()) {
                 mTokenModules.removeAll(mTokenModuleScheduled);
                 if (mTotalShards != 1) {
                     // Only print the warnings if we are sharding.
                     CLog.e("Could not find any token for %s. Adding to last shard.", mTokenModules);
                 }
                 modules.addAll(mTokenModules);
             }
             mInitCount++;
         }
         Collections.sort(modules, new ExecutionOrderComparator());
         int uniqueCount = UniqueModuleCountUtil.countUniqueModules(modules);
         CLog.logAndDisplay(LogLevel.INFO, "%s running %s test sub-modules, expected to complete "
                 + "in %s.", serial, uniqueCount, TimeUtil.formatElapsedTime(estimatedTime));
         CLog.d("module list for this shard: %s", modules);
         LinkedList<IModuleDef> tests = new LinkedList<>();
         tests.addAll(modules);
         return tests;
     }

     /**
      * Helper to linearly split the list into shards with balanced runtimeHint.
      * Exposed for testing.
      */
     protected List<IModuleDef> getShard(List<IModuleDef> fullList, int shardIndex, int totalShard) {
         List<List<IModuleDef>> res = LinearPartition.split(fullList, totalShard);
         if (res.isEmpty()) {
             return null;
         }
         if (shardIndex >= res.size()) {
             // If we could not shard up to expectation
             return null;
         }
         return res.get(shardIndex);
     }

     /**
      * @return the {@link List} of modules whose name contains the given pattern.
      */
     public static List<String> getModuleNamesMatching(File directory, String pattern) {
         String[] names = directory.list(new NameFilter(pattern));
         List<String> modules = new ArrayList<String>(names.length);
         for (String name : names) {
             int index = name.indexOf(CONFIG_EXT);
             if (index > 0) {
                 String module = name.substring(0, index);
                 if (module.equals(pattern)) {
                     // Pattern represents a single module, just return a single-item list
                     modules = new ArrayList<>(1);
                     modules.add(module);
                     return modules;
                 }
                 modules.add(module);
             }
         }
         return modules;
     }

     private static void putArgs(List<String> args,
             Map<String, Map<String, List<String>>> argsMap) {
         for (String arg : args) {
             String[] parts = arg.split(":");
             String target = parts[0];
             String name = parts[1];
             String value;
             if (parts.length == 4) {
                 // key and value given, keep the pair delimited by ':' and stored as value
                 value = String.format("%s:%s", parts[2], parts[3]);
             } else {
                 value = parts[2];
             }
             Map<String, List<String>> map = argsMap.get(target);
             if (map == null) {
                 map = new HashMap<>();
                 argsMap.put(target, map);
             }
             List<String> valueList = map.get(name);
             if (valueList == null) {
                 valueList = new ArrayList<>();
                 map.put(name, valueList);
             }
             valueList.add(value);
         }
     }

     /**
      * Sort by name and use runtimeHint for separation, shortest test first.
      */
     private static class ExecutionOrderComparator implements Comparator<IModuleDef> {
         @Override
         public int compare(IModuleDef def1, IModuleDef def2) {
             int value1 = 0;
             int value2 = 0;
             if (ENDING_MODULES.containsKey(def1.getName())) {
                 value1 = ENDING_MODULES.get(def1.getName());
             }
             if (ENDING_MODULES.containsKey(def2.getName())) {
                 value2 = ENDING_MODULES.get(def2.getName());
             }
             if (value1 == 0 && value2 == 0) {
                 int time = (int) Math.signum(def1.getRuntimeHint() - def2.getRuntimeHint());
                 if (time == 0) {
                     return def1.getName().compareTo(def2.getName());
                 }
                 return time;
             }
             return (int) Math.signum(value1 - value2);
         }
     }

     /**
      * {@inheritDoc}
      */
     @Override
     public void tearDown() {
         mNonTokenModules.clear();
         mTokenModules.clear();
         mIncludeFilters.clear();
         mExcludeFilters.clear();
         mTestArgs.clear();
         mModuleArgs.clear();
     }
 }
	/*
	* Copyright (C) 2015 The Android Open Source Project
	*
	* Licensed under the Apache License, Version 2.0 (the "License");
	* you may not use this file except in compliance with the License.
	* You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/
	package com.android.compatibility.common.tradefed.testtype;

	import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper;
	import com.android.compatibility.common.tradefed.result.TestRunHandler;
	import com.android.compatibility.common.tradefed.util.LinearPartition;
	import com.android.compatibility.common.tradefed.util.UniqueModuleCountUtil;
	import com.android.compatibility.common.util.TestFilter;
	import com.android.ddmlib.Log.LogLevel;
	import com.android.tradefed.build.IBuildInfo;
	import com.android.tradefed.config.ConfigurationException;
	import com.android.tradefed.config.ConfigurationFactory;
	import com.android.tradefed.config.IConfiguration;
	import com.android.tradefed.config.IConfigurationFactory;
	import com.android.tradefed.log.LogUtil.CLog;
	import com.android.tradefed.testtype.IAbi;
	import com.android.tradefed.testtype.IBuildReceiver;
	import com.android.tradefed.testtype.IRemoteTest;
	import com.android.tradefed.testtype.IStrictShardableTest;
	import com.android.tradefed.testtype.ITestFileFilterReceiver;
	import com.android.tradefed.testtype.ITestFilterReceiver;
	import com.android.tradefed.util.AbiUtils;
	import com.android.tradefed.util.FileUtil;
	import com.android.tradefed.util.TimeUtil;

	import java.io.File;
	import java.io.FilenameFilter;
	import java.io.IOException;
	import java.io.PrintWriter;
	import java.util.ArrayList;
	import java.util.Collection;
	import java.util.Collections;
	import java.util.Comparator;
	import java.util.HashMap;
	import java.util.HashSet;
	import java.util.LinkedList;
	import java.util.List;
	import java.util.Map;
	import java.util.Map.Entry;
	import java.util.Set;

	/**
	* Retrieves Compatibility test module definitions from the repository.
	*/
	public class ModuleRepo implements IModuleRepo {

	private static final String CONFIG_EXT = ".config";
	private static final Map<String, Integer> ENDING_MODULES = new HashMap<>();
	static {
	ENDING_MODULES.put("CtsMonkeyTestCases", 1);
	}
	// Synchronization objects for Token Modules.
	private int mInitCount = 0;
	private Set<IModuleDef> mTokenModuleScheduled;
	private static Object lock = new Object();

	private int mTotalShards;
	private Integer mShardIndex;

	private Map<String, Set<String>> mDeviceTokens = new HashMap<>();
	private Map<String, Map<String, List<String>>> mTestArgs = new HashMap<>();
	private Map<String, Map<String, List<String>>> mModuleArgs = new HashMap<>();
	private boolean mIncludeAll;
	private Map<String, List<TestFilter>> mIncludeFilters = new HashMap<>();
	private Map<String, List<TestFilter>> mExcludeFilters = new HashMap<>();
	private IConfigurationFactory mConfigFactory = ConfigurationFactory.getInstance();

	private volatile boolean mInitialized = false;

	// Holds all the tests with tokens waiting to be run. Meaning the DUT must have a specific token.
	private List<IModuleDef> mTokenModules = new ArrayList<>();
	private List<IModuleDef> mNonTokenModules = new ArrayList<>();

	/**
	* {@inheritDoc}
	*/
	@Override
	public int getNumberOfShards() {
	return mTotalShards;
	}

	/**
	* Returns the device tokens of this module repo. Exposed for testing.
	*/
	protected Map<String, Set<String>> getDeviceTokens() {
	return mDeviceTokens;
	}

	/**
	* A {@link FilenameFilter} to find all modules in a directory who match the given pattern.
	*/
	public static class NameFilter implements FilenameFilter {

	private String mPattern;

	public NameFilter(String pattern) {
	mPattern = pattern;
	}

	/**
	* {@inheritDoc}
	*/
	@Override
	public boolean accept(File dir, String name) {
	return name.contains(mPattern) && name.endsWith(CONFIG_EXT);
	}
	}

	/**
	* {@inheritDoc}
	*/
	@Override
	public List<IModuleDef> getNonTokenModules() {
	return mNonTokenModules;
	}

	/**
	* {@inheritDoc}
	*/
	@Override
	public List<IModuleDef> getTokenModules() {
	return mTokenModules;
	}

	/**
	* {@inheritDoc}
	*/
	@Override
	public String[] getModuleIds() {
	Set<String> moduleIdSet = new HashSet<>();
	for (IModuleDef moduleDef : mNonTokenModules) {
	moduleIdSet.add(moduleDef.getId());
	}
	for (IModuleDef moduleDef : mTokenModules) {
	moduleIdSet.add(moduleDef.getId());
	}
	return moduleIdSet.toArray(new String[moduleIdSet.size()]);
	}

	/**
	* {@inheritDoc}
	*/
	@Override
	public boolean isInitialized() {
	return mInitialized;
	}

	/**
	* {@inheritDoc}
	*/
	@Override
	public void initialize(int totalShards, Integer shardIndex, File testsDir, Set<IAbi> abis,
	List<String> deviceTokens, List<String> testArgs, List<String> moduleArgs,
	Set<String> includeFilters, Set<String> excludeFilters, IBuildInfo buildInfo) {
	CLog.d("Initializing ModuleRepo\nShards:%d\nTests Dir:%s\nABIs:%s\nDevice Tokens:%s\n" +
	"Test Args:%s\nModule Args:%s\nIncludes:%s\nExcludes:%s",
	totalShards, testsDir.getAbsolutePath(), abis, deviceTokens, testArgs, moduleArgs,
	includeFilters, excludeFilters);
	mInitialized = true;
	mTotalShards = totalShards;
	mShardIndex = shardIndex;
	synchronized (lock) {
	if (mTokenModuleScheduled == null) {
	mTokenModuleScheduled = new HashSet<>();
	}
	}

	for (String line : deviceTokens) {
	String[] parts = line.split(":");
	if (parts.length == 2) {
	String key = parts[0];
	String value = parts[1];
	Set<String> list = mDeviceTokens.get(key);
	if (list == null) {
	list = new HashSet<>();
	mDeviceTokens.put(key, list);
	}
	list.add(value);
	} else {
	throw new IllegalArgumentException(
	String.format("Could not parse device token: %s", line));
	}
	}
	putArgs(testArgs, mTestArgs);
	putArgs(moduleArgs, mModuleArgs);
	mIncludeAll = includeFilters.isEmpty();
	// Include all the inclusions
	addFilters(includeFilters, mIncludeFilters, abis);
	// Exclude all the exclusions
	addFilters(excludeFilters, mExcludeFilters, abis);

	File[] configFiles = testsDir.listFiles(new ConfigFilter());
	if (configFiles.length == 0) {
	throw new IllegalArgumentException(
	String.format("No config files found in %s", testsDir.getAbsolutePath()));
	}
	Map<String, Integer> shardedTestCounts = new HashMap<>();
	for (File configFile : configFiles) {
	final String name = configFile.getName().replace(CONFIG_EXT, "");
	final String[] pathArg = new String[] { configFile.getAbsolutePath() };
	try {
	// Invokes parser to process the test module config file
	// Need to generate a different config for each ABI as we cannot guarantee the
	// configs are idempotent. This however means we parse the same file multiple times
	for (IAbi abi : abis) {
	String id = AbiUtils.createId(abi.getName(), name);
	if (!shouldRunModule(id)) {
	// If the module should not run tests based on the state of filters,
	// skip this name/abi combination.
	continue;
	}

	IConfiguration config = mConfigFactory.createConfigurationFromArgs(pathArg);
	Map<String, List<String>> args = new HashMap<>();
	if (mModuleArgs.containsKey(name)) {
	args.putAll(mModuleArgs.get(name));
	}
	if (mModuleArgs.containsKey(id)) {
	args.putAll(mModuleArgs.get(id));
	}
	for (Entry<String, List<String>> entry : args.entrySet()) {
	for (String entryValue : entry.getValue()) {
	// Collection-type options can be injected with multiple values
	String entryName = entry.getKey();
	if (entryValue.contains(":")) {
	// entryValue is key-value pair
	String key = entryValue.split(":")[0];
	String value = entryValue.split(":")[1];
	config.injectOptionValue(entryName, key, value);
	} else {
	// entryValue is just the argument value
	config.injectOptionValue(entryName, entryValue);
	}
	}
	}

	List<IRemoteTest> tests = config.getTests();
	for (IRemoteTest test : tests) {
	String className = test.getClass().getName();
	Map<String, List<String>> testArgsMap = new HashMap<>();
	if (mTestArgs.containsKey(className)) {
	testArgsMap.putAll(mTestArgs.get(className));
	}
	for (Entry<String, List<String>> entry : testArgsMap.entrySet()) {
	for (String entryValue : entry.getValue()) {
	String entryName = entry.getKey();
	if (entryValue.contains(":")) {
	// entryValue is key-value pair
	String key = entryValue.split(":")[0];
	String value = entryValue.split(":")[1];
	config.injectOptionValue(entryName, key, value);
	} else {
	// entryValue is just the argument value
	config.injectOptionValue(entryName, entryValue);
	}
	}
	}
	addFiltersToTest(test, abi, name);
	}
	List<IRemoteTest> shardedTests = tests;
	if (mTotalShards > 1) {
	shardedTests = splitShardableTests(tests, buildInfo);
	}
	if (shardedTests.size() > 1) {
	shardedTestCounts.put(id, shardedTests.size());
	}
	for (IRemoteTest test : shardedTests) {
	addModuleDef(name, abi, test, pathArg);
	}
	}
	} catch (ConfigurationException e) {
	throw new RuntimeException(String.format("error parsing config file: %s",
	configFile.getName()), e);
	}
	}
	mExcludeFilters.clear();
	TestRunHandler.setTestRuns(new CompatibilityBuildHelper(buildInfo), shardedTestCounts);
	}

	private List<IRemoteTest> splitShardableTests(List<IRemoteTest> tests, IBuildInfo buildInfo) {
	ArrayList<IRemoteTest> shardedList = new ArrayList<>(tests.size());
	for (IRemoteTest test : tests) {
	if (test instanceof IBuildReceiver) {
	((IBuildReceiver)test).setBuild(buildInfo);
	}
	if (mShardIndex != null && test instanceof IStrictShardableTest) {
	for (int i = 0; i < mTotalShards; i++) {
	shardedList.add(((IStrictShardableTest)test).getTestShard(mTotalShards, i));
	}
	} else {
	shardedList.add(test);
	}
	}
	return shardedList;
	}

	private void addFilters(Set<String> stringFilters,
	Map<String, List<TestFilter>> filters, Set<IAbi> abis) {
	for (String filterString : stringFilters) {
	TestFilter filter = TestFilter.createFrom(filterString);
	String abi = filter.getAbi();
	if (abi == null) {
	for (IAbi a : abis) {
	addFilter(a.getName(), filter, filters);
	}
	} else {
	addFilter(abi, filter, filters);
	}
	}
	}

	private void addFilter(String abi, TestFilter filter,
	Map<String, List<TestFilter>> filters) {
	getFilter(filters, AbiUtils.createId(abi, filter.getName())).add(filter);
	}

	private List<TestFilter> getFilter(Map<String, List<TestFilter>> filters, String id) {
	List<TestFilter> fs = filters.get(id);
	if (fs == null) {
	fs = new ArrayList<>();
	filters.put(id, fs);
	}
	return fs;
	}

	private void addModuleDef(String name, IAbi abi, IRemoteTest test,
	String[] configPaths) throws ConfigurationException {
	// Invokes parser to process the test module config file
	IConfiguration config = mConfigFactory.createConfigurationFromArgs(configPaths);
	addModuleDef(new ModuleDef(name, abi, test, config.getTargetPreparers(),
	config.getConfigurationDescription()));
	}

	private void addModuleDef(IModuleDef moduleDef) {
	Set<String> tokens = moduleDef.getTokens();
	if (tokens != null && !tokens.isEmpty()) {
	mTokenModules.add(moduleDef);
	} else {
	mNonTokenModules.add(moduleDef);
	}
	}

	private void addFiltersToTest(IRemoteTest test, IAbi abi, String name) {
	String moduleId = AbiUtils.createId(abi.getName(), name);
	if (!(test instanceof ITestFilterReceiver)) {
	throw new IllegalArgumentException(String.format(
	"Test in module %s must implement ITestFilterReceiver.", moduleId));
	}
	List<TestFilter> mdIncludes = getFilter(mIncludeFilters, moduleId);
	List<TestFilter> mdExcludes = getFilter(mExcludeFilters, moduleId);
	if (!mdIncludes.isEmpty()) {
	addTestIncludes((ITestFilterReceiver) test, mdIncludes, name);
	}
	if (!mdExcludes.isEmpty()) {
	addTestExcludes((ITestFilterReceiver) test, mdExcludes, name);
	}
	}

	private boolean shouldRunModule(String moduleId) {
	List<TestFilter> mdIncludes = getFilter(mIncludeFilters, moduleId);
	List<TestFilter> mdExcludes = getFilter(mExcludeFilters, moduleId);
	// if including all modules or includes exist for this module, and there are not excludes
	// for the entire module, this module should be run.
	return (mIncludeAll \|\| !mdIncludes.isEmpty()) && !containsModuleExclude(mdExcludes);
	}

	private void addTestIncludes(ITestFilterReceiver test, List<TestFilter> includes,
	String name) {
	if (test instanceof ITestFileFilterReceiver) {
	File includeFile = createFilterFile(name, ".include", includes);
	((ITestFileFilterReceiver)test).setIncludeTestFile(includeFile);
	} else {
	// add test includes one at a time
	for (TestFilter include : includes) {
	String filterTestName = include.getTest();
	if (filterTestName != null) {
	test.addIncludeFilter(filterTestName);
	}
	}
	}
	}

	private void addTestExcludes(ITestFilterReceiver test, List<TestFilter> excludes,
	String name) {
	if (test instanceof ITestFileFilterReceiver) {
	File excludeFile = createFilterFile(name, ".exclude", excludes);
	((ITestFileFilterReceiver)test).setExcludeTestFile(excludeFile);
	} else {
	// add test excludes one at a time
	for (TestFilter exclude : excludes) {
	test.addExcludeFilter(exclude.getTest());
	}
	}
	}

	private File createFilterFile(String prefix, String suffix, List<TestFilter> filters) {
	File filterFile = null;
	PrintWriter out = null;
	try {
	filterFile = FileUtil.createTempFile(prefix, suffix);
	out = new PrintWriter(filterFile);
	for (TestFilter filter : filters) {
	String filterTest = filter.getTest();
	if (filterTest != null) {
	out.println(filterTest);
	}
	}
	out.flush();
	} catch (IOException e) {
	throw new RuntimeException("Failed to create filter file");
	} finally {
	if (out != null) {
	out.close();
	}
	}
	filterFile.deleteOnExit();
	return filterFile;
	}

	/*
	* Returns true iff one or more test filters in excludes apply to the entire module.
	*/
	private boolean containsModuleExclude(Collection<TestFilter> excludes) {
	for (TestFilter exclude : excludes) {
	if (exclude.getTest() == null) {
	return true;
	}
	}
	return false;
	}

	/**
	* A {@link FilenameFilter} to find all the config files in a directory.
	*/
	public static class ConfigFilter implements FilenameFilter {

	/**
	* {@inheritDoc}
	*/
	@Override
	public boolean accept(File dir, String name) {
	CLog.d("%s/%s", dir.getAbsolutePath(), name);
	return name.endsWith(CONFIG_EXT);
	}
	}

	/**
	* {@inheritDoc}
	*/
	@Override
	public LinkedList<IModuleDef> getModules(String serial, int shardIndex) {
	Collections.sort(mNonTokenModules, new ExecutionOrderComparator());
	List<IModuleDef> modules = getShard(mNonTokenModules, shardIndex, mTotalShards);
	if (modules == null) {
	modules = new LinkedList<IModuleDef>();
	}
	long estimatedTime = 0;
	for (IModuleDef def : modules) {
	estimatedTime += def.getRuntimeHint();
	}

	// FIXME: Token Modules are the only last part that is not deterministic.
	synchronized (lock) {
	// Get tokens from the device
	Set<String> tokens = mDeviceTokens.get(serial);
	if (tokens != null && !tokens.isEmpty()) {
	// if it matches any of the token modules, add them
	for (IModuleDef def : mTokenModules) {
	if (!mTokenModuleScheduled.contains(def)) {
	if (tokens.equals(def.getTokens())) {
	modules.add(def);
	CLog.d("Adding %s to scheduled token", def);
	mTokenModuleScheduled.add(def);
	}
	}
	}
	}
	// the last shard going through may add everything remaining.
	if (mInitCount == (mTotalShards - 1) &&
	mTokenModuleScheduled.size() != mTokenModules.size()) {
	mTokenModules.removeAll(mTokenModuleScheduled);
	if (mTotalShards != 1) {
	// Only print the warnings if we are sharding.
	CLog.e("Could not find any token for %s. Adding to last shard.", mTokenModules);
	}
	modules.addAll(mTokenModules);
	}
	mInitCount++;
	}
	Collections.sort(modules, new ExecutionOrderComparator());
	int uniqueCount = UniqueModuleCountUtil.countUniqueModules(modules);
	CLog.logAndDisplay(LogLevel.INFO, "%s running %s test sub-modules, expected to complete "
	+ "in %s.", serial, uniqueCount, TimeUtil.formatElapsedTime(estimatedTime));
	CLog.d("module list for this shard: %s", modules);
	LinkedList<IModuleDef> tests = new LinkedList<>();
	tests.addAll(modules);
	return tests;
	}

	/**
	* Helper to linearly split the list into shards with balanced runtimeHint.
	* Exposed for testing.
	*/
	protected List<IModuleDef> getShard(List<IModuleDef> fullList, int shardIndex, int totalShard) {
	List<List<IModuleDef>> res = LinearPartition.split(fullList, totalShard);
	if (res.isEmpty()) {
	return null;
	}
	if (shardIndex >= res.size()) {
	// If we could not shard up to expectation
	return null;
	}
	return res.get(shardIndex);
	}

	/**
	* @return the {@link List} of modules whose name contains the given pattern.
	*/
	public static List<String> getModuleNamesMatching(File directory, String pattern) {
	String[] names = directory.list(new NameFilter(pattern));
	List<String> modules = new ArrayList<String>(names.length);
	for (String name : names) {
	int index = name.indexOf(CONFIG_EXT);
	if (index > 0) {
	String module = name.substring(0, index);
	if (module.equals(pattern)) {
	// Pattern represents a single module, just return a single-item list
	modules = new ArrayList<>(1);
	modules.add(module);
	return modules;
	}
	modules.add(module);
	}
	}
	return modules;
	}

	private static void putArgs(List<String> args,
	Map<String, Map<String, List<String>>> argsMap) {
	for (String arg : args) {
	String[] parts = arg.split(":");
	String target = parts[0];
	String name = parts[1];
	String value;
	if (parts.length == 4) {
	// key and value given, keep the pair delimited by ':' and stored as value
	value = String.format("%s:%s", parts[2], parts[3]);
	} else {
	value = parts[2];
	}
	Map<String, List<String>> map = argsMap.get(target);
	if (map == null) {
	map = new HashMap<>();
	argsMap.put(target, map);
	}
	List<String> valueList = map.get(name);
	if (valueList == null) {
	valueList = new ArrayList<>();
	map.put(name, valueList);
	}
	valueList.add(value);
	}
	}

	/**
	* Sort by name and use runtimeHint for separation, shortest test first.
	*/
	private static class ExecutionOrderComparator implements Comparator<IModuleDef> {
	@Override
	public int compare(IModuleDef def1, IModuleDef def2) {
	int value1 = 0;
	int value2 = 0;
	if (ENDING_MODULES.containsKey(def1.getName())) {
	value1 = ENDING_MODULES.get(def1.getName());
	}
	if (ENDING_MODULES.containsKey(def2.getName())) {
	value2 = ENDING_MODULES.get(def2.getName());
	}
	if (value1 == 0 && value2 == 0) {
	int time = (int) Math.signum(def1.getRuntimeHint() - def2.getRuntimeHint());
	if (time == 0) {
	return def1.getName().compareTo(def2.getName());
	}
	return time;
	}
	return (int) Math.signum(value1 - value2);
	}
	}

	/**
	* {@inheritDoc}
	*/
	@Override
	public void tearDown() {
	mNonTokenModules.clear();
	mTokenModules.clear();
	mIncludeFilters.clear();
	mExcludeFilters.clear();
	mTestArgs.clear();
	mModuleArgs.clear();
	}
	}