DoU profile: running tests at timestamps.
Test: make LongevityPlatformLibSamples; install; instrument with "-e
profile sample_profile" and note the start time of the tests or check
LongevitySuite entries in logcat.
Bug: 117168431
Change-Id: I3d458d20e299562f753fb680c1e94905ec3484e5
diff --git a/libraries/composer/host/res/sample_profile.textpb b/libraries/composer/host/res/sample_profile.textpb
index a80d08f..7e2496b 100644
--- a/libraries/composer/host/res/sample_profile.textpb
+++ b/libraries/composer/host/res/sample_profile.textpb
@@ -3,17 +3,17 @@
}
scenarios [{
at: "00:00:10"
- journey: "android.platform.test.scenario.calendar.FlingDayPage"
+ journey: "android.longevity.platform.samples.SimpleSuite$PassingTest"
}, {
at: "00:01:00"
- journey: "android.platform.test.scenario.calendar.FlingWeekPage"
+ journey: "android.longevity.platform.samples.SimpleSuite$FailingTest"
}, {
at: "00:02:00"
- journey: "android.platform.test.scenario.gmail.OpenApp"
+ journey: "android.longevity.platform.samples.SimpleSuite$PassingTest"
}, {
at: "00:03:00"
- journey: "android.platform.test.scenario.quicksettings.ToggleWifiOff"
+ journey: "android.longevity.platform.samples.SimpleSuite$PassingTest"
}, {
at: "00:04:00"
- journey: "android.platform.test.scenario.quicksettings.ToggleWifiOn"
+ journey: "android.longevity.platform.samples.SimpleSuite$FailingTest"
}]
diff --git a/libraries/composer/host/src/android/host/test/composer/Profile.java b/libraries/composer/host/src/android/host/test/composer/Profile.java
index 51ff7b9..51c7032 100644
--- a/libraries/composer/host/src/android/host/test/composer/Profile.java
+++ b/libraries/composer/host/src/android/host/test/composer/Profile.java
@@ -27,6 +27,10 @@
* An extension of {@link android.host.test.composer.ProfileBase} for host-side testing.
*/
public class Profile extends ProfileBase<Map<String, String>> {
+ public Profile(Map<String, String> args) {
+ super(args);
+ }
+
@Override
protected Configuration getConfigurationArgument(Map<String, String> args) {
if (!args.containsKey(PROFILE_OPTION_NAME)) {
diff --git a/libraries/composer/host/src/android/host/test/composer/ProfileBase.java b/libraries/composer/host/src/android/host/test/composer/ProfileBase.java
index 2424b9a..628ecb4 100644
--- a/libraries/composer/host/src/android/host/test/composer/ProfileBase.java
+++ b/libraries/composer/host/src/android/host/test/composer/ProfileBase.java
@@ -22,16 +22,21 @@
import org.junit.runner.Runner;
import java.lang.IllegalArgumentException;
+import java.text.SimpleDateFormat;
+import java.text.ParseException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
+import java.util.TimeZone;
import java.util.function.Function;
import java.util.stream.Collectors;
/**
- * A {@link Compose} function base class for taking a profile and returning the tests in the specified sequence.
+ * A {@link Compose} function base class for taking a profile and returning the tests in the
+ * specified sequence.
*/
public abstract class ProfileBase<T> implements Compose<T, Runner> {
protected static final String PROFILE_OPTION_NAME = "profile";
@@ -39,8 +44,34 @@
private static final String LOG_TAG = ProfileBase.class.getSimpleName();
- // Store the configuration to be read by the test runner.
- private Configuration mConfiguration;
+ // Parser for parsing "at" timestamps in profiles.
+ private static final SimpleDateFormat TIMESTAMP_FORMATTER = new SimpleDateFormat("HH:mm:ss");
+
+ // Keeps track of the current scenario to run.
+ private int mScenarioIndex = 0;
+ // A list of scenarios in the order that they will be run.
+ private List<Scenario> mOrderedScenariosList;
+ // Timestamp when the test run starts, defaults to time when the ProfileBase object is
+ // constructed. Can be overridden by {@link setTestRunStartTimeMillis}.
+ // TODO(b/118843085): Clarify whether timestamps are relative to run start time or device clock.
+ private long mRunStartTimeMillis = System.currentTimeMillis();
+
+ public ProfileBase(T args) {
+ super();
+ // Set the timestamp parser to UTC to get test timstamps as "time elapsed since zero".
+ TIMESTAMP_FORMATTER.setTimeZone(TimeZone.getTimeZone("UTC"));
+
+ // Load configuration from arguments and stored the list of scenarios sorted according to
+ // their timestamps.
+ Configuration config = getConfigurationArgument(args);
+ if (config == null) {
+ return;
+ }
+ mOrderedScenariosList = new ArrayList<Scenario>(config.getScenariosList());
+ if (config.hasScheduled()) {
+ Collections.sort(mOrderedScenariosList, new ScenarioTimestampComparator());
+ }
+ }
// Comparator for sorting timstamped CUJs.
private static class ScenarioTimestampComparator implements Comparator<Scenario> {
@@ -55,19 +86,11 @@
@Override
public List<Runner> apply(T args, List<Runner> input) {
- mConfiguration = getConfigurationArgument(args);
- if (mConfiguration == null) {
+ Configuration config = getConfigurationArgument(args);
+ if (config == null) {
return input;
}
- return getTestSequenceFromConfiguration(mConfiguration, input);
- }
-
- public boolean isConfigurationLoaded() {
- return (mConfiguration == null);
- }
-
- public Configuration getLoadedConfiguration() {
- return mConfiguration;
+ return getTestSequenceFromConfiguration(config, input);
}
protected List<Runner> getTestSequenceFromConfiguration(
@@ -77,13 +100,9 @@
Collectors.toMap(
r -> r.getDescription().getDisplayName(), Function.identity()));
logInfo(LOG_TAG, String.format(
- "Available scenarios: %s",
+ "Available journeys: %s",
nameToRunner.keySet().stream().collect(Collectors.joining(", "))));
- List<Scenario> scenarios = new ArrayList<Scenario>(config.getScenariosList());
- if (config.hasScheduled()) {
- Collections.sort(scenarios, new ScenarioTimestampComparator());
- }
- List<Runner> result = scenarios
+ List<Runner> result = mOrderedScenariosList
.stream()
.map(Configuration.Scenario::getJourney)
.map(
@@ -93,7 +112,8 @@
} else {
throw new IllegalArgumentException(
String.format(
- "Journey %s in profile does not exist.",
+ "Journey %s in profile not found. "
+ + "Check logcat to see available journeys.",
journeyName));
}
})
@@ -108,6 +128,55 @@
}
/**
+ * Enables classes using the profile composer to set the test run start time.
+ */
+ public void setTestRunStartTimeMillis(long timestamp) {
+ mRunStartTimeMillis = timestamp;
+ }
+
+ /**
+ * Called by suite runners to signal that a scenario/test has ended; increments the scenario
+ * index.
+ */
+ public void scenarioEnded() {
+ mScenarioIndex += 1;
+ }
+
+ /**
+ * Returns true if there is a next scheduled scenario to run. If no profile is supplied, returns
+ * false.
+ */
+ public boolean hasNextScheduledScenario() {
+ return (mOrderedScenariosList != null) && (mScenarioIndex < mOrderedScenariosList.size());
+ }
+
+ /**
+ * Returns time in milliseconds until the next scenario.
+ */
+ public long getMillisecondsUntilNextScenario() {
+ Scenario nextScenario = mOrderedScenariosList.get(mScenarioIndex);
+ if (nextScenario.hasAt()) {
+ try {
+ long startTimeMillis = TIMESTAMP_FORMATTER.parse(nextScenario.getAt()).getTime();
+ // Time in milliseconds from the start of the test run to the current point in time.
+ long currentTimeMillis = System.currentTimeMillis() - mRunStartTimeMillis;
+ // If the next test should not start yet, sleep until its start time. Otherwise,
+ // start it immediately.
+ // TODO(b/118495360): Deal with the IfLate situation.
+ if (startTimeMillis > currentTimeMillis) {
+ return startTimeMillis - currentTimeMillis;
+ }
+ } catch (ParseException e) {
+ throw new IllegalArgumentException(
+ String.format("Time %s from scenario %s could not be parsed",
+ nextScenario.getAt(), nextScenario.getJourney()));
+ }
+ }
+ // For non-scheduled profiles (not a priority at this point), simply return 0.
+ return 0L;
+ }
+
+ /**
* Parses the arguments, reads the configuration file and returns the Configuraiton object.
*
* If no profile option is found in the arguments, function should return null, in which case
diff --git a/libraries/composer/host/tests/src/android/host/test/composer/ProfileTest.java b/libraries/composer/host/tests/src/android/host/test/composer/ProfileTest.java
index 764218c..8f785b3 100644
--- a/libraries/composer/host/tests/src/android/host/test/composer/ProfileTest.java
+++ b/libraries/composer/host/tests/src/android/host/test/composer/ProfileTest.java
@@ -30,6 +30,10 @@
@RunWith(JUnit4.class)
public class ProfileTest extends ProfileTestBase<Map<String, String>> {
protected class TestableProfile extends ProfileBase<Map<String, String>> {
+ public TestableProfile(Map<String, String> args) {
+ super(args);
+ }
+
@Override
protected Configuration getConfigurationArgument(Map<String, String> args) {
return TEST_CONFIGS.get(args.get(PROFILE_OPTION_NAME));
@@ -37,8 +41,8 @@
}
@Override
- protected ProfileBase<Map<String, String>> getProfile() {
- return new TestableProfile();
+ protected ProfileBase<Map<String, String>> getProfile(Map<String, String> args) {
+ return new TestableProfile(args);
}
@Override
diff --git a/libraries/composer/host/tests/src/android/host/test/composer/ProfileTestBase.java b/libraries/composer/host/tests/src/android/host/test/composer/ProfileTestBase.java
index 10ed8cf..9b2d403 100644
--- a/libraries/composer/host/tests/src/android/host/test/composer/ProfileTestBase.java
+++ b/libraries/composer/host/tests/src/android/host/test/composer/ProfileTestBase.java
@@ -138,7 +138,8 @@
"android.platform.test.scenario.calendar.FlingWeekPage",
"android.platform.test.scenario.calendar.FlingDayPage");
- List<Runner> output = getProfile().apply(getArguments(VALID_CONFIG_KEY), mMockInput);
+ List<Runner> output = getProfile(getArguments(VALID_CONFIG_KEY))
+ .apply(getArguments(VALID_CONFIG_KEY), mMockInput);
List<String> outputDescriptions = output.stream().map(r ->
r.getDescription().getDisplayName()).collect(Collectors.toList());
boolean respected = outputDescriptions.equals(expectedJourneyOrder);
@@ -153,12 +154,12 @@
public void testProfileWithInvalidScenarioThrows() {
// An exception about nonexistent user journey should be thrown.
exceptionThrown.expect(IllegalArgumentException.class);
- exceptionThrown.expectMessage("does not exist");
+ exceptionThrown.expectMessage("not found");
exceptionThrown.expectMessage("invalid");
// Attempt to apply a profile with invalid CUJ; the above exception should be thrown.
- List<Runner> output = getProfile().apply(
- getArguments(CONFIG_WITH_INVALID_JOURNEY_KEY), mMockInput);
+ List<Runner> output = getProfile(getArguments(CONFIG_WITH_INVALID_JOURNEY_KEY))
+ .apply(getArguments(CONFIG_WITH_INVALID_JOURNEY_KEY), mMockInput);
}
/**
@@ -173,11 +174,11 @@
// Attempt to apply a scheduled profile with missing timestamps; the above exception should
// be thrown.
- List<Runner> output = getProfile().apply(
- getArguments(CONFIG_WITH_MISSING_TIMESTAMPS_KEY), mMockInput);
+ List<Runner> output = getProfile(getArguments(CONFIG_WITH_MISSING_TIMESTAMPS_KEY))
+ .apply(getArguments(CONFIG_WITH_MISSING_TIMESTAMPS_KEY), mMockInput);
}
- protected abstract ProfileBase<T> getProfile();
+ protected abstract ProfileBase<T> getProfile(T args);
protected abstract T getArguments(String configName);
}
diff --git a/libraries/composer/platform/src/android/platform/test/composer/Profile.java b/libraries/composer/platform/src/android/platform/test/composer/Profile.java
index b612714..78764fa 100644
--- a/libraries/composer/platform/src/android/platform/test/composer/Profile.java
+++ b/libraries/composer/platform/src/android/platform/test/composer/Profile.java
@@ -40,6 +40,11 @@
*
* TODO(harrytczhang@): Write tests for this logic.
*/
+
+ public Profile(Bundle args) {
+ super(args);
+ }
+
@Override
protected Configuration getConfigurationArgument(Bundle args) {
// profileValue is either the name of a profile bundled with an APK or a path to a
diff --git a/libraries/composer/platform/tests/src/android/platform/test/composer/ProfileTest.java b/libraries/composer/platform/tests/src/android/platform/test/composer/ProfileTest.java
index 83b1849..fcdbed6 100644
--- a/libraries/composer/platform/tests/src/android/platform/test/composer/ProfileTest.java
+++ b/libraries/composer/platform/tests/src/android/platform/test/composer/ProfileTest.java
@@ -30,6 +30,10 @@
@RunWith(JUnit4.class)
public class ProfileTest extends ProfileTestBase<Bundle> {
protected class TestableProfile extends ProfileBase<Bundle> {
+ public TestableProfile(Bundle args) {
+ super(args);
+ }
+
@Override
protected Configuration getConfigurationArgument(Bundle args) {
return TEST_CONFIGS.get(args.getString(PROFILE_OPTION_NAME));
@@ -37,8 +41,8 @@
}
@Override
- protected ProfileBase<Bundle> getProfile() {
- return new TestableProfile();
+ protected ProfileBase<Bundle> getProfile(Bundle args) {
+ return new TestableProfile(args);
}
@Override
diff --git a/libraries/longevity/src/android/longevity/core/LongevitySuite.java b/libraries/longevity/src/android/longevity/core/LongevitySuite.java
index 355d50f..12f1fa3 100644
--- a/libraries/longevity/src/android/longevity/core/LongevitySuite.java
+++ b/libraries/longevity/src/android/longevity/core/LongevitySuite.java
@@ -94,7 +94,7 @@
}
// Construct and store custom runners for the full suite.
BiFunction<Map<String, String>, List<Runner>, List<Runner>> modifier =
- new Iterate<Runner>().andThen(new Shuffle<Runner>()).andThen(new Profile());
+ new Iterate<Runner>().andThen(new Shuffle<Runner>()).andThen(new Profile(args));
return modifier.apply(args, builder.runners(suite, annotation.value()));
}
diff --git a/libraries/longevity/src/android/longevity/platform/LongevitySuite.java b/libraries/longevity/src/android/longevity/platform/LongevitySuite.java
index aa88961..6fb8817 100644
--- a/libraries/longevity/src/android/longevity/platform/LongevitySuite.java
+++ b/libraries/longevity/src/android/longevity/platform/LongevitySuite.java
@@ -24,6 +24,7 @@
import android.longevity.platform.listener.TimeoutTerminator;
import android.os.BatteryManager;
import android.os.Bundle;
+import android.os.SystemClock;
import android.platform.test.composer.Iterate;
import android.platform.test.composer.Shuffle;
import android.platform.test.composer.Profile;
@@ -52,6 +53,9 @@
private Instrumentation mInstrumentation;
private Context mContext;
+ // Platform Profile instance for scheduling tests.
+ private Profile mProfile;
+
/**
* Takes a {@link Bundle} and maps all String K/V pairs into a {@link Map<String, String>}.
*
@@ -90,6 +94,7 @@
super(klass, constructClassRunners(klass, builder, arguments), toMap(arguments));
mInstrumentation = instrumentation;
mContext = context;
+ mProfile = new Profile(arguments);
}
/**
@@ -108,7 +113,7 @@
}
// Construct and store custom runners for the full suite.
BiFunction<Bundle, List<Runner>, List<Runner>> modifier =
- new Iterate<Runner>().andThen(new Shuffle<Runner>()).andThen(new Profile());
+ new Iterate<Runner>().andThen(new Shuffle<Runner>()).andThen(new Profile(args));
return modifier.apply(args, builder.runners(suite, annotation.value()));
}
@@ -118,10 +123,28 @@
if (hasBattery()) {
notifier.addListener(new BatteryTerminator(notifier, mArguments, mContext));
}
+ // Set the test run start time in the profile composer and sleep until the first scheduled
+ // test starts. When no profile is supplied, hasNextScheduledScenario() returns false and
+ // no sleep is performed.
+ if (mProfile.hasNextScheduledScenario()) {
+ mProfile.setTestRunStartTimeMillis(System.currentTimeMillis());
+ SystemClock.sleep(mProfile.getMillisecondsUntilNextScenario());
+ }
// Register other listeners and continue with standard longevity run.
super.run(notifier);
}
+ @Override
+ protected void runChild(Runner runner, final RunNotifier notifier) {
+ super.runChild(runner, notifier);
+ mProfile.scenarioEnded();
+ // If there are remaining scenarios, Sleep until the next one starts.
+ // When no profile is supplied, allScenariosDone() returns true and no sleep is performed.
+ if (mProfile.hasNextScheduledScenario()) {
+ SystemClock.sleep(mProfile.getMillisecondsUntilNextScenario());
+ }
+ }
+
/**
* Returns the platform-specific {@link TimeoutTerminator} for Android devices.
*/
