Allow running multiple dEQP tests per process
This CL adds a new command line argument, --max-tests-per-proc,
which can be set to a value greater than 1 to run, at most, that
many tests per process (before this CL, there would always be 1
test per process).
If there is anything unexpected in the output while running
multiple tests at once, Regres will try again and re-run each test
individually in order to get a proper per test output.
Bug: b/253530501
Change-Id: I70e7f1fa86e001c834ef5b98f766e5db5d20ebe7
Reviewed-on: https://swiftshader-review.googlesource.com/c/SwiftShader/+/69128
Commit-Queue: Alexis Hétu <sugoi@google.com>
Tested-by: Alexis Hétu <sugoi@google.com>
Reviewed-by: Nicolas Capens <nicolascapens@google.com>
Presubmit-Ready: Alexis Hétu <sugoi@google.com>
Kokoro-Result: kokoro <noreply+kokoro@google.com>
diff --git a/tests/regres/cmd/regres/main.go b/tests/regres/cmd/regres/main.go
index 5048199..e7ed94e 100644
--- a/tests/regres/cmd/regres/main.go
+++ b/tests/regres/cmd/regres/main.go
@@ -83,20 +83,21 @@
numParallelTests = runtime.NumCPU()
llvmVersion = llvm.Version{Major: 10}
- cacheDir = flag.String("cache", "cache", "path to the output cache directory")
- gerritEmail = flag.String("email", "$SS_REGRES_EMAIL", "gerrit email address for posting regres results")
- gerritUser = flag.String("user", "$SS_REGRES_USER", "gerrit username for posting regres results")
- gerritPass = flag.String("pass", "$SS_REGRES_PASS", "gerrit password for posting regres results")
- githubUser = flag.String("gh-user", "$SS_GITHUB_USER", "github user for posting coverage results")
- githubPass = flag.String("gh-pass", "$SS_GITHUB_PASS", "github password for posting coverage results")
- keepCheckouts = flag.Bool("keep", false, "don't delete checkout directories after use")
- dryRun = flag.Bool("dry", false, "don't post regres reports to gerrit")
- maxProcMemory = flag.Uint64("max-proc-mem", shell.MaxProcMemory, "maximum virtual memory per child process")
- dailyNow = flag.Bool("dailynow", false, "Start by running the daily pass")
- dailyOnly = flag.Bool("dailyonly", false, "Run only the daily pass")
- dailyChange = flag.String("dailychange", "", "Change hash to use for daily pass, HEAD if not provided")
- priority = flag.Int("priority", 0, "Prioritize a single change with the given number")
- limit = flag.Int("limit", 0, "only run a maximum of this number of tests")
+ cacheDir = flag.String("cache", "cache", "path to the output cache directory")
+ gerritEmail = flag.String("email", "$SS_REGRES_EMAIL", "gerrit email address for posting regres results")
+ gerritUser = flag.String("user", "$SS_REGRES_USER", "gerrit username for posting regres results")
+ gerritPass = flag.String("pass", "$SS_REGRES_PASS", "gerrit password for posting regres results")
+ githubUser = flag.String("gh-user", "$SS_GITHUB_USER", "github user for posting coverage results")
+ githubPass = flag.String("gh-pass", "$SS_GITHUB_PASS", "github password for posting coverage results")
+ keepCheckouts = flag.Bool("keep", false, "don't delete checkout directories after use")
+ dryRun = flag.Bool("dry", false, "don't post regres reports to gerrit")
+ maxTestsPerProc = flag.Int("max-tests-per-proc", 1, "maximum number of tests running in a single process")
+ maxProcMemory = flag.Uint64("max-proc-mem", shell.MaxProcMemory, "maximum virtual memory per child process")
+ dailyNow = flag.Bool("dailynow", false, "Start by running the daily pass")
+ dailyOnly = flag.Bool("dailyonly", false, "Run only the daily pass")
+ dailyChange = flag.String("dailychange", "", "Change hash to use for daily pass, HEAD if not provided")
+ priority = flag.Int("priority", 0, "Prioritize a single change with the given number")
+ limit = flag.Int("limit", 0, "only run a maximum of this number of tests")
)
func main() {
@@ -1362,6 +1363,7 @@
t.checkoutDir: "<SwiftShader>",
},
NumParallelTests: numParallelTests,
+ MaxTestsPerProc: *maxTestsPerProc,
TestTimeout: testTimeout,
CoverageEnv: t.coverageEnv,
}
diff --git a/tests/regres/cmd/run_testlist/main.go b/tests/regres/cmd/run_testlist/main.go
index 8674502..4150dfa 100644
--- a/tests/regres/cmd/run_testlist/main.go
+++ b/tests/regres/cmd/run_testlist/main.go
@@ -56,6 +56,7 @@
deqpVkBinary = flag.String("deqp-vk", "deqp-vk", "path to the deqp-vk binary")
testList = flag.String("test-list", "vk-master-PASS.txt", "path to a test list file")
numThreads = flag.Int("num-threads", min(runtime.NumCPU(), 100), "number of parallel test runner processes")
+ maxTestsPerProc = flag.Int("max-tests-per-proc", 1, "maximum number of tests running in a single process")
maxProcMemory = flag.Uint64("max-proc-mem", shell.MaxProcMemory, "maximum virtual memory per child process")
output = flag.String("output", "results.json", "path to an output JSON results file")
filter = flag.String("filter", "", "filter for test names. Start with a '/' to indicate regex")
@@ -110,6 +111,7 @@
ExeVulkan: *deqpVkBinary,
Env: os.Environ(),
NumParallelTests: *numThreads,
+ MaxTestsPerProc: *maxTestsPerProc,
TestLists: testlist.Lists{group},
TestTimeout: testTimeout,
ValidationLayer: *enableValidation,
diff --git a/tests/regres/deqp/deqp.go b/tests/regres/deqp/deqp.go
index 1e49a74..2946706 100644
--- a/tests/regres/deqp/deqp.go
+++ b/tests/regres/deqp/deqp.go
@@ -53,6 +53,8 @@
assertRE = regexp.MustCompile(`[^\n]*ASSERT\([^\)]*\)[^\n]*`)
// Regular expression to parse a test that failed due to ABORT()
abortRE = regexp.MustCompile(`[^\n]*ABORT:[^\n]*`)
+ // Regular expression to parse individual test names and output
+ caseOutputRE = regexp.MustCompile("Test case '([^']*)'..")
)
// Config contains the inputs required for running dEQP on a group of test lists.
@@ -66,6 +68,7 @@
Env []string
LogReplacements map[string]string
NumParallelTests int
+ MaxTestsPerProc int
CoverageEnv *cov.Env
TestTimeout time.Duration
ValidationLayer bool
@@ -312,13 +315,22 @@
logPath = filepath.Join(c.TempDir, fmt.Sprintf("%v.log", goroutineIndex))
}
+ testNames := []string{}
for name := range tests {
- results <- c.PerformTest(exe, env, coverageFile, logPath, name, supportsCoverage)
+ testNames = append(testNames, name)
+ if len(testNames) >= c.MaxTestsPerProc {
+ c.PerformTests(exe, env, coverageFile, logPath, testNames, supportsCoverage, results)
+ // Clear list of test names
+ testNames = testNames[:0]
+ }
+ }
+ if len(testNames) > 0 {
+ c.PerformTests(exe, env, coverageFile, logPath, testNames, supportsCoverage, results)
}
}
-func (c *Config) PerformTest(exe string, env []string, coverageFile string, logPath string, name string, supportsCoverage bool) TestResult {
- // log.Printf("Running test '%s'\n", name)
+func (c *Config) PerformTests(exe string, env []string, coverageFile string, logPath string, testNames []string, supportsCoverage bool, results chan<- TestResult) {
+ // log.Printf("Running test(s) '%s'\n", testNames)
start := time.Now()
// Set validation layer according to flag.
@@ -328,9 +340,11 @@
}
// The list of test names will be passed to stdin, since the deqp-stdin-caselist option is used
- testNames := name + "\n"
+ stdin := strings.Join(testNames, "\n") + "\n"
- outRaw, err := shell.Exec(c.TestTimeout, exe, filepath.Dir(exe), env, testNames,
+ numTests := len(testNames)
+ timeout := c.TestTimeout * time.Duration(numTests)
+ outRaw, err := shell.Exec(timeout, exe, filepath.Dir(exe), env, stdin,
"--deqp-validation="+validation,
"--deqp-surface-type=pbuffer",
"--deqp-shadercache=disable",
@@ -352,11 +366,41 @@
if c.CoverageEnv != nil && supportsCoverage {
coverage, err = c.CoverageEnv.Import(coverageFile)
if err != nil {
- log.Printf("Warning: Failed to process test coverage for test '%v'. %v", name, err)
+ log.Printf("Warning: Failed to process test coverage for test '%v'. %v", testNames, err)
}
os.Remove(coverageFile)
}
+ if numTests > 1 {
+ // Separate output per test case
+ caseOutputs := caseOutputRE.Split(out, -1)
+
+ // If the output isn't as expected, a crash may have happened, so re-run tests separately
+ if len(caseOutputs) != (numTests + 1) {
+ // Re-run tests one by one
+ for _, testName := range testNames {
+ singleTest := []string{testName}
+ c.PerformTests(exe, env, coverageFile, logPath, singleTest, supportsCoverage, results)
+ }
+ } else {
+ caseOutputs = caseOutputs[1:] // Ignore text up to first "Test case '...'"
+ caseNameMatches := caseOutputRE.FindAllStringSubmatch(out, -1)
+ caseNames := make([]string, len(caseNameMatches))
+ for i, m := range caseNameMatches {
+ caseNames[i] = m[1]
+ }
+
+ averageDuration := duration / time.Duration(numTests)
+ for i, caseOutput := range caseOutputs {
+ results <- c.AnalyzeOutput(caseNames[i], caseOutput, averageDuration, coverage)
+ }
+ }
+ } else {
+ results <- c.AnalyzeOutput(testNames[0], out, duration, coverage)
+ }
+}
+
+func (c *Config) AnalyzeOutput(name string, out string, duration time.Duration, coverage *cov.Coverage) TestResult {
for _, test := range []struct {
re *regexp.Regexp
s testlist.Status
@@ -379,6 +423,7 @@
}
// Don't treat non-zero error codes as crashes.
+ var err error
var exitErr *exec.ExitError
if errors.As(err, &exitErr) {
if exitErr.ExitCode() != 255 {
