Refactor test parameters to TracePerfTest. Reduces much of the code duplication by adding some new helper methods. The helpers allow combining test parameters similarly to how the GoogleTest Combine() and Values() generators work. They are more general and work by returning collections of test parameters instead of combining generator functions. Also updates the GLMark2 benchmark runner to use the new methods. Bug: angleproject:3630 Change-Id: Ibc10f9afb401e119d67a7119974a1a8d9b5abb60 Reviewed-on: https://chromium-review.googlesource.com/c/angle/angle/+/2057353 Commit-Queue: Jamie Madill <jmadill@chromium.org> Reviewed-by: Yuly Novikov <ynovikov@chromium.org> Reviewed-by: Cody Northrop <cnorthrop@google.com>
diff --git a/src/tests/perf_tests/ANGLEPerfTest.h b/src/tests/perf_tests/ANGLEPerfTest.h index 50ea187..5ca2466 100644 --- a/src/tests/perf_tests/ANGLEPerfTest.h +++ b/src/tests/perf_tests/ANGLEPerfTest.h
@@ -195,5 +195,4 @@ return output; } } // namespace params - #endif // PERF_TESTS_ANGLE_PERF_TEST_H_
diff --git a/src/tests/perf_tests/TracePerfTest.cpp b/src/tests/perf_tests/TracePerfTest.cpp index 04b9f5d..c2c3510 100644 --- a/src/tests/perf_tests/TracePerfTest.cpp +++ b/src/tests/perf_tests/TracePerfTest.cpp
@@ -8,6 +8,7 @@ // #include <gtest/gtest.h> +#include "common/PackedEnums.h" #include "common/system_utils.h" #include "tests/perf_tests/ANGLEPerfTest.h" #include "util/egl_loader_autogen.h" @@ -33,6 +34,7 @@ TRex800, TRex900, TRex1300, + InvalidEnum, }; struct TracePerfParams final : public RenderTestParams @@ -167,83 +169,43 @@ stopGpuTimer(); } -TracePerfParams TRexReplayPerfOpenGLOrGLESParams_200_210() -{ - TracePerfParams params; - params.eglParameters = OPENGL_OR_GLES(); - params.testID = TracePerfTestID::TRex200; - return params; -} - -TracePerfParams TRexReplayPerfOpenGLOrGLESParams_800_810() -{ - TracePerfParams params; - params.eglParameters = OPENGL_OR_GLES(); - params.testID = TracePerfTestID::TRex800; - return params; -} - -TracePerfParams TRexReplayPerfOpenGLOrGLESParams_900_910() -{ - TracePerfParams params; - params.eglParameters = OPENGL_OR_GLES(); - params.testID = TracePerfTestID::TRex900; - return params; -} - -TracePerfParams TRexReplayPerfOpenGLOrGLESParams_1300_1310() -{ - TracePerfParams params; - params.eglParameters = OPENGL_OR_GLES(); - params.testID = TracePerfTestID::TRex1300; - return params; -} - -TracePerfParams TRexReplayPerfVulkanParams_200_210() -{ - TracePerfParams params; - params.eglParameters = VULKAN(); - params.testID = TracePerfTestID::TRex200; - return params; -} - -TracePerfParams TRexReplayPerfVulkanParams_800_810() -{ - TracePerfParams params; - params.eglParameters = VULKAN(); - params.testID = TracePerfTestID::TRex800; - return params; -} - -TracePerfParams TRexReplayPerfVulkanParams_900_910() -{ - TracePerfParams params; - params.eglParameters = VULKAN(); - params.testID = TracePerfTestID::TRex900; - return params; -} - -TracePerfParams TRexReplayPerfVulkanParams_1300_1310() -{ - TracePerfParams params; - params.eglParameters = VULKAN(); - params.testID = TracePerfTestID::TRex1300; - return params; -} - TEST_P(TracePerfTest, Run) { run(); } -ANGLE_INSTANTIATE_TEST(TracePerfTest, - TRexReplayPerfOpenGLOrGLESParams_200_210(), - TRexReplayPerfOpenGLOrGLESParams_800_810(), - TRexReplayPerfOpenGLOrGLESParams_900_910(), - TRexReplayPerfOpenGLOrGLESParams_1300_1310(), - TRexReplayPerfVulkanParams_200_210(), - TRexReplayPerfVulkanParams_800_810(), - TRexReplayPerfVulkanParams_900_910(), - TRexReplayPerfVulkanParams_1300_1310()); +TracePerfParams GL(const TracePerfParams &in) +{ + TracePerfParams out = in; + out.eglParameters = OPENGL_OR_GLES(); + return out; +} + +TracePerfParams Vulkan(const TracePerfParams &in) +{ + TracePerfParams out = in; + out.eglParameters = VULKAN(); + return out; +} + +// Note: WGL replay currently broken because interface locations are not remapped. +ANGLE_MAYBE_UNUSED TracePerfParams WGL(const TracePerfParams &in) +{ + TracePerfParams out = in; + out.driver = angle::GLESDriverType::SystemWGL; + return out; +} + +TracePerfParams CombineTestID(const TracePerfParams &in, TracePerfTestID id) +{ + TracePerfParams out = in; + out.testID = id; + return out; +} + +std::vector<TracePerfParams> gTestsWithID = + CombineWithValues({TracePerfParams()}, AllEnums<TracePerfTestID>(), CombineTestID); +std::vector<TracePerfParams> gTestsWithRenderer = CombineWithFuncs(gTestsWithID, {GL, Vulkan}); +ANGLE_INSTANTIATE_TEST_ARRAY(TracePerfTest, gTestsWithRenderer); } // anonymous namespace
diff --git a/src/tests/perf_tests/glmark2Benchmark.cpp b/src/tests/perf_tests/glmark2Benchmark.cpp index 174fcc6..013bb24 100644 --- a/src/tests/perf_tests/glmark2Benchmark.cpp +++ b/src/tests/perf_tests/glmark2Benchmark.cpp
@@ -77,25 +77,23 @@ {"loop:fragment-steps=5:fragment-uniform=true:vertex-steps=5", "loop"}, }; -using GLMark2BenchmarkTestParams = std::tuple<PlatformParameters, size_t>; -std::string GLMark2BenchmarkPrint( - const ::testing::TestParamInfo<GLMark2BenchmarkTestParams> ¶msInfo) +struct GLMark2TestParams : public PlatformParameters { - const GLMark2BenchmarkTestParams ¶ms = paramsInfo.param; - std::ostringstream out; + BenchmarkInfo info; +}; - out << std::get<0>(params) << '_'; - out << kBenchmarks[std::get<1>(params)].name; - - return out.str(); +std::ostream &operator<<(std::ostream &os, const GLMark2TestParams ¶ms) +{ + os << static_cast<const PlatformParameters &>(params) << "_" << params.info.name; + return os; } -class GLMark2Benchmark : public testing::TestWithParam<GLMark2BenchmarkTestParams> +class GLMark2Benchmark : public testing::TestWithParam<GLMark2TestParams> { public: GLMark2Benchmark() { - switch (std::get<0>(GetParam()).getRenderer()) + switch (GetParam().getRenderer()) { case EGL_PLATFORM_ANGLE_TYPE_D3D11_ANGLE: mBackend = "d3d11"; @@ -109,7 +107,7 @@ default: break; } - std::string story = kBenchmarks[std::get<1>(GetParam())].name; + std::string story = GetParam().info.name; mReporter = std::make_unique<perf_test::PerfResultReporter>("glmark2_" + mBackend, story); mReporter->RegisterImportantMetric(".fps", "fps"); mReporter->RegisterImportantMetric(".score", "score"); @@ -123,7 +121,7 @@ return; } - const BenchmarkInfo benchmarkInfo = kBenchmarks[std::get<1>(GetParam())]; + const BenchmarkInfo benchmarkInfo = GetParam().info; const char *benchmark = benchmarkInfo.glmark2Config; const char *benchmarkName = benchmarkInfo.name; bool completeRun = benchmark == nullptr || benchmark[0] == '\0'; @@ -275,11 +273,28 @@ run(); } -ANGLE_INSTANTIATE_TEST_COMBINE_1(GLMark2Benchmark, - GLMark2BenchmarkPrint, - testing::Range(static_cast<size_t>(0), ArraySize(kBenchmarks)), - ES2_D3D11(), - ES2_OPENGLES(), - ES2_VULKAN()); +GLMark2TestParams CombineEGLPlatform(const GLMark2TestParams &in, EGLPlatformParameters eglParams) +{ + GLMark2TestParams out = in; + out.eglParameters = eglParams; + return out; +} + +GLMark2TestParams CombineInfo(const GLMark2TestParams &in, BenchmarkInfo info) +{ + GLMark2TestParams out = in; + out.info = info; + return out; +} + +using namespace egl_platform; + +std::vector<GLMark2TestParams> gTestsWithInfo = + CombineWithValues({GLMark2TestParams()}, kBenchmarks, CombineInfo); +std::vector<EGLPlatformParameters> gEGLPlatforms = {D3D11(), OPENGLES(), VULKAN()}; +std::vector<GLMark2TestParams> gTestsWithPlatform = + CombineWithValues(gTestsWithInfo, gEGLPlatforms, CombineEGLPlatform); + +ANGLE_INSTANTIATE_TEST_ARRAY(GLMark2Benchmark, gTestsWithPlatform); } // namespace
diff --git a/src/tests/test_utils/angle_test_instantiate.h b/src/tests/test_utils/angle_test_instantiate.h index 3d8e866..eb83d90 100644 --- a/src/tests/test_utils/angle_test_instantiate.h +++ b/src/tests/test_utils/angle_test_instantiate.h
@@ -113,6 +113,10 @@ INSTANTIATE_TEST_SUITE_P(, testName, ANGLE_INSTANTIATE_TEST_PLATFORMS(testName), \ testing::PrintToStringParamName()) +#define ANGLE_INSTANTIATE_TEST_ARRAY(testName, valuesin) \ + INSTANTIATE_TEST_SUITE_P(, testName, testing::ValuesIn(::angle::FilterTestParams(valuesin)), \ + testing::PrintToStringParamName()) + #define ANGLE_ALL_TEST_PLATFORMS_ES1 \ ES1_D3D11(), ES1_OPENGL(), ES1_OPENGLES(), ES1_VULKAN(), ES1_VULKAN_SWIFTSHADER() @@ -185,13 +189,10 @@ INSTANTIATE_TEST_SUITE_P(, testName, ANGLE_INSTANTIATE_TEST_PLATFORMS(testName), \ testing::PrintToStringParamName()) -// Instantiate the test for a combination of N parameters and the enumeration of platforms in the -// extra args, similar to ANGLE_INSTANTIATE_TEST. The macros are defined only for the Ns currently -// in use, and can be expanded as necessary. -#define ANGLE_INSTANTIATE_TEST_COMBINE_1(testName, print, combine1, first, ...) \ - const decltype(first) testName##params[] = {first, ##__VA_ARGS__}; \ - INSTANTIATE_TEST_SUITE_P( \ - , testName, testing::Combine(ANGLE_INSTANTIATE_TEST_PLATFORMS(testName), combine1), print) +// Instantiate the test for a combination of N parameters and the +// enumeration of platforms in the extra args, similar to +// ANGLE_INSTANTIATE_TEST. The macros are defined only for the Ns +// currently in use, and can be expanded as necessary. #define ANGLE_INSTANTIATE_TEST_COMBINE_4(testName, print, combine1, combine2, combine3, combine4, \ first, ...) \ const decltype(first) testName##params[] = {first, ##__VA_ARGS__}; \ @@ -210,21 +211,60 @@ // Checks if a config is expected to be supported by checking a system-based white list. bool IsConfigWhitelisted(const SystemInfo &systemInfo, const PlatformParameters ¶m); -// Determines if a config is supported by trying to initialize it. Does not require SystemInfo. +// Determines if a config is supported by trying to initialize it. Does +// not require SystemInfo. bool IsConfigSupported(const PlatformParameters ¶m); -// Returns shared test system information. Can be used globally in the tests. +// Returns shared test system information. Can be used globally in the +// tests. SystemInfo *GetTestSystemInfo(); -// Returns a list of all enabled test platform names. For use in configuration enumeration. +// Returns a list of all enabled test platform names. For use in +// configuration enumeration. std::vector<std::string> GetAvailableTestPlatformNames(); // Active config (e.g. ES2_Vulkan). extern std::string gSelectedConfig; -// Use a separate isolated process per test config. This works around driver flakiness when using -// multiple APIs/windows/etc in the same process. +// Use a separate isolated process per test config. This works around +// driver flakiness when using multiple APIs/windows/etc in the same +// process. extern bool gSeparateProcessPerConfig; + +// For use with ANGLE_INSTANTIATE_TEST_ARRAY +template <typename ParamsT> +using ModifierFunc = std::function<ParamsT(const ParamsT &)>; + +template <typename ParamsT> +std::vector<ParamsT> CombineWithFuncs(const std::vector<ParamsT> &in, + const std::vector<ModifierFunc<ParamsT>> &modifiers) +{ + std::vector<ParamsT> out; + for (const ParamsT ¶msIn : in) + { + for (ModifierFunc<ParamsT> modifier : modifiers) + { + out.push_back(modifier(paramsIn)); + } + } + return out; +} + +template <typename ParamT, typename ModifiersT, typename ModifierT> +std::vector<ParamT> CombineWithValues(const std::vector<ParamT> &in, + const ModifiersT &modifiers, + ParamT combine(const ParamT &, ModifierT)) +{ + std::vector<ParamT> out; + for (const ParamT ¶msIn : in) + { + for (ModifierT modifier : modifiers) + { + out.push_back(combine(paramsIn, modifier)); + } + } + return out; +} } // namespace angle #define ANGLE_SKIP_TEST_IF(COND) \