Relax present time comparison in get_frame_timestamps am: 24e5494e43 am: e49fda884d Original change: https://android-review.googlesource.com/c/platform/external/deqp/+/3542313 Change-Id: Ia17e77ad8b31c5e6d683eb0bcd408df75c553caf Signed-off-by: Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>
diff --git a/Android.bp b/Android.bp index 4e9c74f..de4e633 100644 --- a/Android.bp +++ b/Android.bp
@@ -1,4 +1,5 @@ package { + default_team: "trendy_team_android_gpu", default_applicable_licenses: ["external_deqp_license"], } @@ -35,7 +36,10 @@ ], } -build = ["AndroidGen.bp"] +build = [ + "AndroidGen.bp", + "AndroidKhronosCTSGen.bp", +] // Used by Amber. // Amber includes "vkDefs.h". @@ -281,6 +285,97 @@ ], } +cc_library_shared { + name: "libkhronosopenglcts", + defaults: ["khronoscts_default"], + + srcs: [ + "framework/platform/android/tcuAndroidMain.cpp", + "framework/platform/android/tcuAndroidJNI.cpp", + "framework/platform/android/tcuAndroidPlatformCapabilityQueryJNI.cpp", + "framework/platform/android/tcuTestLogParserJNI.cpp", + "external/openglcts/modules/runner/glcAndroidMain.cpp", + "external/openglcts/modules/glcTestPackageEntry.cpp", + "modules/gles2/tes2TestPackageEntry.cpp", + "modules/gles3/tes3TestPackageEntry.cpp", + "modules/gles31/tes31TestPackageEntry.cpp", + "modules/egl/teglTestPackageEntry.cpp", + "modules/internal/ditTestPackageEntry.cpp", + ], + + local_include_dirs: [ + "external/openglcts/modules/runner", + "external/openglcts/modules", + "framework/platform/android", + "modules/gles2", + "modules/gles3", + "modules/gles31", + "modules/egl", + "modules/internal", + ], + + static_libs: [ + "libkhronoscts_common", + "libkhronoscts_modules_gles", + "libkhronoscts_openglcts", + "libkhronoscts_vulkancts", + "libkhronoscts_platform", + ], +} + +cc_defaults { + name: "khronoscts_default", + + defaults: [ + "khronosctscompilationflag_default", + ], + + shared_libs: [ + "libEGL", + "libGLESv2", + "libandroid", + "liblog", + "libm", + "libc", + "libz", + "libdl", + ], + + static_libs: [ + "libpng_ndk", + "deqp_glslang_glslang", + "deqp_glslang_OSDependent", + "deqp_glslang_MachineIndependent", + "deqp_glslang_GenericCodeGen", + "deqp_glslang_SPIRV", + "deqp_glslang_SPVRemapper", + "deqp_spirv-tools", + "deqp_amber", + ], +} + +android_test { + name: "org.khronos.gl_cts", + + srcs: ["android/openglcts/src/**/*.java"], + manifest: "android/openglcts/AndroidManifest.xml", + + asset_dirs: [ + "data", + "external/openglcts/data/", + "external/graphicsfuzz/data", + "external/vulkancts/data", + ], + + jni_libs: ["libkhronosopenglcts"], + compile_multilib: "both", + + sdk_version: "test_current", + + min_sdk_version: "31", + target_sdk_version: "34", +} + filegroup { name: "deqp_binary_incremental_test_lists", srcs: [ @@ -312,3 +407,11 @@ ], path: "external/graphicsfuzz/data", } + +filegroup { + name: "khronos_cts_gles_caselists", + srcs: [ + "external/openglcts/data/gl_cts/data/mustpass/**/*.txt", + ], + path: "external/openglcts/data/", +}
diff --git a/OWNERS b/OWNERS index 63a689a..b1c2142 100644 --- a/OWNERS +++ b/OWNERS
@@ -2,4 +2,4 @@ chrisforbes@google.com tomnom@google.com ianelliott@google.com -nexa@google.com +include platform/system/core:/janitors/OWNERS #{LAST_RESORT_SUGGESTION}
diff --git a/android/cts/Android.bp b/android/cts/Android.bp index e3749f9..82d6102 100644 --- a/android/cts/Android.bp +++ b/android/cts/Android.bp
@@ -58,14 +58,27 @@ per_testcase_directory: true, data: [ - ":com.drawelements.deqp", ":deqp_binary_data", ":deqp_binary_data_vulkancts", ":deqp_binary_data_graphicsfuzz", ":deqp_main_caselists", ":deqp_angle_exclude_caselists", ], + device_common_data: [ + ":com.drawelements.deqp", + ], data_device_bins_both: [ "deqp-binary", ], } + +java_library_host { + name: "CtsDeqpTestCasesJavaHostLib", + srcs: ["runner/src/**/*.java"], + libs: [ + "cts-tradefed", + "compatibility-tradefed", + "compatibility-host-util", + "tradefed", + ], +}
diff --git a/android/cts/AndroidTest.xml b/android/cts/AndroidTest.xml index 825b6b9..6df4f9e 100644 --- a/android/cts/AndroidTest.xml +++ b/android/cts/AndroidTest.xml
@@ -89,11 +89,11 @@ </test> <test class="com.drawelements.deqp.runner.DeqpTestRunner"> <option name="deqp-package" value="dEQP-EGL"/> - <option name="deqp-caselist-file" value="egl-main-risky.txt"/> + <option name="deqp-caselist-file" value="egl-main-2025-03-01.txt"/> <option name="deqp-gl-config-name" value="rgba8888d24s8ms0"/> <option name="deqp-surface-type" value="window"/> <option name="deqp-screen-rotation" value="unspecified"/> - <option name="runtime-hint" value="2m"/> + <option name="runtime-hint" value="5m"/> <option name="deqp-config-required" value="true"/> </test> <test class="com.drawelements.deqp.runner.DeqpTestRunner"> @@ -142,6 +142,15 @@ <option name="deqp-config-required" value="true"/> </test> <test class="com.drawelements.deqp.runner.DeqpTestRunner"> + <option name="deqp-package" value="dEQP-GLES2"/> + <option name="deqp-caselist-file" value="gles2-main-2025-03-01.txt"/> + <option name="deqp-gl-config-name" value="rgba8888d24s8ms0"/> + <option name="deqp-surface-type" value="window"/> + <option name="deqp-screen-rotation" value="unspecified"/> + <option name="runtime-hint" value="10m"/> + <option name="deqp-config-required" value="true"/> + </test> + <test class="com.drawelements.deqp.runner.DeqpTestRunner"> <option name="deqp-package" value="dEQP-GLES3"/> <option name="deqp-caselist-file" value="gles3-main-2020-03-01.txt"/> <option name="incremental-deqp-include-file" value="gles3-incremental-deqp.txt"/> @@ -193,6 +202,16 @@ </test> <test class="com.drawelements.deqp.runner.DeqpTestRunner"> <option name="deqp-package" value="dEQP-GLES3"/> + <option name="deqp-caselist-file" value="gles3-main-2025-03-01.txt"/> + <option name="incremental-deqp-include-file" value="gles3-incremental-deqp.txt"/> + <option name="deqp-gl-config-name" value="rgba8888d24s8ms0"/> + <option name="deqp-surface-type" value="window"/> + <option name="deqp-screen-rotation" value="unspecified"/> + <option name="runtime-hint" value="10m"/> + <option name="deqp-config-required" value="true"/> + </test> + <test class="com.drawelements.deqp.runner.DeqpTestRunner"> + <option name="deqp-package" value="dEQP-GLES3"/> <option name="deqp-caselist-file" value="gles3-rotate-portrait.txt"/> <option name="incremental-deqp-include-file" value="gles3-incremental-deqp.txt"/> <option name="deqp-gl-config-name" value="rgba8888d24s8ms0"/> @@ -297,6 +316,16 @@ </test> <test class="com.drawelements.deqp.runner.DeqpTestRunner"> <option name="deqp-package" value="dEQP-GLES31"/> + <option name="deqp-caselist-file" value="gles31-main-2025-03-01.txt"/> + <option name="incremental-deqp-include-file" value="gles3-incremental-deqp.txt"/> + <option name="deqp-gl-config-name" value="rgba8888d24s8ms0"/> + <option name="deqp-surface-type" value="window"/> + <option name="deqp-screen-rotation" value="unspecified"/> + <option name="runtime-hint" value="10m"/> + <option name="deqp-config-required" value="true"/> + </test> + <test class="com.drawelements.deqp.runner.DeqpTestRunner"> + <option name="deqp-package" value="dEQP-GLES31"/> <option name="deqp-caselist-file" value="gles31-rotate-portrait.txt"/> <option name="incremental-deqp-include-file" value="gles3-incremental-deqp.txt"/> <option name="deqp-gl-config-name" value="rgba8888d24s8ms0"/> @@ -385,4 +414,10 @@ <option name="incremental-deqp-include-file" value="vk-incremental-deqp.txt"/> <option name="runtime-hint" value="10m"/> </test> + <test class="com.drawelements.deqp.runner.DeqpTestRunner"> + <option name="deqp-package" value="dEQP-VK"/> + <option name="deqp-caselist-file" value="vk-main-2025-03-01.txt"/> + <option name="incremental-deqp-include-file" value="vk-incremental-deqp.txt"/> + <option name="runtime-hint" value="10m"/> + </test> </configuration>
diff --git a/android/cts/main/egl-main-risky.txt b/android/cts/main/egl-main-2025-03-01.txt similarity index 100% copy from android/cts/main/egl-main-risky.txt copy to android/cts/main/egl-main-2025-03-01.txt
diff --git a/android/cts/main/egl-main-risky.txt b/android/cts/main/gles2-main-2025-03-01.txt similarity index 100% copy from android/cts/main/egl-main-risky.txt copy to android/cts/main/gles2-main-2025-03-01.txt
diff --git a/android/cts/main/egl-main-risky.txt b/android/cts/main/gles3-main-2025-03-01.txt similarity index 100% copy from android/cts/main/egl-main-risky.txt copy to android/cts/main/gles3-main-2025-03-01.txt
diff --git a/android/cts/main/egl-main-risky.txt b/android/cts/main/gles31-main-2025-03-01.txt similarity index 100% copy from android/cts/main/egl-main-risky.txt copy to android/cts/main/gles31-main-2025-03-01.txt
diff --git a/android/cts/main/mustpass.xml b/android/cts/main/mustpass.xml index fa3e9bd..3d63d98 100644 --- a/android/cts/main/mustpass.xml +++ b/android/cts/main/mustpass.xml
@@ -23,7 +23,7 @@ <Configuration caseListFile="egl-main-2022-03-01.txt" commandLine="--deqp-gl-config-name=rgba8888d24s8ms0 --deqp-screen-rotation=unspecified --deqp-surface-type=window --deqp-watchdog=enable" name="main-2022-03-01"/> <Configuration caseListFile="egl-main-2023-03-01.txt" commandLine="--deqp-gl-config-name=rgba8888d24s8ms0 --deqp-screen-rotation=unspecified --deqp-surface-type=window --deqp-watchdog=enable" name="main-2023-03-01"/> <Configuration caseListFile="egl-main-2024-03-01.txt" commandLine="--deqp-gl-config-name=rgba8888d24s8ms0 --deqp-screen-rotation=unspecified --deqp-surface-type=window --deqp-watchdog=enable" name="main-2024-03-01"/> - <Configuration caseListFile="egl-main-risky.txt" commandLine="--deqp-gl-config-name=rgba8888d24s8ms0 --deqp-screen-rotation=unspecified --deqp-surface-type=window --deqp-watchdog=enable" name="main-risky"/> + <Configuration caseListFile="egl-main-2025-03-01.txt" commandLine="--deqp-gl-config-name=rgba8888d24s8ms0 --deqp-screen-rotation=unspecified --deqp-surface-type=window --deqp-watchdog=enable" name="main-2025-03-01"/> </TestPackage> <TestPackage name="dEQP-GLES2"> <Configuration caseListFile="gles2-main-2020-03-01.txt" commandLine="--deqp-gl-config-name=rgba8888d24s8ms0 --deqp-screen-rotation=unspecified --deqp-surface-type=window --deqp-watchdog=enable" name="main-2020-03-01"/> @@ -31,6 +31,7 @@ <Configuration caseListFile="gles2-main-2022-03-01.txt" commandLine="--deqp-gl-config-name=rgba8888d24s8ms0 --deqp-screen-rotation=unspecified --deqp-surface-type=window --deqp-watchdog=enable" name="main-2022-03-01"/> <Configuration caseListFile="gles2-main-2023-03-01.txt" commandLine="--deqp-gl-config-name=rgba8888d24s8ms0 --deqp-screen-rotation=unspecified --deqp-surface-type=window --deqp-watchdog=enable" name="main-2023-03-01"/> <Configuration caseListFile="gles2-main-2024-03-01.txt" commandLine="--deqp-gl-config-name=rgba8888d24s8ms0 --deqp-screen-rotation=unspecified --deqp-surface-type=window --deqp-watchdog=enable" name="main-2024-03-01"/> + <Configuration caseListFile="gles2-main-2025-03-01.txt" commandLine="--deqp-gl-config-name=rgba8888d24s8ms0 --deqp-screen-rotation=unspecified --deqp-surface-type=window --deqp-watchdog=enable" name="main-2025-03-01"/> <Configuration caseListFile="gles2-incremental-deqp-baseline.txt" commandLine="--deqp-watchdog=enable" name="incremental-deqp-baseline"/> </TestPackage> <TestPackage name="dEQP-GLES3"> @@ -39,6 +40,7 @@ <Configuration caseListFile="gles3-main-2022-03-01.txt" commandLine="--deqp-gl-config-name=rgba8888d24s8ms0 --deqp-screen-rotation=unspecified --deqp-surface-type=window --deqp-watchdog=enable" name="main-2022-03-01"/> <Configuration caseListFile="gles3-main-2023-03-01.txt" commandLine="--deqp-gl-config-name=rgba8888d24s8ms0 --deqp-screen-rotation=unspecified --deqp-surface-type=window --deqp-watchdog=enable" name="main-2023-03-01"/> <Configuration caseListFile="gles3-main-2024-03-01.txt" commandLine="--deqp-gl-config-name=rgba8888d24s8ms0 --deqp-screen-rotation=unspecified --deqp-surface-type=window --deqp-watchdog=enable" name="main-2024-03-01"/> + <Configuration caseListFile="gles3-main-2025-03-01.txt" commandLine="--deqp-gl-config-name=rgba8888d24s8ms0 --deqp-screen-rotation=unspecified --deqp-surface-type=window --deqp-watchdog=enable" name="main-2025-03-01"/> <Configuration caseListFile="gles3-rotate-portrait.txt" commandLine="--deqp-gl-config-name=rgba8888d24s8ms0 --deqp-screen-rotation=0 --deqp-surface-type=window --deqp-watchdog=enable" name="rotate-portrait"/> <Configuration caseListFile="gles3-rotate-landscape.txt" commandLine="--deqp-gl-config-name=rgba8888d24s8ms0 --deqp-screen-rotation=90 --deqp-surface-type=window --deqp-watchdog=enable" name="rotate-landscape"/> <Configuration caseListFile="gles3-rotate-reverse-portrait.txt" commandLine="--deqp-gl-config-name=rgba8888d24s8ms0 --deqp-screen-rotation=180 --deqp-surface-type=window --deqp-watchdog=enable" name="rotate-reverse-portrait"/> @@ -54,6 +56,7 @@ <Configuration caseListFile="gles31-main-2022-03-01.txt" commandLine="--deqp-gl-config-name=rgba8888d24s8ms0 --deqp-screen-rotation=unspecified --deqp-surface-type=window --deqp-watchdog=enable" name="main-2022-03-01"/> <Configuration caseListFile="gles31-main-2023-03-01.txt" commandLine="--deqp-gl-config-name=rgba8888d24s8ms0 --deqp-screen-rotation=unspecified --deqp-surface-type=window --deqp-watchdog=enable" name="main-2023-03-01"/> <Configuration caseListFile="gles31-main-2024-03-01.txt" commandLine="--deqp-gl-config-name=rgba8888d24s8ms0 --deqp-screen-rotation=unspecified --deqp-surface-type=window --deqp-watchdog=enable" name="main-2024-03-01"/> + <Configuration caseListFile="gles31-main-2025-03-01.txt" commandLine="--deqp-gl-config-name=rgba8888d24s8ms0 --deqp-screen-rotation=unspecified --deqp-surface-type=window --deqp-watchdog=enable" name="main-2025-03-01"/> <Configuration caseListFile="gles31-rotate-portrait.txt" commandLine="--deqp-gl-config-name=rgba8888d24s8ms0 --deqp-screen-rotation=0 --deqp-surface-type=window --deqp-watchdog=enable" name="rotate-portrait"/> <Configuration caseListFile="gles31-rotate-landscape.txt" commandLine="--deqp-gl-config-name=rgba8888d24s8ms0 --deqp-screen-rotation=90 --deqp-surface-type=window --deqp-watchdog=enable" name="rotate-landscape"/> <Configuration caseListFile="gles31-rotate-reverse-portrait.txt" commandLine="--deqp-gl-config-name=rgba8888d24s8ms0 --deqp-screen-rotation=180 --deqp-surface-type=window --deqp-watchdog=enable" name="rotate-reverse-portrait"/> @@ -69,6 +72,7 @@ <Configuration caseListFile="vk-main-2022-03-01.txt" commandLine="--deqp-watchdog=enable" name="main-2022-03-01"/> <Configuration caseListFile="vk-main-2023-03-01.txt" commandLine="--deqp-watchdog=enable" name="main-2023-03-01"/> <Configuration caseListFile="vk-main-2024-03-01.txt" commandLine="--deqp-watchdog=enable" name="main-2024-03-01"/> + <Configuration caseListFile="vk-main-2025-03-01.txt" commandLine="--deqp-watchdog=enable" name="main-2025-03-01"/> <Configuration caseListFile="vk-incremental-deqp.txt" commandLine="--deqp-watchdog=enable" name="incremental-deqp"/> <Configuration caseListFile="vk-incremental-deqp-baseline.txt" commandLine="--deqp-watchdog=enable" name="incremental-deqp-baseline"/> </TestPackage>
diff --git a/android/cts/main/src/egl-main-2024-03-01.txt b/android/cts/main/src/egl-main-2024-03-01.txt new file mode 100644 index 0000000..a50aacf --- /dev/null +++ b/android/cts/main/src/egl-main-2024-03-01.txt
@@ -0,0 +1,23 @@ +dEQP-EGL.functional.choose_config.simple.selection_and_sort.recordable_android +dEQP-EGL.functional.choose_config.simple.selection_only.recordable_android +dEQP-EGL.functional.fence_sync.valid.egl_fence_persistent_buffer +dEQP-EGL.functional.get_proc_address.extension.egl_angle_sync_control_rate +dEQP-EGL.functional.get_proc_address.extension.egl_ext_device_persistent_id +dEQP-EGL.functional.get_proc_address.extension.egl_ext_surface_compression +dEQP-EGL.functional.get_proc_address.extension.egl_mesa_query_driver +dEQP-EGL.functional.get_proc_address.extension.egl_nv_stream_consumer_eglimage +dEQP-EGL.functional.get_proc_address.extension.egl_wl_bind_wayland_display +dEQP-EGL.functional.get_proc_address.extension.egl_wl_create_wayland_buffer_from_image +dEQP-EGL.functional.query_config.get_config_attrib.recordable_android +dEQP-EGL.functional.wide_color.pbuffer_1010102_colorspace_bt2020_hlg +dEQP-EGL.functional.wide_color.pbuffer_1010102_colorspace_bt2020_linear +dEQP-EGL.functional.wide_color.pbuffer_1010102_colorspace_bt2020_pq +dEQP-EGL.functional.wide_color.pbuffer_fp16_colorspace_bt2020_hlg +dEQP-EGL.functional.wide_color.pbuffer_fp16_colorspace_bt2020_linear +dEQP-EGL.functional.wide_color.pbuffer_fp16_colorspace_bt2020_pq +dEQP-EGL.functional.wide_color.window_1010102_colorspace_bt2020_hlg +dEQP-EGL.functional.wide_color.window_1010102_colorspace_bt2020_linear +dEQP-EGL.functional.wide_color.window_1010102_colorspace_bt2020_pq +dEQP-EGL.functional.wide_color.window_fp16_colorspace_bt2020_hlg +dEQP-EGL.functional.wide_color.window_fp16_colorspace_bt2020_linear +dEQP-EGL.functional.wide_color.window_fp16_colorspace_bt2020_pq
diff --git a/android/cts/main/src/egl-temp-excluded.txt b/android/cts/main/src/egl-temp-excluded.txt deleted file mode 100644 index 2974117..0000000 --- a/android/cts/main/src/egl-temp-excluded.txt +++ /dev/null
@@ -1 +0,0 @@ -# Tests to be temporarily skipped for Android CI, but still enforced in CTS.
diff --git a/android/cts/main/egl-main-risky.txt b/android/cts/main/src/gles2-main-2024-03-01.txt similarity index 100% copy from android/cts/main/egl-main-risky.txt copy to android/cts/main/src/gles2-main-2024-03-01.txt
diff --git a/android/cts/main/src/gles2-temp-excluded.txt b/android/cts/main/src/gles2-temp-excluded.txt deleted file mode 100644 index 2974117..0000000 --- a/android/cts/main/src/gles2-temp-excluded.txt +++ /dev/null
@@ -1 +0,0 @@ -# Tests to be temporarily skipped for Android CI, but still enforced in CTS.
diff --git a/android/cts/main/egl-main-risky.txt b/android/cts/main/src/gles3-main-2024-03-01.txt similarity index 100% copy from android/cts/main/egl-main-risky.txt copy to android/cts/main/src/gles3-main-2024-03-01.txt
diff --git a/android/cts/main/src/gles3-temp-excluded.txt b/android/cts/main/src/gles3-temp-excluded.txt deleted file mode 100644 index 2974117..0000000 --- a/android/cts/main/src/gles3-temp-excluded.txt +++ /dev/null
@@ -1 +0,0 @@ -# Tests to be temporarily skipped for Android CI, but still enforced in CTS.
diff --git a/android/cts/main/src/gles31-main-2024-03-01.txt b/android/cts/main/src/gles31-main-2024-03-01.txt new file mode 100644 index 0000000..aa1fef4 --- /dev/null +++ b/android/cts/main/src/gles31-main-2024-03-01.txt
@@ -0,0 +1,32 @@ +dEQP-GLES31.functional.image_load_store.2d.atomic.comp_swap_r32i_return_value +dEQP-GLES31.functional.image_load_store.2d.atomic.comp_swap_r32ui_return_value +dEQP-GLES31.functional.image_load_store.2d_array.atomic.comp_swap_r32i_return_value +dEQP-GLES31.functional.image_load_store.2d_array.atomic.comp_swap_r32ui_return_value +dEQP-GLES31.functional.image_load_store.3d.atomic.comp_swap_r32i_return_value +dEQP-GLES31.functional.image_load_store.3d.atomic.comp_swap_r32ui_return_value +dEQP-GLES31.functional.image_load_store.buffer.atomic.comp_swap_r32i_return_value +dEQP-GLES31.functional.image_load_store.buffer.atomic.comp_swap_r32ui_return_value +dEQP-GLES31.functional.image_load_store.cube.atomic.comp_swap_r32i_return_value +dEQP-GLES31.functional.image_load_store.cube.atomic.comp_swap_r32ui_return_value +dEQP-GLES31.functional.texture.border_clamp.formats.r32f.linear_size_npot +dEQP-GLES31.functional.texture.border_clamp.formats.r32f.linear_size_pot +dEQP-GLES31.functional.texture.border_clamp.formats.rg32f.linear_size_npot +dEQP-GLES31.functional.texture.border_clamp.formats.rg32f.linear_size_pot +dEQP-GLES31.functional.texture.border_clamp.formats.rgb32f.linear_size_npot +dEQP-GLES31.functional.texture.border_clamp.formats.rgb32f.linear_size_pot +dEQP-GLES31.functional.texture.border_clamp.formats.rgba32f.linear_size_npot +dEQP-GLES31.functional.texture.border_clamp.formats.rgba32f.linear_size_pot +dEQP-GLES31.functional.texture.border_clamp.per_axis_wrap_mode.texture_2d.float_color.linear.s_clamp_to_edge_t_clamp_to_border_npot +dEQP-GLES31.functional.texture.border_clamp.per_axis_wrap_mode.texture_2d.float_color.linear.s_clamp_to_edge_t_clamp_to_border_pot +dEQP-GLES31.functional.texture.border_clamp.per_axis_wrap_mode.texture_2d.float_color.linear.s_mirrored_repeat_t_clamp_to_border_npot +dEQP-GLES31.functional.texture.border_clamp.per_axis_wrap_mode.texture_2d.float_color.linear.s_mirrored_repeat_t_clamp_to_border_pot +dEQP-GLES31.functional.texture.border_clamp.per_axis_wrap_mode.texture_2d.float_color.linear.s_repeat_t_clamp_to_border_npot +dEQP-GLES31.functional.texture.border_clamp.per_axis_wrap_mode.texture_2d.float_color.linear.s_repeat_t_clamp_to_border_pot +dEQP-GLES31.functional.texture.border_clamp.per_axis_wrap_mode.texture_3d.float_color.linear.s_clamp_to_border_t_clamp_to_border_r_clamp_to_border_npot +dEQP-GLES31.functional.texture.border_clamp.per_axis_wrap_mode.texture_3d.float_color.linear.s_clamp_to_border_t_clamp_to_border_r_clamp_to_border_pot +dEQP-GLES31.functional.texture.border_clamp.per_axis_wrap_mode.texture_3d.float_color.linear.s_clamp_to_border_t_clamp_to_border_r_repeat_npot +dEQP-GLES31.functional.texture.border_clamp.per_axis_wrap_mode.texture_3d.float_color.linear.s_clamp_to_border_t_clamp_to_border_r_repeat_pot +dEQP-GLES31.functional.texture.border_clamp.per_axis_wrap_mode.texture_3d.float_color.linear.s_mirrored_repeat_t_clamp_to_border_r_repeat_npot +dEQP-GLES31.functional.texture.border_clamp.per_axis_wrap_mode.texture_3d.float_color.linear.s_mirrored_repeat_t_clamp_to_border_r_repeat_pot +dEQP-GLES31.functional.texture.border_clamp.per_axis_wrap_mode.texture_3d.float_color.linear.s_repeat_t_mirrored_repeat_r_clamp_to_border_npot +dEQP-GLES31.functional.texture.border_clamp.per_axis_wrap_mode.texture_3d.float_color.linear.s_repeat_t_mirrored_repeat_r_clamp_to_border_pot
diff --git a/android/cts/main/src/gles31-temp-excluded.txt b/android/cts/main/src/gles31-temp-excluded.txt deleted file mode 100644 index bf72f10..0000000 --- a/android/cts/main/src/gles31-temp-excluded.txt +++ /dev/null
@@ -1,2 +0,0 @@ -# Tests to be temporarily skipped for Android CI, but still enforced in CTS. -
diff --git a/android/cts/main/src/vk-main-2019-03-01.txt b/android/cts/main/src/vk-main-2019-03-01.txt index 412f105..aa88e50 100644 --- a/android/cts/main/src/vk-main-2019-03-01.txt +++ b/android/cts/main/src/vk-main-2019-03-01.txt
@@ -3849,7 +3849,6 @@ dEQP-VK.api.device_init.create_device_unsupported_features.coverage_reduction_mode_features_nv dEQP-VK.api.device_init.create_device_unsupported_features.fragment_shader_interlock_features_ext dEQP-VK.api.device_init.create_device_unsupported_features.ycbcr_image_arrays_features_ext -dEQP-VK.api.device_init.create_device_unsupported_features.line_rasterization_features_ext dEQP-VK.api.device_init.create_device_unsupported_features.shader_atomic_float_features_ext dEQP-VK.api.device_init.create_device_unsupported_features.index_type_uint8_features_khr dEQP-VK.api.device_init.create_device_unsupported_features.extended_dynamic_state_features_ext
diff --git a/android/cts/main/src/vk-main-2023-03-01-part1.txt b/android/cts/main/src/vk-main-2023-03-01-part1.txt index f44211b..f46c080 100644 --- a/android/cts/main/src/vk-main-2023-03-01-part1.txt +++ b/android/cts/main/src/vk-main-2023-03-01-part1.txt
@@ -303737,7 +303737,3 @@ dEQP-VK.pipeline.fast_linked_library.misc.implicit_primitive_id dEQP-VK.pipeline.fast_linked_library.misc.implicit_primitive_id_with_tessellation dEQP-VK.pipeline.fast_linked_library.misc.interpolate_at_sample_no_sample_shading -dEQP-VK.pipeline.fast_linked_library.misc.unused_shader_stages -dEQP-VK.pipeline.fast_linked_library.misc.unused_shader_stages_include_geom -dEQP-VK.pipeline.fast_linked_library.misc.unused_shader_stages_include_tess -dEQP-VK.pipeline.fast_linked_library.misc.unused_shader_stages_include_tess_include_geom
diff --git a/android/cts/main/src/vk-main-2023-03-01-part2.txt b/android/cts/main/src/vk-main-2023-03-01-part2.txt index 12bc44b..cb03571 100644 --- a/android/cts/main/src/vk-main-2023-03-01-part2.txt +++ b/android/cts/main/src/vk-main-2023-03-01-part2.txt
@@ -107782,10 +107782,6 @@ dEQP-VK.pipeline.pipeline_library.color_write_enable_maxa.cwe_after_bind.attachments5_more3 dEQP-VK.pipeline.pipeline_library.misc.implicit_primitive_id dEQP-VK.pipeline.pipeline_library.misc.implicit_primitive_id_with_tessellation -dEQP-VK.pipeline.pipeline_library.misc.unused_shader_stages -dEQP-VK.pipeline.pipeline_library.misc.unused_shader_stages_include_geom -dEQP-VK.pipeline.pipeline_library.misc.unused_shader_stages_include_tess -dEQP-VK.pipeline.pipeline_library.misc.unused_shader_stages_include_tess_include_geom dEQP-VK.pipeline.pipeline_library.graphics_library.fast.4 dEQP-VK.pipeline.pipeline_library.graphics_library.fast.0_1111 dEQP-VK.pipeline.pipeline_library.graphics_library.fast.0_112
diff --git a/android/cts/main/src/vk-main-2024-03-01.txt b/android/cts/main/src/vk-main-2024-03-01.txt new file mode 100644 index 0000000..14295c6 --- /dev/null +++ b/android/cts/main/src/vk-main-2024-03-01.txt Binary files differ
diff --git a/android/cts/main/src/vk-temp-excluded.txt b/android/cts/main/src/vk-temp-excluded.txt deleted file mode 100644 index bf72f10..0000000 --- a/android/cts/main/src/vk-temp-excluded.txt +++ /dev/null
@@ -1,2 +0,0 @@ -# Tests to be temporarily skipped for Android CI, but still enforced in CTS. -
diff --git a/android/cts/main/egl-main-risky.txt b/android/cts/main/vk-main-2025-03-01.txt similarity index 100% rename from android/cts/main/egl-main-risky.txt rename to android/cts/main/vk-main-2025-03-01.txt
diff --git a/android/cts/runner/src/com/drawelements/deqp/runner/BatchRunConfiguration.java b/android/cts/runner/src/com/drawelements/deqp/runner/BatchRunConfiguration.java index 6d54f22..66ad895 100644 --- a/android/cts/runner/src/com/drawelements/deqp/runner/BatchRunConfiguration.java +++ b/android/cts/runner/src/com/drawelements/deqp/runner/BatchRunConfiguration.java
@@ -30,6 +30,14 @@ private final String mSurfaceType; private final boolean mRequired; + // Added for sub-class KhronosCTSBatchRunConfiguration + public BatchRunConfiguration() { + mGlConfig = ""; + mRotation = ""; + mSurfaceType = ""; + mRequired = false; + } + public BatchRunConfiguration(String glConfig, String rotation, String surfaceType, boolean required) { mGlConfig = glConfig;
diff --git a/android/cts/runner/src/com/drawelements/deqp/runner/DeqpTestRunner.java b/android/cts/runner/src/com/drawelements/deqp/runner/DeqpTestRunner.java index 9229d39..8541927 100644 --- a/android/cts/runner/src/com/drawelements/deqp/runner/DeqpTestRunner.java +++ b/android/cts/runner/src/com/drawelements/deqp/runner/DeqpTestRunner.java
@@ -86,18 +86,18 @@ IShardableTest, ITestCollector, IRuntimeHintProvider { private static final String DEQP_ONDEVICE_APK = "com.drawelements.deqp.apk"; private static final String DEQP_ONDEVICE_PKG = "com.drawelements.deqp"; - private static final String INCOMPLETE_LOG_MESSAGE = + protected static final String INCOMPLETE_LOG_MESSAGE = "Crash: Incomplete test log"; - private static final String TIMEOUT_LOG_MESSAGE = "Timeout: Test timeout"; + protected static final String TIMEOUT_LOG_MESSAGE = "Timeout: Test timeout"; private static final String SKIPPED_INSTANCE_LOG_MESSAGE = "Configuration skipped"; public static final String ASSUMPTION_FAILURE_DEQP_LEVEL_LOG_MESSAGE = "Features to be tested are not supported by device"; - private static final String NOT_EXECUTABLE_LOG_MESSAGE = + protected static final String NOT_EXECUTABLE_LOG_MESSAGE = "Abort: Test cannot be executed"; - private static final String APP_DIR = "/sdcard/"; - private static final String CASE_LIST_FILE_NAME = "dEQP-TestCaseList.txt"; - private static final String LOG_FILE_NAME = "TestLog.qpa"; + protected static final String APP_DIR = "/sdcard/"; + protected static final String CASE_LIST_FILE_NAME = "dEQP-TestCaseList.txt"; + protected static final String LOG_FILE_NAME = "TestLog.qpa"; public static final String FEATURE_LANDSCAPE = "android.hardware.screen.landscape"; public static final String FEATURE_PORTRAIT = @@ -115,9 +115,9 @@ private static final int R_API_LEVEL = 30; private static final int DEQP_LEVEL_R_2020 = 132383489; - private static final String ANGLE_NONE = "none"; - private static final String ANGLE_VULKAN = "vulkan"; - private static final String ANGLE_OPENGLES = "opengles"; + protected static final String ANGLE_NONE = "none"; + protected static final String ANGLE_VULKAN = "vulkan"; + protected static final String ANGLE_OPENGLES = "opengles"; // !NOTE: There's a static method copyOptions() for copying options during // split. If you add state update copyOptions() as appropriate! @@ -163,7 +163,7 @@ name = "include-filter", description = "Test include filter. '*' is zero or more letters. '.' has no special meaning.") - private List<String> mIncludeFilters = new ArrayList<>(); + protected List<String> mIncludeFilters = new ArrayList<>(); @Option(name = "include-filter-file", description = "Load list of includes from the files given.") private List<String> mIncludeFilterFiles = new ArrayList<>(); @@ -171,7 +171,7 @@ name = "exclude-filter", description = "Test exclude filter. '*' is zero or more letters. '.' has no special meaning.") - private List<String> mExcludeFilters = new ArrayList<>(); + protected List<String> mExcludeFilters = new ArrayList<>(); @Option(name = "exclude-filter-file", description = "Load list of excludes from the files given.") private List<String> mExcludeFilterFiles = new ArrayList<>(); @@ -199,14 +199,14 @@ @Option(name = "collect-raw-logs", description = "whether to collect raw deqp test log data") - private boolean mLogData = false; + protected boolean mLogData = false; @Option( name = "deqp-use-angle", description = "ANGLE backend ('none', 'vulkan', 'opengles'). Defaults to 'none' (don't use ANGLE)", importance = Option.Importance.NEVER) - private String mAngle = "none"; + protected String mAngle = "none"; @Option(name = "disable-watchdog", description = "Disable the native testrunner's per-test watchdog.") @@ -219,23 +219,23 @@ + "'all' enforces all dEQP tests to run") private String mForceDeqpLevel = ""; - private Set<TestDescription> mRemainingTests = null; + protected Set<TestDescription> mRemainingTests = null; private Map<TestDescription, Set<BatchRunConfiguration>> mTestInstances = null; private final TestInstanceResultListener mInstanceListerner = new TestInstanceResultListener(); private final Map<TestDescription, Integer> mTestInstabilityRatings = new HashMap<>(); - private IAbi mAbi; - private CompatibilityBuildHelper mBuildHelper; - private ITestDevice mDevice; + protected IAbi mAbi; + protected CompatibilityBuildHelper mBuildHelper; + protected ITestDevice mDevice; private Map<String, Optional<Integer>> mDeviceFeatures; private Map<String, Boolean> mConfigQuerySupportCache = new HashMap<>(); - private IRunUtil mRunUtil = RunUtil.getDefault(); + protected IRunUtil mRunUtil = RunUtil.getDefault(); private Set<String> mIncrementalDeqpIncludeTests = new HashSet<>(); - private long mTimeOfLastRun = 0; + protected long mTimeOfLastRun = 0; - private IRecovery mDeviceRecovery = new Recovery(); + protected IRecovery mDeviceRecovery = new Recovery(); { mDeviceRecovery.setSleepProvider(new SleepProvider()); } public DeqpTestRunner() {} @@ -316,16 +316,17 @@ private static final class CapabilityQueryFailureException extends Exception {} + protected TestInstanceResultListener getInstanceListener() {return mInstanceListerner;} + /** * dEQP test instance listerer and invocation result forwarded */ - private class TestInstanceResultListener { - private ITestInvocationListener mSink; + protected class TestInstanceResultListener { private BatchRunConfiguration mRunConfig; - - private TestDescription mCurrentTestId; - private boolean mGotTestResult; - private String mCurrentTestLog; + protected ITestInvocationListener mSink; + protected TestDescription mCurrentTestId; + protected boolean mGotTestResult; + protected String mCurrentTestLog; private class PendingResult { boolean allInstancesPassed; @@ -352,7 +353,7 @@ /** * Forward result to sink */ - private void forwardFinalizedPendingResult(TestDescription testId) { + protected void forwardFinalizedPendingResult(TestDescription testId) { if (mRemainingTests.contains(testId)) { final PendingResult result = mPendingResults.get(testId); @@ -482,21 +483,21 @@ /** * Handles beginning of dEQP session. */ - private void handleBeginSession(Map<String, String> values) { + protected void handleBeginSession(Map<String, String> values) { // ignore } /** * Handles end of dEQP session. */ - private void handleEndSession(Map<String, String> values) { + protected void handleEndSession(Map<String, String> values) { // ignore } /** * Handles beginning of dEQP testcase. */ - private void handleBeginTestCase(Map<String, String> values) { + protected void handleBeginTestCase(Map<String, String> values) { mCurrentTestId = pathToIdentifier(values.get("dEQP-BeginTestCase-TestCasePath")); mCurrentTestLog = ""; @@ -514,7 +515,7 @@ /** * Handles end of dEQP testcase. */ - private void handleEndTestCase(Map<String, String> values) { + protected void handleEndTestCase(Map<String, String> values) { final PendingResult result = mPendingResults.get(mCurrentTestId); if (result != null) { @@ -542,7 +543,7 @@ /** * Handles dEQP testcase result. */ - private void handleTestCaseResult(Map<String, String> values) { + protected void handleTestCaseResult(Map<String, String> values) { String code = values.get("dEQP-TestCaseResult-Code"); String details = values.get("dEQP-TestCaseResult-Details"); @@ -581,7 +582,7 @@ /** * Handles terminated dEQP testcase. */ - private void handleTestCaseTerminate(Map<String, String> values) { + protected void handleTestCaseTerminate(Map<String, String> values) { final PendingResult result = mPendingResults.get(mCurrentTestId); if (result != null) { @@ -605,7 +606,7 @@ /** * Handles dEQP testlog data. */ - private void handleTestLogData(Map<String, String> values) { + protected void handleTestLogData(Map<String, String> values) { mCurrentTestLog = mCurrentTestLog + values.get("dEQP-TestLogData-Log"); } @@ -660,14 +661,16 @@ /** * dEQP instrumentation parser */ - private static class InstrumentationParser extends MultiLineReceiver { - private TestInstanceResultListener mListener; + protected static class InstrumentationParser extends MultiLineReceiver { + protected TestInstanceResultListener mListener; - private Map<String, String> mValues; - private String mCurrentName; - private String mCurrentValue; - private int mResultCode; - private boolean mGotExitValue = false; + protected Map<String, String> mValues; + protected String mCurrentName; + protected String mCurrentValue; + protected int mResultCode; + protected boolean mGotExitValue = false; + + protected InstrumentationParser(){} public InstrumentationParser(TestInstanceResultListener listener) { mListener = listener; @@ -1139,7 +1142,7 @@ /** * Converts dEQP testcase path to TestDescription. */ - private static TestDescription pathToIdentifier(String testPath) { + protected static TestDescription pathToIdentifier(String testPath) { int indexOfLastDot = testPath.lastIndexOf('.'); String className = testPath.substring(0, indexOfLastDot); String testName = testPath.substring(indexOfLastDot + 1); @@ -1148,7 +1151,7 @@ } // \todo [2015-10-16 kalle] How unique should this be? - private String getId() { + protected String getId() { return AbiUtils.createId(mAbi.getName(), mDeqpPackage); } @@ -1156,7 +1159,7 @@ * Generates tescase trie from dEQP testcase paths. Used to define which * testcases to execute. */ - private static String + protected static String generateTestCaseTrieFromPaths(Collection<String> tests) { String result = "{"; boolean first = true; @@ -1211,7 +1214,7 @@ /** * Generates testcase trie from TestDescriptions. */ - private static String + protected static String generateTestCaseTrie(Collection<TestDescription> tests) { ArrayList<String> testPaths = new ArrayList<String>(); @@ -1222,9 +1225,14 @@ return generateTestCaseTrieFromPaths(testPaths); } - private static class TestBatch { - public BatchRunConfiguration config; - public List<TestDescription> tests; + protected static class TestBatch { + private BatchRunConfiguration mConfig; + protected List<TestDescription> mTests; + + public BatchRunConfiguration getTestBatchConfig() {return mConfig;} + public List<TestDescription> getTestBatchTestDescriptionList() {return mTests;} + public void setTestBatchConfig(BatchRunConfiguration config) {mConfig = config;} + public void setTestBatchTestDescriptionList(List<TestDescription> tests) {mTests = tests;} } /** @@ -1234,18 +1242,17 @@ * @param requiredConfig Select only instances with pending requiredConfig, * or null to select any run configuration. */ - private TestBatch selectRunBatch(Collection<TestDescription> pool, + protected TestBatch selectRunBatch(Collection<TestDescription> pool, BatchRunConfiguration requiredConfig) { // select one test (leading test) that is going to be executed and then // pack along as many other compatible instances as possible. - TestDescription leadingTest = null; for (TestDescription test : pool) { if (!mRemainingTests.contains(test)) { continue; } if (requiredConfig != null && - !mInstanceListerner.isPendingTestInstance(test, + !getInstanceListener().isPendingTestInstance(test, requiredConfig)) { continue; } @@ -1264,7 +1271,7 @@ } else { for (BatchRunConfiguration runConfig : getTestRunConfigs(leadingTest)) { - if (mInstanceListerner.isPendingTestInstance(leadingTest, + if (getInstanceListener().isPendingTestInstance(leadingTest, runConfig)) { leadingTestConfig = runConfig; break; @@ -1280,16 +1287,16 @@ final int leadingInstability = getTestInstabilityRating(leadingTest); final TestBatch runBatch = new TestBatch(); - runBatch.config = leadingTestConfig; - runBatch.tests = new ArrayList<>(); - runBatch.tests.add(leadingTest); + runBatch.setTestBatchConfig(leadingTestConfig); + List<TestDescription> runBatchTests = new ArrayList<>(); + runBatchTests.add(leadingTest); for (TestDescription test : pool) { if (test == leadingTest) { // do not re-select the leading tests continue; } - if (!mInstanceListerner.isPendingTestInstance(test, + if (!getInstanceListener().isPendingTestInstance(test, leadingTestConfig)) { // select only compatible continue; @@ -1301,13 +1308,14 @@ // stability rating. continue; } - if (runBatch.tests.size() >= + if (runBatchTests.size() >= getBatchSizeLimitForInstability(leadingInstability)) { // batch size is limited. break; } - runBatch.tests.add(test); + runBatchTests.add(test); } + runBatch.setTestBatchTestDescriptionList(runBatchTests); return runBatch; } @@ -1316,22 +1324,22 @@ return TESTCASE_BATCH_LIMIT; } - private int getBatchNumPendingCases(TestBatch batch) { + protected int getBatchNumPendingCases(TestBatch batch) { int numPending = 0; - for (TestDescription test : batch.tests) { - if (mInstanceListerner.isPendingTestInstance(test, batch.config)) { + for (TestDescription test : batch.getTestBatchTestDescriptionList()) { + if (getInstanceListener().isPendingTestInstance(test, batch.getTestBatchConfig())) { ++numPending; } } return numPending; } - private int getBatchSizeLimitForInstability(int batchInstabilityRating) { + protected int getBatchSizeLimitForInstability(int batchInstabilityRating) { // reduce group size exponentially down to one return Math.max(1, getBatchSizeLimit() / (1 << batchInstabilityRating)); } - private int getTestInstabilityRating(TestDescription testId) { + protected int getTestInstabilityRating(TestDescription testId) { if (mTestInstabilityRatings.containsKey(testId)) { return mTestInstabilityRatings.get(testId); } else { @@ -1339,12 +1347,12 @@ } } - private void recordTestInstability(TestDescription testId) { + protected void recordTestInstability(TestDescription testId) { mTestInstabilityRatings.put(testId, getTestInstabilityRating(testId) + 1); } - private void clearTestInstability(TestDescription testId) { + protected void clearTestInstability(TestDescription testId) { mTestInstabilityRatings.put(testId, 0); } @@ -1367,19 +1375,19 @@ /** * Runs a TestBatch by either faking it or executing it on a device. */ - private void runTestRunBatch(TestBatch batch) + protected void runTestRunBatch(TestBatch batch) throws DeviceNotAvailableException, CapabilityQueryFailureException { // prepare instance listener - mInstanceListerner.setCurrentConfig(batch.config); - for (TestDescription test : batch.tests) { - mInstanceListerner.setTestInstances(test, getTestRunConfigs(test)); + getInstanceListener().setCurrentConfig(batch.getTestBatchConfig()); + for (TestDescription test : batch.getTestBatchTestDescriptionList()) { + getInstanceListener().setTestInstances(test, getTestRunConfigs(test)); } // execute only if config is executable, else fake results - if (isSupportedRunConfiguration(batch.config)) { + if (isSupportedRunConfiguration(batch.getTestBatchConfig())) { executeTestRunBatch(batch); } else { - if (batch.config.isRequired()) { + if (batch.getTestBatchConfig().isRequired()) { fakeFailTestRunBatch(batch); } else { fakePassTestRunBatch(batch); @@ -1425,19 +1433,19 @@ } } - private static final class AdbComLinkOpenError extends Exception { + protected static final class AdbComLinkOpenError extends Exception { public AdbComLinkOpenError(String description, Throwable inner) { super(description, inner); } } - private static final class AdbComLinkKilledError extends Exception { + protected static final class AdbComLinkKilledError extends Exception { public AdbComLinkKilledError(String description, Throwable inner) { super(description, inner); } } - private static final class AdbComLinkUnresponsiveError extends Exception { + protected static final class AdbComLinkUnresponsiveError extends Exception { public AdbComLinkUnresponsiveError(String description, Throwable inner) { super(description, inner); @@ -1451,7 +1459,7 @@ * @throws AdbComLinkKilledError if established connection is killed * prematurely. */ - private void + protected void executeShellCommandAndReadOutput(final String command, final IShellOutputReceiver receiver) throws AdbComLinkOpenError, AdbComLinkKilledError, @@ -1480,18 +1488,45 @@ /** * Executes given test batch on a device */ - private void executeTestRunBatch(TestBatch batch) + protected void executeTestRunBatch(TestBatch batch) throws DeviceNotAvailableException { + final String instrumentationName = + "com.drawelements.deqp/com.drawelements.deqp.testercore.DeqpInstrumentation"; + + final StringBuilder deqpCmdLine = new StringBuilder(); + deqpCmdLine.append("--deqp-caselist-file="); + deqpCmdLine.append(APP_DIR + CASE_LIST_FILE_NAME); + deqpCmdLine.append(" "); + deqpCmdLine.append(getRunConfigDisplayCmdLine(batch.getTestBatchConfig())); + + // If we are not logging data, do not bother outputting the images from + // the test exe. + if (!mLogData) { + deqpCmdLine.append(" --deqp-log-images=disable"); + } + + if (!mDisableWatchdog) { + deqpCmdLine.append(" --deqp-watchdog=enable"); + } + + final String command = String.format( + "am instrument %s -w -e deqpLogFilename \"%s\" -e deqpCmdLine \"%s\"" + + " -e deqpLogData \"%s\" %s", + AbiUtils.createAbiFlag(mAbi.getName()), APP_DIR + LOG_FILE_NAME, + deqpCmdLine.toString(), mLogData, instrumentationName); + + final InstrumentationParser parser = + new InstrumentationParser(getInstanceListener()); // attempt full run once - executeTestRunBatchRun(batch); + executeTestRunBatchRun(batch, instrumentationName, command, parser); // split remaining tests to two sub batches and execute both. This will // terminate since executeTestRunBatchRun will always progress for a // batch of size 1. final ArrayList<TestDescription> pendingTests = new ArrayList<>(); - for (TestDescription test : batch.tests) { - if (mInstanceListerner.isPendingTestInstance(test, batch.config)) { + for (TestDescription test : batch.getTestBatchTestDescriptionList()) { + if (getInstanceListener().isPendingTestInstance(test, batch.getTestBatchConfig())) { pendingTests.add(test); } } @@ -1504,7 +1539,7 @@ // head for (;;) { - TestBatch subBatch = selectRunBatch(headList, batch.config); + TestBatch subBatch = selectRunBatch(headList, batch.getTestBatchConfig()); if (subBatch == null) { break; @@ -1515,7 +1550,7 @@ // tail for (;;) { - TestBatch subBatch = selectRunBatch(tailList, batch.config); + TestBatch subBatch = selectRunBatch(tailList, batch.getTestBatchConfig()); if (subBatch == null) { break; @@ -1536,16 +1571,16 @@ * Tries to run the batch. Always makes progress (executes instances or * modifies stability scores). */ - private void executeTestRunBatchRun(TestBatch batch) + protected void executeTestRunBatchRun(TestBatch batch, final String instrumentationName, final String runCommand, final InstrumentationParser parser) throws DeviceNotAvailableException { - if (getBatchNumPendingCases(batch) != batch.tests.size()) { + if (getBatchNumPendingCases(batch) != batch.getTestBatchTestDescriptionList().size()) { throw new AssertionError( "executeTestRunBatchRun precondition failed"); } checkInterrupted(); // throws if interrupted - final String testCases = generateTestCaseTrie(batch.tests); + final String testCases = generateTestCaseTrie(batch.getTestBatchTestDescriptionList()); final String testCaseFilename = APP_DIR + CASE_LIST_FILE_NAME; mDevice.executeShellCommand("rm " + testCaseFilename); mDevice.executeShellCommand("rm " + APP_DIR + LOG_FILE_NAME); @@ -1554,34 +1589,7 @@ testCaseFilename); } - final String instrumentationName = - "com.drawelements.deqp/com.drawelements.deqp.testercore.DeqpInstrumentation"; - - final StringBuilder deqpCmdLine = new StringBuilder(); - deqpCmdLine.append("--deqp-caselist-file="); - deqpCmdLine.append(APP_DIR + CASE_LIST_FILE_NAME); - deqpCmdLine.append(" "); - deqpCmdLine.append(getRunConfigDisplayCmdLine(batch.config)); - - // If we are not logging data, do not bother outputting the images from - // the test exe. - if (!mLogData) { - deqpCmdLine.append(" --deqp-log-images=disable"); - } - - if (!mDisableWatchdog) { - deqpCmdLine.append(" --deqp-watchdog=enable"); - } - - final String command = String.format( - "am instrument %s -w -e deqpLogFilename \"%s\" -e deqpCmdLine \"%s\"" - + " -e deqpLogData \"%s\" %s", - AbiUtils.createAbiFlag(mAbi.getName()), APP_DIR + LOG_FILE_NAME, - deqpCmdLine.toString(), mLogData, instrumentationName); - final int numRemainingInstancesBefore = getNumRemainingInstances(); - final InstrumentationParser parser = - new InstrumentationParser(mInstanceListerner); Throwable interruptingError = null; // Fix the requirement of sleep() between batches @@ -1592,7 +1600,7 @@ } try { - executeShellCommandAndReadOutput(command, parser); + executeShellCommandAndReadOutput(runCommand, parser); } catch (Throwable ex) { interruptingError = ex; } finally { @@ -1601,7 +1609,7 @@ } final boolean progressedSinceLastCall = - mInstanceListerner.getCurrentTestId() != null || + getInstanceListener().getCurrentTestId() != null || getNumRemainingInstances() < numRemainingInstancesBefore; if (progressedSinceLastCall) { @@ -1615,8 +1623,8 @@ // execution. Device is likely fine, so we won't attempt to recover // the device. if (interruptingError instanceof AdbComLinkUnresponsiveError) { - mInstanceListerner.abortTest( - mInstanceListerner.getCurrentTestId(), TIMEOUT_LOG_MESSAGE); + getInstanceListener().abortTest( + getInstanceListener().getCurrentTestId(), TIMEOUT_LOG_MESSAGE); } else if (interruptingError instanceof AdbComLinkOpenError) { mDeviceRecovery.recoverConnectionRefused(); } else if (interruptingError instanceof AdbComLinkKilledError) { @@ -1636,12 +1644,12 @@ } // Progress guarantees. - if (batch.tests.size() == 1) { - final TestDescription onlyTest = batch.tests.iterator().next(); + if (batch.getTestBatchTestDescriptionList().size() == 1) { + final TestDescription onlyTest = batch.getTestBatchTestDescriptionList().iterator().next(); final boolean wasTestExecuted = - !mInstanceListerner.isPendingTestInstance(onlyTest, - batch.config) && - mInstanceListerner.getCurrentTestId() == null; + !getInstanceListener().isPendingTestInstance(onlyTest, + batch.getTestBatchConfig()) && + getInstanceListener().getCurrentTestId() == null; final boolean wasLinkFailure = !parser.wasSuccessful() || interruptingError != null; @@ -1656,11 +1664,11 @@ // non-executable. This is required so that a consistently // crashing or non-existent tests will not cause futile // (non-terminating) re-execution attempts. - if (mInstanceListerner.getCurrentTestId() != null) { - mInstanceListerner.abortTest(onlyTest, + if (getInstanceListener().getCurrentTestId() != null) { + getInstanceListener().abortTest(onlyTest, INCOMPLETE_LOG_MESSAGE); } else { - mInstanceListerner.abortTest(onlyTest, + getInstanceListener().abortTest(onlyTest, NOT_EXECUTABLE_LOG_MESSAGE); } } else if (wasTestExecuted) { @@ -1673,34 +1681,34 @@ // only its instability rating. // // A successful run of tests clears instability rating. - if (mInstanceListerner.getCurrentTestId() == null) { - for (TestDescription test : batch.tests) { - if (mInstanceListerner.isPendingTestInstance( - test, batch.config)) { + if (getInstanceListener().getCurrentTestId() == null) { + for (TestDescription test : batch.getTestBatchTestDescriptionList()) { + if (getInstanceListener().isPendingTestInstance( + test, batch.getTestBatchConfig())) { recordTestInstability(test); } else { clearTestInstability(test); } } } else { - recordTestInstability(mInstanceListerner.getCurrentTestId()); - for (TestDescription test : batch.tests) { + recordTestInstability(getInstanceListener().getCurrentTestId()); + for (TestDescription test : batch.getTestBatchTestDescriptionList()) { // \note: isPendingTestInstance is false for // getCurrentTestId. Current ID is considered 'running' and // will be restored to 'pending' in endBatch(). - if (!test.equals(mInstanceListerner.getCurrentTestId()) && - !mInstanceListerner.isPendingTestInstance( - test, batch.config)) { + if (!test.equals(getInstanceListener().getCurrentTestId()) && + !getInstanceListener().isPendingTestInstance( + test, batch.getTestBatchConfig())) { clearTestInstability(test); } } } } - mInstanceListerner.endBatch(); + getInstanceListener().endBatch(); } - private static String + protected String getRunConfigDisplayCmdLine(BatchRunConfiguration runConfig) { final StringBuilder deqpCmdLine = new StringBuilder(); if (!runConfig.getGlConfig().isEmpty()) { @@ -1724,14 +1732,14 @@ return deqpCmdLine.toString(); } - private int getNumRemainingInstances() { + protected int getNumRemainingInstances() { int retVal = 0; for (TestDescription testId : mRemainingTests) { // If case is in current working set, sum only not yet executed // instances. If case is not in current working set, sum all // instances (since they are not yet executed). - if (mInstanceListerner.mPendingResults.containsKey(testId)) { - retVal += mInstanceListerner.mPendingResults.get(testId) + if (getInstanceListener().mPendingResults.containsKey(testId)) { + retVal += getInstanceListener().mPendingResults.get(testId) .remainingConfigs.size(); } else { retVal += mTestInstances.get(testId).size(); @@ -1744,7 +1752,7 @@ * Checks if this execution has been marked as interrupted and throws if it * has. */ - private void checkInterrupted() throws RunInterruptedException { + protected void checkInterrupted() throws RunInterruptedException { // Work around the API. RunUtil::checkInterrupted is private but we can // call it indirectly by sleeping a value <= 0. mRunUtil.sleep(0); @@ -1754,11 +1762,11 @@ * Pass given batch tests without running it */ private void fakePassTestRunBatch(TestBatch batch) { - for (TestDescription test : batch.tests) { + for (TestDescription test : batch.getTestBatchTestDescriptionList()) { CLog.d( "Marking '%s' invocation in config '%s' as passed without running", - test.toString(), batch.config.getId()); - mInstanceListerner.skipTest(test); + test.toString(), batch.getTestBatchConfig().getId()); + getInstanceListener().skipTest(test); } } @@ -1766,11 +1774,11 @@ * Fail given batch tests without running it */ private void fakeFailTestRunBatch(TestBatch batch) { - for (TestDescription test : batch.tests) { + for (TestDescription test : batch.getTestBatchTestDescriptionList()) { CLog.d( "Marking '%s' invocation in config '%s' as failed without running", - test.toString(), batch.config.getId()); - mInstanceListerner.abortTest(test, "Required config not supported"); + test.toString(), batch.getTestBatchConfig().getId()); + getInstanceListener().abortTest(test, "Required config not supported"); } } @@ -1903,6 +1911,7 @@ CLog.d(" 2022-03-01 -> 132514561"); CLog.d(" 2023-03-01 -> 132580097"); CLog.d(" 2024-03-01 -> 132645633"); + CLog.d(" 2025-03-01 -> 132711169"); CLog.d("Minimum level required to run this caselist is %d", minimumLevel); @@ -2197,7 +2206,7 @@ } } - private static List<Pattern> getPatternFilters(List<String> filters) { + protected static List<Pattern> getPatternFilters(List<String> filters) { List<Pattern> patterns = new ArrayList<Pattern>(); for (String filter : filters) { if (filter.contains("*")) { @@ -2208,7 +2217,7 @@ return patterns; } - private static Set<String> getNonPatternFilters(List<String> filters) { + protected static Set<String> getNonPatternFilters(List<String> filters) { Set<String> nonPatternFilters = new HashSet<String>(); for (String filter : filters) { if (filter.startsWith("#") || filter.isEmpty()) { @@ -2232,7 +2241,7 @@ return nonPatternFilters; } - private static boolean matchesAny(TestDescription test, + protected static boolean matchesAny(TestDescription test, List<Pattern> patterns) { for (Pattern pattern : patterns) { if (pattern.matcher(test.toString()).matches()) { @@ -2417,7 +2426,7 @@ /** * Set up the test environment. */ - private void setupTestEnvironment() throws DeviceNotAvailableException { + protected void setupTestEnvironment(String testPackageName) throws DeviceNotAvailableException { try { // Get the system into a known state. // Clear ANGLE Global.Settings values @@ -2432,7 +2441,7 @@ // Force dEQP to use ANGLE mDevice.executeShellCommand( "settings put global angle_gl_driver_selection_pkgs " + - DEQP_ONDEVICE_PKG); + testPackageName); mDevice.executeShellCommand( "settings put global angle_gl_driver_selection_values angle"); // Configure ANGLE to use Vulkan @@ -2442,7 +2451,7 @@ // Force dEQP to use ANGLE mDevice.executeShellCommand( "settings put global angle_gl_driver_selection_pkgs " + - DEQP_ONDEVICE_PKG); + testPackageName); mDevice.executeShellCommand( "settings put global angle_gl_driver_selection_values angle"); // Configure ANGLE to use Vulkan @@ -2459,7 +2468,7 @@ /** * Clean up the test environment. */ - private void teardownTestEnvironment() throws DeviceNotAvailableException { + protected void teardownTestEnvironment() throws DeviceNotAvailableException { // ANGLE try { CLog.i("Cleaning up ANGLE"); @@ -2519,9 +2528,9 @@ // - the device's deqp level do not claim to pass the tests ignoreTests(listener); } else if (!mRemainingTests.isEmpty()) { - mInstanceListerner.setSink(listener); + getInstanceListener().setSink(listener); mDeviceRecovery.setDevice(mDevice); - setupTestEnvironment(); + setupTestEnvironment(DEQP_ONDEVICE_PKG); runTests(); teardownTestEnvironment(); }
diff --git a/android/cts/runner/tests/src/com/drawelements/deqp/runner/DeqpTestRunnerTest.java b/android/cts/runner/tests/src/com/drawelements/deqp/runner/DeqpTestRunnerTest.java index 38c638c..c939789 100644 --- a/android/cts/runner/tests/src/com/drawelements/deqp/runner/DeqpTestRunnerTest.java +++ b/android/cts/runner/tests/src/com/drawelements/deqp/runner/DeqpTestRunnerTest.java
@@ -61,7 +61,6 @@ * Unit tests for {@link DeqpTestRunner}. */ public class DeqpTestRunnerTest extends TestCase { - private static final String NAME = "dEQP-GLES3"; private static final IAbi ABI = new Abi("armeabi-v7a", "32"); private static final String APP_DIR = "/sdcard/"; private static final String CASE_LIST_FILE_NAME = "dEQP-TestCaseList.txt"; @@ -70,8 +69,6 @@ "com.drawelements.deqp/com.drawelements.deqp.testercore.DeqpInstrumentation"; private static final String QUERY_INSTRUMENTATION_NAME = "com.drawelements.deqp/com.drawelements.deqp.platformutil.DeqpPlatformCapabilityQueryInstrumentation"; - private static final String DEQP_ONDEVICE_APK = "com.drawelements.deqp.apk"; - private static final String DEQP_ONDEVICE_PKG = "com.drawelements.deqp"; private static final String ONLY_LANDSCAPE_FEATURES = "feature:" + DeqpTestRunner.FEATURE_LANDSCAPE; private static final String ALL_FEATURES =
diff --git a/android/openglcts/Android.bp b/android/openglcts/Android.bp new file mode 100644 index 0000000..3bbb811 --- /dev/null +++ b/android/openglcts/Android.bp
@@ -0,0 +1,45 @@ +// Copyright (C) 2024 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package { + // See: http://go/android-license-faq + // A large-scale-change added 'default_applicable_licenses' to import + // all of the 'license_kinds' from "external_deqp_license" + // to get the below license kinds: + // SPDX-license-identifier-Apache-2.0 + default_applicable_licenses: ["external_deqp_license"], +} + +java_test_host { + name: "KhronosCTSTestCases", + test_suites: ["general-tests"], + team: "trendy_team_android_gpu", + srcs: ["runner/src/**/*.java"], + libs: [ + "cts-tradefed", + "compatibility-host-util", + "tradefed", + ], + static_libs: [ + "CtsDeqpTestCasesJavaHostLib", + ], + per_testcase_directory: true, + data: [ + ":deqp_binary_data", + ":khronos_cts_gles_caselists", + ], + device_common_data: [ + ":org.khronos.gl_cts", + ], +}
diff --git a/android/openglcts/AndroidManifest.xml b/android/openglcts/AndroidManifest.xml index 422aabe..5570eb3 100644 --- a/android/openglcts/AndroidManifest.xml +++ b/android/openglcts/AndroidManifest.xml
@@ -4,7 +4,8 @@ android:versionCode="1" android:versionName="1.0"> - <application android:label="Khronos OpenGL Conformance Tests"> + <application android:label="Khronos OpenGL Conformance Tests" + android:requestLegacyExternalStorage="true"> <!-- separate test runner - supports full command line --> <activity android:name="android.app.NativeActivity" android:theme="@android:style/Theme.NoTitleBar.Fullscreen" @@ -13,7 +14,7 @@ android:exported="true" android:process=":testercore"> <meta-data android:name="android.app.lib_name" - android:value="deqp" /> + android:value="khronosopenglcts" /> <meta-data android:name="android.app.func_name" android:value="createTestActivity" /> </activity> @@ -25,7 +26,7 @@ android:configChanges="orientation|keyboardHidden|screenLayout" android:exported="true"> <meta-data android:name="android.app.lib_name" - android:value="deqp" /> + android:value="khronosopenglcts" /> <meta-data android:name="android.app.func_name" android:value="createExportES32TestParamActivity" /> </activity> @@ -36,9 +37,10 @@ android:label="ES2 CTS" android:configChanges="orientation|keyboardHidden|screenLayout" android:launchMode="singleTask" - android:exported="true"> + android:exported="true" + android:process=":testercore"> <meta-data android:name="android.app.lib_name" - android:value="deqp" /> + android:value="khronosopenglcts" /> <meta-data android:name="android.app.func_name" android:value="createES2CTSActivity" /> <intent-filter> @@ -51,9 +53,10 @@ android:label="ES3 CTS" android:configChanges="orientation|keyboardHidden|screenLayout" android:launchMode="singleTask" - android:exported="true"> + android:exported="true" + android:process=":testercore"> <meta-data android:name="android.app.lib_name" - android:value="deqp" /> + android:value="khronosopenglcts" /> <meta-data android:name="android.app.func_name" android:value="createES3CTSActivity" /> <intent-filter> @@ -66,9 +69,10 @@ android:label="ES3.1 CTS" android:configChanges="orientation|keyboardHidden|screenLayout" android:launchMode="singleTask" - android:exported="true"> + android:exported="true" + android:process=":testercore"> <meta-data android:name="android.app.lib_name" - android:value="deqp" /> + android:value="khronosopenglcts" /> <meta-data android:name="android.app.func_name" android:value="createES31CTSActivity" /> <intent-filter> @@ -81,9 +85,10 @@ android:label="ES3.2 CTS" android:configChanges="orientation|keyboardHidden|screenLayout" android:launchMode="singleTask" - android:exported="true"> + android:exported="true" + android:process=":testercore"> <meta-data android:name="android.app.lib_name" - android:value="deqp" /> + android:value="khronosopenglcts" /> <meta-data android:name="android.app.func_name" android:value="createES32CTSActivity" /> <intent-filter> @@ -96,9 +101,10 @@ android:label="GL4.5 CTS" android:configChanges="orientation|keyboardHidden|screenLayout" android:launchMode="singleTask" - android:exported="true"> + android:exported="true" + android:process=":testercore"> <meta-data android:name="android.app.lib_name" - android:value="deqp" /> + android:value="khronosopenglcts" /> <meta-data android:name="android.app.func_name" android:value="createGL45CTSActivity" /> <intent-filter> @@ -111,9 +117,10 @@ android:label="GL4.6 CTS" android:configChanges="orientation|keyboardHidden|screenLayout" android:launchMode="singleTask" - android:exported="true"> + android:exported="true" + android:process=":testercore"> <meta-data android:name="android.app.lib_name" - android:value="deqp" /> + android:value="khronosopenglcts" /> <meta-data android:name="android.app.func_name" android:value="createGL46CTSActivity" /> <intent-filter> @@ -126,4 +133,13 @@ <uses-feature android:glEsVersion="0x00030001"/> <uses-permission android:name="android.permission.FOREGROUND_SERVICE" /> <uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" /> + <uses-permission android:name="android.permission.READ_EXTERNAL_STORAGE" /> + <uses-permission android:name="android.permission.MANAGE_EXTERNAL_STORAGE" /> + <uses-permission android:name="android.permission.GET_TASKS" /> + <uses-permission android:name="android.permission.INTERNET" /> + <uses-permission android:name="android.permission.RUN_INSTRUMENTATION" /> + <uses-permission android:name="android.permission.SYSTEM_ALERT_WINDOW" /> + <instrumentation android:label="KhronosCTS-Instrumentation" + android:name="org.khronos.cts.testercore.KhronosCTSInstrumentation" + android:targetPackage="org.khronos.gl_cts"/> </manifest>
diff --git a/android/openglcts/AndroidTest.xml b/android/openglcts/AndroidTest.xml new file mode 100644 index 0000000..6a39760 --- /dev/null +++ b/android/openglcts/AndroidTest.xml
@@ -0,0 +1,10 @@ +<?xml version="1.0" encoding="utf-8"?> +<configuration> + <target_preparer class="com.android.tradefed.targetprep.suite.SuiteApkInstaller"> + <option name="cleanup-apks" value="true"/> + <option name="test-file-name" value="org.khronos.gl_cts.apk"/> + </target_preparer> + <test class="org.khronos.cts.runner.KhronosCTSRunner"> + <option name="collect-raw-logs" value="false"/> + </test> +</configuration>
diff --git a/android/openglcts/runner/src/org/khronos/cts/runner/KhronosCTSBatchRunConfiguration.java b/android/openglcts/runner/src/org/khronos/cts/runner/KhronosCTSBatchRunConfiguration.java new file mode 100644 index 0000000..c08842c --- /dev/null +++ b/android/openglcts/runner/src/org/khronos/cts/runner/KhronosCTSBatchRunConfiguration.java
@@ -0,0 +1,59 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.khronos.cts.runner; +import com.drawelements.deqp.runner.BatchRunConfiguration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +/** + * Test configuration of dEPQ test instance execution. + */ +public class KhronosCTSBatchRunConfiguration extends BatchRunConfiguration{ + private HashMap<String, String> mRunConfigs; + public KhronosCTSBatchRunConfiguration(HashMap<String, String> runConfigs) { + mRunConfigs = runConfigs; + } + @Override + public String getId() { + List<String> runConfigKeys = new ArrayList<>(mRunConfigs.keySet()); + Collections.sort(runConfigKeys); + StringBuilder runConfigString = new StringBuilder(); + for (String key : runConfigKeys) { + if (runConfigString.length() != 0) { + runConfigString.append(" "); + } + runConfigString.append(key); + runConfigString.append("="); + runConfigString.append(mRunConfigs.get(key)); + } + return runConfigString.toString(); + } + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } else if (!(other instanceof KhronosCTSBatchRunConfiguration)) { + return false; + } else { + return getId().equals(((KhronosCTSBatchRunConfiguration)other).getId()); + } + } + @Override + public int hashCode() { + return getId().hashCode(); + } +}
diff --git a/android/openglcts/runner/src/org/khronos/cts/runner/KhronosCTSRunner.java b/android/openglcts/runner/src/org/khronos/cts/runner/KhronosCTSRunner.java new file mode 100644 index 0000000..c517305 --- /dev/null +++ b/android/openglcts/runner/src/org/khronos/cts/runner/KhronosCTSRunner.java
@@ -0,0 +1,824 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.khronos.cts.runner; + +import com.android.ddmlib.MultiLineReceiver; +import com.drawelements.deqp.runner.DeqpTestRunner; +import com.drawelements.deqp.runner.BatchRunConfiguration; +import com.android.tradefed.config.OptionClass; +import com.android.tradefed.result.ITestInvocationListener; +import com.android.tradefed.device.DeviceNotAvailableException; +import com.android.tradefed.log.LogUtil.CLog; +import com.android.tradefed.metrics.proto.MetricMeasurement.Metric; +import com.android.tradefed.result.ByteArrayInputStreamSource; +import com.android.tradefed.result.LogDataType; +import com.android.tradefed.result.TestDescription; +import com.android.tradefed.util.AbiUtils; +import com.android.tradefed.util.FileUtil; +import com.android.tradefed.util.RunInterruptedException; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; +import java.util.Set; + +import javax.annotation.Nullable; + + +@OptionClass(alias="khronos-gl-cts-test-runner") +public class KhronosCTSRunner extends DeqpTestRunner { + @Nullable + private List<String> mTestRunParams = null; + + private static final String KHRONOS_CTS_ONDEVICE_PKG = "org.khronos.gl_cts"; + + private static final String TEST_ID_NAME = "KhronosGLCTS"; + + @Nullable + private Map<TestDescription, Set<KhronosCTSBatchRunConfiguration>> mTestInstances = null; + + private final KhronosCTSTestInstanceResultListener mInstanceListener = new KhronosCTSTestInstanceResultListener(); + + /** + * Test instance listerer and invocation result forwarded. + * Declared as private nested class of KhronosCTSRunner because + * it can access the private KhronosCTSRunner members such as + * mRemainingTests and mPendingResults + */ + private class KhronosCTSTestInstanceResultListener extends DeqpTestRunner.TestInstanceResultListener { + private KhronosCTSBatchRunConfiguration mKhronosCTSRunConfig; + private class KhronosCTSPendingResult { + boolean allInstancesPassed; + Map<KhronosCTSBatchRunConfiguration, String> testLogs; + Map<KhronosCTSBatchRunConfiguration, String> errorMessages; + Set<KhronosCTSBatchRunConfiguration> remainingConfigs; + } + private final Map<TestDescription, KhronosCTSPendingResult> mKhronosCTSPendingResults = new HashMap<>(); + + @Override + public void setCurrentConfig(BatchRunConfiguration runConfig) { + mKhronosCTSRunConfig = (KhronosCTSBatchRunConfiguration)runConfig; + } + + /** + * Forward result to Tradefed Test Invocation Listener + */ + @Override + protected void forwardFinalizedPendingResult(TestDescription testId) { + if (mRemainingTests.contains(testId)) { + final KhronosCTSPendingResult result = mKhronosCTSPendingResults.get(testId); + mKhronosCTSPendingResults.remove(testId); + mRemainingTests.remove(testId); + // Forward results to the sink + mSink.testStarted(testId); + // Test Log + if (mLogData) { + for (Map.Entry<KhronosCTSBatchRunConfiguration, String> entry : + result.testLogs.entrySet()) { + final ByteArrayInputStreamSource source + = new ByteArrayInputStreamSource(entry.getValue().getBytes()); + mSink.testLog(testId.getClassName() + "." + testId.getTestName() + "@" + + entry.getKey().getId(), LogDataType.XML, source); + source.close(); + } + } + // Error message + if (!result.allInstancesPassed) { + final StringBuilder errorLog = new StringBuilder(); + for (Map.Entry<KhronosCTSBatchRunConfiguration, String> entry : + result.errorMessages.entrySet()) { + if (errorLog.length() > 0) { + errorLog.append('\n'); + } + errorLog.append(String.format("=== with config %s ===\n", + entry.getKey().getId())); + errorLog.append(entry.getValue()); + } + mSink.testFailed(testId, errorLog.toString()); + } + final HashMap<String, Metric> emptyMap = new HashMap<>(); + mSink.testEnded(testId, emptyMap); + } + } + + /** + * Declare existence of a test and instances + */ + public void setTestInstancesExperiment(TestDescription testId, Set<KhronosCTSBatchRunConfiguration> configs) { + + // Test instances cannot change at runtime, ignore if we have already set this + if (!mKhronosCTSPendingResults.containsKey(testId)) { + final KhronosCTSPendingResult pendingResult = new KhronosCTSPendingResult(); + pendingResult.allInstancesPassed = true; + pendingResult.testLogs = new LinkedHashMap<>(); + pendingResult.errorMessages = new LinkedHashMap<>(); + pendingResult.remainingConfigs = new HashSet<>(configs); // avoid mutating argument + mKhronosCTSPendingResults.put(testId, pendingResult); + } + } + + /** + * Query if test instance has not yet been executed + */ + @Override + public boolean isPendingTestInstance(TestDescription testId, + BatchRunConfiguration config) { + KhronosCTSBatchRunConfiguration khronosCTSBatchConfig = (KhronosCTSBatchRunConfiguration)config; + final KhronosCTSPendingResult result = mKhronosCTSPendingResults.get(testId); + if (result == null) { + // test is not in the current working batch of the runner, i.e. it cannot be + // "partially" completed. + if (!mRemainingTests.contains(testId)) { + // The test has been fully executed. Not pending. + return false; + } else { + // Test has not yet been executed. Check if such instance exists + return mTestInstances.get(testId).contains(khronosCTSBatchConfig); + } + } else { + // could be partially completed, check this particular config + return result.remainingConfigs.contains(config); + } + } + + /** + * Fake failure of an instance with current config + */ + @Override + public void abortTest(TestDescription testId, String errorMessage) { + final KhronosCTSPendingResult result = mKhronosCTSPendingResults.get(testId); + // Mark as executed + result.allInstancesPassed = false; + result.errorMessages.put(mKhronosCTSRunConfig, errorMessage); + result.remainingConfigs.remove(mKhronosCTSRunConfig); + // Pending result finished, report result + if (result.remainingConfigs.isEmpty()) { + forwardFinalizedPendingResult(testId); + } + if (testId.equals(mCurrentTestId)) { + mCurrentTestId = null; + } + } + + /** + * Handles beginning of dEQP testcase. + */ + @Override + protected void handleBeginTestCase(Map<String, String> values) { + mCurrentTestId = pathToIdentifier(values.get("dEQP-BeginTestCase-TestCasePath")); + mCurrentTestLog = ""; + mGotTestResult = false; + // mark instance as started + if (mKhronosCTSPendingResults.get(mCurrentTestId) != null) { + mKhronosCTSPendingResults.get(mCurrentTestId).remainingConfigs.remove(mKhronosCTSRunConfig); + } else { + CLog.w("Got unexpected start of %s", mCurrentTestId); + } + } + + /** + * Handles end of dEQP testcase. + */ + @Override + protected void handleEndTestCase(Map<String, String> values) { + final KhronosCTSPendingResult result = mKhronosCTSPendingResults.get(mCurrentTestId); + if (result != null) { + if (!mGotTestResult) { + result.allInstancesPassed = false; + result.errorMessages.put(mKhronosCTSRunConfig, INCOMPLETE_LOG_MESSAGE); + } + if (mLogData && mCurrentTestLog != null && mCurrentTestLog.length() > 0) { + result.testLogs.put(mKhronosCTSRunConfig, mCurrentTestLog); + } + // Pending result finished, report result + if (result.remainingConfigs.isEmpty()) { + forwardFinalizedPendingResult(mCurrentTestId); + } + } else { + CLog.w("Got unexpected end of %s", mCurrentTestId); + } + mCurrentTestId = null; + } + + /** + * Handles dEQP testcase result. + */ + @Override + protected void handleTestCaseResult(Map<String, String> values) { + String code = values.get("dEQP-TestCaseResult-Code"); + String details = values.get("dEQP-TestCaseResult-Details"); + if (mKhronosCTSPendingResults.get(mCurrentTestId) == null) { + CLog.w("Got unexpected result for %s", mCurrentTestId); + mGotTestResult = true; + return; + } + if (code.compareTo("Pass") == 0) { + mGotTestResult = true; + } else if (code.compareTo("NotSupported") == 0) { + mGotTestResult = true; + } else if (code.compareTo("QualityWarning") == 0) { + mGotTestResult = true; + } else if (code.compareTo("CompatibilityWarning") == 0) { + mGotTestResult = true; + } else if (code.compareTo("Fail") == 0 || code.compareTo("ResourceError") == 0 + || code.compareTo("InternalError") == 0 || code.compareTo("Crash") == 0 + || code.compareTo("Timeout") == 0) { + mKhronosCTSPendingResults.get(mCurrentTestId).allInstancesPassed = false; + mKhronosCTSPendingResults.get(mCurrentTestId) + .errorMessages.put(mKhronosCTSRunConfig, code + ": " + details); + mGotTestResult = true; + } else { + String codeError = "Unknown result code: " + code; + mKhronosCTSPendingResults.get(mCurrentTestId).allInstancesPassed = false; + mKhronosCTSPendingResults.get(mCurrentTestId) + .errorMessages.put(mKhronosCTSRunConfig, codeError + ": " + details); + mGotTestResult = true; + } + } + + /** + * Handles terminated dEQP testcase. + */ + @Override + protected void handleTestCaseTerminate(Map<String, String> values) { + final KhronosCTSPendingResult result = mKhronosCTSPendingResults.get(mCurrentTestId); + if (result != null) { + String reason = values.get("dEQP-TerminateTestCase-Reason"); + mKhronosCTSPendingResults.get(mCurrentTestId).allInstancesPassed = false; + mKhronosCTSPendingResults.get(mCurrentTestId) + .errorMessages.put(mKhronosCTSRunConfig, "Terminated: " + reason); + // Pending result finished, report result + if (result.remainingConfigs.isEmpty()) { + forwardFinalizedPendingResult(mCurrentTestId); + } + } else { + CLog.w("Got unexpected termination of %s", mCurrentTestId); + } + mCurrentTestId = null; + mGotTestResult = true; + } + + private void handleTestRunParams(Map<String, String> values) { + mTestRunParams.add(values.get("dEQP-TestRunParam")); + } + + /** + * Handles new instrumentation status message. + */ + @Override + public void handleStatus(Map<String, String> values) { + String eventType = values.get("dEQP-EventType"); + if (eventType == null) { + return; + } + if (eventType.compareTo("BeginSession") == 0) { + handleBeginSession(values); + } else if (eventType.compareTo("EndSession") == 0) { + handleEndSession(values); + } else if (eventType.compareTo("BeginTestCase") == 0) { + handleBeginTestCase(values); + } else if (eventType.compareTo("EndTestCase") == 0) { + handleEndTestCase(values); + } else if (eventType.compareTo("TestCaseResult") == 0) { + handleTestCaseResult(values); + } else if (eventType.compareTo("TerminateTestCase") == 0) { + handleTestCaseTerminate(values); + } else if (eventType.compareTo("TestLogData") == 0) { + handleTestLogData(values); + } else if (eventType.compareTo("BeginTestRunParams") == 0) { + handleTestRunParams(values); + } + } + + /** + * Signal listener that batch ended and forget incomplete results. + */ + @Override + public void endBatch() { + // end open test if when stream ends + if (mCurrentTestId != null) { + // Current instance was removed from remainingConfigs when case + // started. Mark current instance as pending. + if (mKhronosCTSPendingResults.get(mCurrentTestId) != null) { + mKhronosCTSPendingResults.get(mCurrentTestId).remainingConfigs.add(mKhronosCTSRunConfig); + } else { + CLog.w("Got unexpected internal state of %s", mCurrentTestId); + } + } + mCurrentTestId = null; + } + } + + private static class KhronosCTSTestBatch extends DeqpTestRunner.TestBatch{ + private KhronosCTSBatchRunConfiguration mConfig; + + @Override + public KhronosCTSBatchRunConfiguration getTestBatchConfig() {return mConfig;} + + @Override + public void setTestBatchConfig(BatchRunConfiguration config){ + mConfig = (KhronosCTSBatchRunConfiguration)config; + } + } + + /** + * KhronosCTS instrumentation parser + */ + private static class KhronosCTSInstrumentationParser extends DeqpTestRunner.InstrumentationParser { + + public KhronosCTSInstrumentationParser(TestInstanceResultListener listener) { + assert (listener instanceof KhronosCTSTestInstanceResultListener); + mListener = listener; + } + } + + // Constructor + public KhronosCTSRunner(){ + + } + + @Override + protected TestInstanceResultListener getInstanceListener() {return mInstanceListener;} + + @Override + protected int getNumRemainingInstances() { + int retVal = 0; + for (TestDescription testId : mRemainingTests) { + // If case is in current working set, sum only not yet executed instances. + // If case is not in current working set, sum all instances (since they are not yet + // executed). + if (mInstanceListener.mKhronosCTSPendingResults.containsKey(testId)) { + retVal += mInstanceListener.mKhronosCTSPendingResults.get(testId).remainingConfigs.size(); + } else { + retVal += mTestInstances.get(testId).size(); + } + } + return retVal; + } + + @Override + protected String getId() { + return AbiUtils.createId(mAbi.getName(), TEST_ID_NAME); + } + + @Override + protected String getRunConfigDisplayCmdLine(BatchRunConfiguration runConfig) + { + assert(runConfig instanceof KhronosCTSBatchRunConfiguration); + return runConfig.getId(); + } + + private Set<KhronosCTSBatchRunConfiguration> getTestRunConfigs(TestDescription testId) { + return mTestInstances.get(testId); + } + + /** + * Checks if a given test should be removed from the test KhronosCTS test run + */ + private static boolean removeTestFromFilters(TestDescription test, + List<String> includeFilters, + List<String> excludeFilters) { + // We could filter faster by building the test case tree. + // Let's see if this is fast enough. + Set<String> includeStrings = getNonPatternFilters(includeFilters); + Set<String> excludeStrings = getNonPatternFilters(excludeFilters); + List<Pattern> includePatterns = getPatternFilters(includeFilters); + List<Pattern> excludePatterns = getPatternFilters(excludeFilters); + if (excludeStrings.contains(test.toString())) { + return true; + } + boolean includesExist = !includeStrings.isEmpty() || !includePatterns.isEmpty(); + boolean testIsIncluded = includeStrings.contains(test.toString()) + || matchesAny(test, includePatterns); + if ((includesExist && !testIsIncluded) || matchesAny(test, excludePatterns)) { + // if this test isn't included and other tests are, + // or if test matches exclude pattern, exclude test + return true; + } + return false; + } + + /** + * Executes given test batch on a device + */ + @Override + protected void executeTestRunBatch(TestBatch batch) throws DeviceNotAvailableException + { + assert(batch instanceof KhronosCTSTestBatch); + final String instrumentationName = + "org.khronos.gl_cts/org.khronos.cts.testercore.KhronosCTSInstrumentation"; + final StringBuilder khronosCTSCmdLine = new StringBuilder(); + khronosCTSCmdLine.append("--deqp-caselist-file="); + khronosCTSCmdLine.append(APP_DIR + CASE_LIST_FILE_NAME); + khronosCTSCmdLine.append(" "); + khronosCTSCmdLine.append(getRunConfigDisplayCmdLine(batch.getTestBatchConfig())); + // If we are not logging data, do not bother outputting the images from the test exe. + if (!mLogData) { + khronosCTSCmdLine.append(" --deqp-log-images=disable"); + } + final String command = String.format( + "am instrument %s -w -e khronosCTSLogFileName \"%s\" -e khronosCTSCmdLine \"%s\" -e deqpLogData \"%s\" %s", + AbiUtils.createAbiFlag(mAbi.getName()), APP_DIR + LOG_FILE_NAME, khronosCTSCmdLine.toString(), mLogData, instrumentationName); + final KhronosCTSInstrumentationParser parser = new KhronosCTSInstrumentationParser(getInstanceListener()); + // attempt full run once + executeTestRunBatchRun(batch, instrumentationName, command, parser); + + // split remaining tests to two sub batches and execute both. This will + // terminate since executeTestRunBatchRun will always progress for a + // batch of size 1. + final ArrayList<TestDescription> pendingTests = new ArrayList<>(); + + for (TestDescription test : batch.getTestBatchTestDescriptionList()) { + if (getInstanceListener().isPendingTestInstance(test, batch.getTestBatchConfig())) { + pendingTests.add(test); + } + } + final int divisorNdx = pendingTests.size() / 2; + final List<TestDescription> headList = pendingTests.subList(0, divisorNdx); + final List<TestDescription> tailList = pendingTests.subList(divisorNdx, pendingTests.size()); + // head + for (;;) { + TestBatch subBatch = selectRunBatch(headList, batch.getTestBatchConfig()); + if (subBatch == null) { + break; + } + executeTestRunBatch(subBatch); + } + // tail + for (;;) { + TestBatch subBatch = selectRunBatch(tailList, batch.getTestBatchConfig()); + if (subBatch == null) { + break; + } + executeTestRunBatch(subBatch); + } + if (getBatchNumPendingCases(batch) != 0) { + throw new AssertionError("executeTestRunBatch postcondition failed"); + } + } + + /** + * Runs a TestBatch by executing it on a device + */ + @Override + protected void runTestRunBatch(TestBatch batch) throws DeviceNotAvailableException{ + // prepare instance listener + assert(batch instanceof KhronosCTSTestBatch); + getInstanceListener().setCurrentConfig(batch.getTestBatchConfig()); + for (TestDescription test: batch.getTestBatchTestDescriptionList()) { + mInstanceListener.setTestInstancesExperiment(test, getTestRunConfigs(test)); + } + executeTestRunBatch(batch); + } + + /** + * Creates a KhronosCTSTestBatch from the given tests or null if not tests remaining. + * + * @param pool List of tests to select from + * @param requiredConfig Select only instances with pending requiredConfig, or null to select + * any run configuration. + */ + @Override + protected TestBatch selectRunBatch(Collection<TestDescription> pool, + BatchRunConfiguration requiredConfig) { + assert(requiredConfig instanceof KhronosCTSBatchRunConfiguration); + // select one test (leading test) that is going to be executed and then pack along as many + // other compatible instances as possible. + TestDescription leadingTest = null; + for (TestDescription test : pool) { + if (!mRemainingTests.contains(test)) { + continue; + } + if (requiredConfig != null && !getInstanceListener().isPendingTestInstance(test, requiredConfig)) { + continue; + } + leadingTest = test; + break; + } + // no remaining tests? + if (leadingTest == null) { + return null; + } + BatchRunConfiguration leadingTestConfig = null; + if (requiredConfig != null) { + leadingTestConfig = requiredConfig; + } else { + for (KhronosCTSBatchRunConfiguration runConfig : getTestRunConfigs(leadingTest)) { + if (getInstanceListener().isPendingTestInstance(leadingTest, runConfig)) { + leadingTestConfig = runConfig; + break; + } + } + } + // test pending <=> test has a pending config + if (leadingTestConfig == null) { + throw new AssertionError("search postcondition failed"); + } + final int leadingInstability = getTestInstabilityRating(leadingTest); + final KhronosCTSTestBatch runBatch = new KhronosCTSTestBatch(); + runBatch.setTestBatchConfig(leadingTestConfig); + List<TestDescription> runBatchTests = new ArrayList<>(); + runBatchTests.add(leadingTest); + for (TestDescription test : pool) { + if (test == leadingTest) { + // do not re-select the leading tests + continue; + } + if (!getInstanceListener().isPendingTestInstance(test, leadingTestConfig)) { + // select only compatible + continue; + } + if (getTestInstabilityRating(test) != leadingInstability) { + // pack along only cases in the same stability category. Packing more dangerous + // tests along jeopardizes the stability of this run. Packing more stable tests + // along jeopardizes their stability rating. + continue; + } + if (runBatchTests.size() >= getBatchSizeLimitForInstability(leadingInstability)) { + // batch size is limited. + break; + } + runBatchTests.add(test); + } + runBatch.setTestBatchTestDescriptionList(runBatchTests); + return runBatch; + } + + /** + * Executes all tests on the device. + */ + private void runTests() + throws DeviceNotAvailableException { + for (;;) { + TestBatch batch = selectRunBatch(mTestInstances.keySet(), null); + + if (batch == null) { + break; + } + + runTestRunBatch(batch); + } + } + + /** + * helper function to read the test names from testlist and add them to the instances map + */ + private void addTestsToInstancesMap(File testlist, KhronosCTSBatchRunConfiguration runConfig, Map<TestDescription, Set<KhronosCTSBatchRunConfiguration>> instances) + { + try (final FileReader testlistInnerReader = new FileReader(testlist); + final BufferedReader testlistReader = new BufferedReader(testlistInnerReader)) { + String testName; + while ((testName = testlistReader.readLine()) != null) { + testName = testName.trim(); + // Skip empty lines. + if (testName.isEmpty()) { + continue; + } + // Lines starting with "#" are comments. + if (testName.startsWith("#")) { + continue; + } + TestDescription test = pathToIdentifier(testName); + if (removeTestFromFilters(test, mIncludeFilters, mExcludeFilters)) + { + continue; + } + Set<KhronosCTSBatchRunConfiguration> testInstanceConfigSet = instances.get(test); + if (testInstanceConfigSet!=null) + { + testInstanceConfigSet.add(runConfig); + } + else + { + testInstanceConfigSet = new LinkedHashSet<>(); + testInstanceConfigSet.add(runConfig); + } + assert testInstanceConfigSet != null; + instances.put(test, testInstanceConfigSet); + } + } + catch (IOException e) + { + throw new RuntimeException("Failure while reading the test case list: " + e.getMessage()); + } + } + + /** + * Helper function to load test names from caseListResourceFileName + */ + private void LoadTestsFromCaselistResource(String caseListResourceFileName, KhronosCTSBatchRunConfiguration runConfig, Map<TestDescription, Set<KhronosCTSBatchRunConfiguration>> instances) + { + try { + String[] paths = caseListResourceFileName.split("/"); + File directoryToSearch = mBuildHelper.getTestsDir(); + for (int i = 0; i < paths.length - 1; ++i) + { + File dir = FileUtil.findDirectory(paths[i], directoryToSearch); + if (dir==null) + { + throw new FileNotFoundException("Cannot find deqp test list file: " + + caseListResourceFileName); + } + directoryToSearch = dir; + } + String fileName = paths[paths.length-1]; + File testlist = new File(directoryToSearch, fileName); + if (testlist == null || !testlist.isFile()) { + throw new FileNotFoundException("Cannot find deqp test list file: " + + caseListResourceFileName); + } + addTestsToInstancesMap(testlist, runConfig, instances); + } + catch (FileNotFoundException e) { + throw new RuntimeException("Cannot read deqp test list file: " + caseListResourceFileName); + } + catch (IOException e) { + throw new RuntimeException("Cannot read deqp test list file: " + caseListResourceFileName); + } + } + + private KhronosCTSBatchRunConfiguration parseRunParam(String runParam) + { + int index = 0; + HashMap<String, String> runConfigParam = new HashMap<String, String>(); + while (index < runParam.length()) + { + int nextParamArgValIndex = runParam.substring(index).indexOf(','); + if (nextParamArgValIndex < 0) + { + break; + } + String nextParamArgVal = runParam.substring(index, index + nextParamArgValIndex); + int argValDivIndex = nextParamArgVal.indexOf('='); + String nextParamArg = nextParamArgVal.substring(0, argValDivIndex); + String nextParamVal = nextParamArgVal.substring(argValDivIndex + 1); + runConfigParam.put(nextParamArg, nextParamVal); + index = index + nextParamArgValIndex+1; + } + return new KhronosCTSBatchRunConfiguration(runConfigParam); + } + + private void generateTestInstanceWithTestRunParameters(List<String> testRunParamLists) { + if (mTestInstances != null) { + throw new AssertionError("Re-load of tests not supported"); + } + + // Note: This is specifically a LinkedHashMap to guarantee that tests + // are iterated in the insertion order. + mTestInstances = new LinkedHashMap<>(); + + final String testRunParamArgCaseListResource = "--deqp-caselist-resource"; + for (String testRunParam : testRunParamLists){ + CLog.i("Debug testRunParam is %s", testRunParam); + int indexOfCaseListFileBegin = testRunParam.indexOf(testRunParamArgCaseListResource); + if (indexOfCaseListFileBegin == -1) + { + continue; + } + int indexOfCaseListFileEnd = testRunParam.substring(indexOfCaseListFileBegin).indexOf(','); + String caseListFileArgValue = testRunParam.substring(indexOfCaseListFileBegin, indexOfCaseListFileEnd); + int equalIndex = caseListFileArgValue.indexOf('='); + if (equalIndex == -1) { + throw new RuntimeException("Invalid caseListFileArgValue. Accepted format is --deqp-caselist-resource=filename"); + } + String caseListFileName = caseListFileArgValue.substring(equalIndex+1); + String runParam = testRunParam.substring(indexOfCaseListFileEnd+1); + KhronosCTSBatchRunConfiguration runConfig = parseRunParam(runParam); + CLog.i("Debug runConfig is %s", runConfig.getId()); + LoadTestsFromCaselistResource(caseListFileName, runConfig, mTestInstances); + } + } + + /** + * Run Android activity org.khronos.cts.ES32GetTestParamActivity through instrumentation process + */ + private void runGetTestsParamsActivity() throws DeviceNotAvailableException { + if (mTestRunParams != null) throw new AssertionError("Re-load of test run params not supported"); + mTestRunParams = new ArrayList(); + checkInterrupted(); // throws if interrupted + final String instrumentationName = + "org.khronos.gl_cts/org.khronos.cts.testercore.KhronosCTSInstrumentation"; + final String getTestsParamsActivity = + "org.khronos.gl_cts/org.khronos.cts.ES32GetTestParamActivity"; + final String testsParamsFileName = + "/sdcard/cts-test-params.xml"; + final String command = String.format( + "am instrument %s -w -e khronosCTSTestName \"%s\" -e khronosCTSTestParamFileName \"%s\" %s", + AbiUtils.createAbiFlag(mAbi.getName()), + getTestsParamsActivity, + testsParamsFileName, + instrumentationName); + final KhronosCTSInstrumentationParser parser = new KhronosCTSInstrumentationParser(getInstanceListener()); + Throwable interruptingError = null; + try { + executeShellCommandAndReadOutput(command, parser); + } catch (Throwable ex) { + CLog.e("runGetTestsParamsActivity() executeShellCommandAndReadOutput() throws exception"); + interruptingError = ex; + } finally { + parser.flush(); + } + // Check interruption error, e.g. adb device lost + if (interruptingError != null) + { + if (interruptingError instanceof AdbComLinkOpenError) { + CLog.e("runGetTestsParamsActivity() interruptingError is AdbComLinkOpenError"); + mDeviceRecovery.recoverConnectionRefused(); + } else if (interruptingError instanceof AdbComLinkKilledError) { + CLog.e("runGetTestsParamsActivity() interruptingError is AdbComLinkKilledError"); + mDeviceRecovery.recoverComLinkKilled(); + } else if (interruptingError instanceof RunInterruptedException) { + CLog.e("runGetTestsParamsActivity() interruptingError is RunInterruptedException"); + throw (RunInterruptedException)interruptingError; + } else { + CLog.e("runGetTestsParamsActivity() interruptingError is other error"); + CLog.e(interruptingError); + throw new RuntimeException(interruptingError); + } + } else if (!parser.wasSuccessful()) { + CLog.e("runGetTestsParamsActivity() parser.was not successful"); + mDeviceRecovery.recoverComLinkKilled(); + } + } + + /** + * {@inheritDoc} + */ + @Override + public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { + final HashMap<String, Metric> emptyMap = new HashMap<>(); + long startTime = System.currentTimeMillis(); + setupTestEnvironment(KHRONOS_CTS_ONDEVICE_PKG); + try { + mDeviceRecovery.setDevice(mDevice); + + // run the activity to retrieve the test run params first + if (mTestRunParams == null) { + runGetTestsParamsActivity(); + } + + // sanity check runGetTestsParamsActivity completed successfully + if (mTestRunParams == null || mTestRunParams.isEmpty()) + { + CLog.e("Debug Failed to load test run parameters"); + throw new RuntimeException("Failed to load test run parameters, abort the rest of tests"); + } + + if (mTestInstances == null) { + generateTestInstanceWithTestRunParameters(mTestRunParams); + } + + mRemainingTests = new HashSet<>(mTestInstances.keySet()); + CLog.d("Debug total test to run: %d", mRemainingTests.size()); + + } catch (Exception ex) { + CLog.e("Exception while generating test run parameters: %s", ex.getMessage()); + teardownTestEnvironment(); + return; + } + + listener.testRunStarted(getId(), mRemainingTests.size()); + + try { + if (mRemainingTests.isEmpty()) { + CLog.d("No tests to run"); + } else { + getInstanceListener().setSink(listener); + runTests(); + } + + } catch (Exception ex) { + // Platform is not behaving correctly, for example crashing when trying to create + // a window. Instead of silently failing, signal failure by leaving the rest of the + // test cases in "NotExecuted" state + CLog.e("Exception while running tests: %s", ex.getMessage()); + } finally { + listener.testRunEnded(System.currentTimeMillis() - startTime, emptyMap); + teardownTestEnvironment(); + return; + } + } +}
diff --git a/android/openglcts/runner/tests/Android.bp b/android/openglcts/runner/tests/Android.bp new file mode 100644 index 0000000..5356b88 --- /dev/null +++ b/android/openglcts/runner/tests/Android.bp
@@ -0,0 +1,38 @@ +// Copyright (C) 2024 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package { + // See: http://go/android-license-faq + // A large-scale-change added 'default_applicable_licenses' to import + // all of the 'license_kinds' from "external_deqp_license" + // to get the below license kinds: + // SPDX-license-identifier-Apache-2.0 + default_applicable_licenses: ["external_deqp_license"], +} + +java_test_host { + name: "KhronosCTSRunnerTests", + team: "trendy_team_android_gpu", + // Only compile source java files in this lib. + srcs: ["src/**/*.java"], + + libs: [ + "cts-tradefed", + "compatibility-host-util", + "tradefed", + "KhronosCTSTestCases", + "CtsDeqpTestCases", + ], + static_libs: ["easymock"], +}
diff --git a/android/openglcts/runner/tests/src/org/khronos/cts/runner/KhronosCTSRunnerTests.java b/android/openglcts/runner/tests/src/org/khronos/cts/runner/KhronosCTSRunnerTests.java new file mode 100644 index 0000000..84cc30d --- /dev/null +++ b/android/openglcts/runner/tests/src/org/khronos/cts/runner/KhronosCTSRunnerTests.java
@@ -0,0 +1,1151 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.khronos.cts.runner; + +import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper; +import com.android.ddmlib.IDevice; +import com.android.ddmlib.IShellOutputReceiver; +import com.android.tradefed.build.IFolderBuildInfo; +import com.android.tradefed.config.ConfigurationException; +import com.android.tradefed.config.OptionSetter; +import com.android.tradefed.device.DeviceNotAvailableException; +import com.android.tradefed.device.IManagedTestDevice; +import com.android.tradefed.device.ITestDevice; +import com.android.tradefed.metrics.proto.MetricMeasurement.Metric; +import com.android.tradefed.result.ITestInvocationListener; +import com.android.tradefed.result.TestDescription; +import com.android.tradefed.testtype.Abi; +import com.android.tradefed.testtype.IAbi; +import com.android.tradefed.util.AbiUtils; +import com.android.tradefed.util.FileUtil; + +import com.drawelements.deqp.runner.DeqpTestRunner; + +import junit.framework.TestCase; + +import org.easymock.EasyMock; +import org.easymock.IAnswer; +import org.easymock.IMocksControl; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +/** + * Unit tests for {@link KhronosCTSRunner}. + * Note: This file is a copy of DeqpTestRunnerTest.java with a few changed types. + */ +public class KhronosCTSRunnerTests extends TestCase { + private static final IAbi ABI = new Abi("armeabi-v7a", "32"); + private static final String APP_DIR = "/sdcard/"; + private static final String CASE_LIST_FILE_NAME = "dEQP-TestCaseList.txt"; + private static final String LOG_FILE_NAME = "TestLog.qpa"; + private static final String TEST_RUN_FILE_AND_PARAM = "--deqp-caselist-resource=gles3-caselist.txt,--deqp-screen-rotation=unspecified,--deqp-surface-width=256,--deqp-surface-height=256,--deqp-watchdog=disable,--deqp-gl-config-name=rgba8888d24s8ms0,"; + private static final String TEST_FAILURE_MESSAGE_CONFIG = "=== with config --deqp-gl-config-name=rgba8888d24s8ms0 --deqp-screen-rotation=unspecified --deqp-surface-height=256 --deqp-surface-width=256 --deqp-watchdog=disable ==="; + private static final String TEST_ID_NAME = "KhronosGLCTS"; + + private File mTestsDir = null; + + private boolean mLogData; + + public static class BuildHelperMock extends CompatibilityBuildHelper { + private File mTestsDir = null; + public BuildHelperMock(IFolderBuildInfo buildInfo, File testsDir) { + super(buildInfo); + mTestsDir = testsDir; + } + @Override + public File getTestsDir() throws FileNotFoundException { + return mTestsDir; + } + } + + private static CompatibilityBuildHelper getMockBuildHelper(File testsDir) { + IFolderBuildInfo mockIFolderBuildInfo = EasyMock.createMock(IFolderBuildInfo.class); + EasyMock.expect(mockIFolderBuildInfo.getBuildAttributes()).andReturn(new HashMap<>()).anyTimes(); + EasyMock.replay(mockIFolderBuildInfo); + return new BuildHelperMock(mockIFolderBuildInfo, testsDir); + } + + private static KhronosCTSBatchRunConfiguration parseRunParam(String runParam) + { + int index = 0; + HashMap<String, String> runConfigParam = new HashMap<String, String>(); + while (index < runParam.length()) + { + int nextParamArgValIndex = runParam.substring(index).indexOf(','); + if (nextParamArgValIndex < 0) + { + break; + } + String nextParamArgVal = runParam.substring(index, index + nextParamArgValIndex); + int argValDivIndex = nextParamArgVal.indexOf('='); + String nextParamArg = nextParamArgVal.substring(0, argValDivIndex); + String nextParamVal = nextParamArgVal.substring(argValDivIndex + 1); + runConfigParam.put(nextParamArg, nextParamVal); + index = index + nextParamArgValIndex+1; + } + return new KhronosCTSBatchRunConfiguration(runConfigParam); + } + + private static KhronosCTSBatchRunConfiguration parseTestRunParams(String testRunParam) { + final String testRunParamArgCaseListResource = "--deqp-caselist-resource"; + int indexOfCaseListFileBegin = testRunParam.indexOf(testRunParamArgCaseListResource); + if (indexOfCaseListFileBegin == -1) + { + return new KhronosCTSBatchRunConfiguration(new HashMap<String, String>()); + } + int indexOfCaseListFileEnd = testRunParam.substring(indexOfCaseListFileBegin).indexOf(','); + String runParam = testRunParam.substring(indexOfCaseListFileEnd+1); + return parseRunParam(runParam); + } + + + private static KhronosCTSRunner buildKhronosCTSRunner(Collection<TestDescription> tests, File testsDir) throws ConfigurationException, IOException { + StringWriter testlist = new StringWriter(); + for (TestDescription test : tests) { + testlist.write(test.getClassName() + "." + test.getTestName() + "\n"); + } + return buildKhronosCTSRunner(testlist.toString(), testsDir); + } + + private static KhronosCTSRunner buildKhronosCTSRunner(String testlist, File testsDir) throws ConfigurationException, IOException { + KhronosCTSRunner runner = new KhronosCTSRunner(); + final File caselistsFile = new File(testsDir, "gles3-caselist.txt"); + FileUtil.writeToFile(testlist, caselistsFile); + + runner.setAbi(ABI); + runner.setBuildHelper(getMockBuildHelper(testsDir)); + + return runner; + } + + private void runInstrumentationLineAndAnswer(ITestDevice mockDevice, IDevice mockIDevice, + final String output) throws Exception { + + runInstrumentationLineAndAnswer(mockDevice, mockIDevice, null, null, output); + } + + private void runInstrumentationLineAndAnswer(ITestDevice mockDevice, IDevice mockIDevice, + final String testTrie, String cmd, final String output) throws Exception { + if (cmd==null){ + StringBuilder khronosCTSCmdLine = new StringBuilder(); + khronosCTSCmdLine.append("--deqp-caselist-file="); + khronosCTSCmdLine.append(APP_DIR + CASE_LIST_FILE_NAME); + khronosCTSCmdLine.append(" "); + khronosCTSCmdLine.append(parseTestRunParams(TEST_RUN_FILE_AND_PARAM).getId()); + if(!mLogData) { + khronosCTSCmdLine.append(" "); + khronosCTSCmdLine.append("--deqp-log-images=disable"); + } + cmd = khronosCTSCmdLine.toString(); + } + + EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("rm " + APP_DIR + CASE_LIST_FILE_NAME))) + .andReturn("").once(); + + EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("rm " + APP_DIR + LOG_FILE_NAME))) + .andReturn("").once(); + + if (testTrie == null) { + mockDevice.pushString((String)EasyMock.anyObject(), EasyMock.eq(APP_DIR + CASE_LIST_FILE_NAME)); + } + else { + mockDevice.pushString(testTrie + "\n", APP_DIR + CASE_LIST_FILE_NAME); + } + EasyMock.expectLastCall().andReturn(true).once(); + + final String instrumentationName = + "org.khronos.gl_cts/org.khronos.cts.testercore.KhronosCTSInstrumentation"; + + final String command = String.format( + "am instrument %s -w -e khronosCTSLogFileName \"%s\" -e khronosCTSCmdLine \"%s\" -e deqpLogData \"%s\" %s", + AbiUtils.createAbiFlag(ABI.getName()), APP_DIR + LOG_FILE_NAME, cmd, mLogData, instrumentationName); + + EasyMock.expect(mockDevice.getIDevice()).andReturn(mockIDevice); + mockIDevice.executeShellCommand(EasyMock.eq(command), + EasyMock.<IShellOutputReceiver>notNull(), EasyMock.anyLong(), + EasyMock.isA(TimeUnit.class)); + + EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() { + @Override + public Object answer() { + IShellOutputReceiver receiver + = (IShellOutputReceiver)EasyMock.getCurrentArguments()[1]; + + receiver.addOutput(output.getBytes(), 0, output.length()); + receiver.flush(); + + return null; + } + }); + } + + static private String buildTestProcessOutput(List<TestDescription> tests) { + /* MultiLineReceiver expects "\r\n" line ending. */ + final String outputHeader = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" + + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" + + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" + + "INSTRUMENTATION_STATUS_CODE: 0\r\n" + + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" + + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" + + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" + + "INSTRUMENTATION_STATUS_CODE: 0\r\n" + + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" + + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" + + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" + + "INSTRUMENTATION_STATUS_CODE: 0\r\n" + + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" + + "INSTRUMENTATION_STATUS_CODE: 0\r\n"; + + final String outputEnd = "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" + + "INSTRUMENTATION_STATUS_CODE: 0\r\n" + + "INSTRUMENTATION_CODE: 0\r\n"; + + StringWriter output = new StringWriter(); + output.write(outputHeader); + for (TestDescription test : tests) { + output.write("INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"); + output.write("INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath="); + output.write(test.getClassName()); + output.write("."); + output.write(test.getTestName()); + output.write("\r\n"); + output.write("INSTRUMENTATION_STATUS_CODE: 0\r\n"); + output.write("INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"); + output.write("INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"); + output.write("INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"); + output.write("INSTRUMENTATION_STATUS_CODE: 0\r\n"); + output.write("INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"); + output.write("INSTRUMENTATION_STATUS_CODE: 0\r\n"); + } + output.write(outputEnd); + return output.toString(); + } + + + private void runGetTestsParamsInstrumentation(ITestDevice mockDevice, IDevice mockIDevice) throws Exception + { + + final String getTestsParamsActivity = + "org.khronos.gl_cts/org.khronos.cts.ES32GetTestParamActivity"; + + final String testsParamsFileName = + "/sdcard/cts-test-params.xml"; + + final String instrumentationName = + "org.khronos.gl_cts/org.khronos.cts.testercore.KhronosCTSInstrumentation"; + + String command = String.format( + "am instrument %s -w -e khronosCTSTestName \"%s\" -e khronosCTSTestParamFileName \"%s\" %s", + AbiUtils.createAbiFlag(ABI.getName()), + getTestsParamsActivity, + testsParamsFileName, + instrumentationName); + + final String output = "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestRunParamsCollection\r\n" + +"INSTRUMENTATION_STATUS_CODE: 0\r\n" + +"INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestRunParams\r\n" + +"INSTRUMENTATION_STATUS: dEQP-TestRunParam="+TEST_RUN_FILE_AND_PARAM+"\r\n" + +"INSTRUMENTATION_STATUS_CODE: 0\r\n" + +"INSTRUMENTATION_STATUS: dEQP-EventType=EndTestRunParams\r\n" + +"INSTRUMENTATION_STATUS_CODE: 0\r\n" + +"INSTRUMENTATION_STATUS: dEQP-EventType=EndTestRunParamsCollection\r\n" + +"INSTRUMENTATION_STATUS_CODE: 0\r\n" + +"INSTRUMENTATION_CODE: 0\r\n"; + + + EasyMock.expect(mockDevice.getIDevice()).andReturn(mockIDevice); + + mockIDevice.executeShellCommand(EasyMock.eq(command), + EasyMock.<IShellOutputReceiver>notNull(), EasyMock.anyLong(), + EasyMock.isA(TimeUnit.class)); + + EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() { + @Override + public Object answer() { + IShellOutputReceiver receiver + = (IShellOutputReceiver)EasyMock.getCurrentArguments()[1]; + + receiver.addOutput(output.getBytes(), 0, output.length()); + receiver.flush(); + + return null; + } + }); + } + + private static String getTestId() { + return AbiUtils.createId(ABI.getName(), TEST_ID_NAME); + } + + private void testFiltering(KhronosCTSRunner khronosCTSRunner, + String expectedTrie, + List<TestDescription> expectedTests) throws Exception { + ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); + IDevice mockIDevice = EasyMock.createMock(IDevice.class); + ITestInvocationListener mockListener = EasyMock.createStrictMock(ITestInvocationListener.class); + + // Expect the calls twice: setupTestEnvironment() and teardownTestEnvironment() + EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_pkgs"))). + andReturn("").once(); + EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_values"))). + andReturn("").once(); + EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_pkgs"))). + andReturn("").once(); + EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_values"))). + andReturn("").once(); + + // Expect the runGetTestsParamsActivity() is called once, no matter if there is test to be ran + runGetTestsParamsInstrumentation(mockDevice, mockIDevice); + + mockListener.testRunStarted(getTestId(), expectedTests.size()); + EasyMock.expectLastCall().once(); + + boolean thereAreTests = !expectedTests.isEmpty(); + if (thereAreTests) + { + String testOut = buildTestProcessOutput(expectedTests); + runInstrumentationLineAndAnswer(mockDevice, mockIDevice, expectedTrie, null, testOut); + + for (int i = 0; i < expectedTests.size(); i++) { + mockListener.testStarted(EasyMock.eq(expectedTests.get(i))); + EasyMock.expectLastCall().once(); + + mockListener.testEnded(EasyMock.eq(expectedTests.get(i)), + EasyMock.<HashMap<String, Metric>>notNull()); + + EasyMock.expectLastCall().once(); + } + } + + mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull()); + EasyMock.expectLastCall().once(); + + EasyMock.replay(mockDevice, mockIDevice); + EasyMock.replay(mockListener); + + khronosCTSRunner.setDevice(mockDevice); + khronosCTSRunner.run(mockListener); + + EasyMock.verify(mockListener); + EasyMock.verify(mockDevice, mockIDevice); + } + + /** + * Test running multiple test cases + */ + public void testRun_multipleTests() throws Exception { + final TestDescription[] testIds = { + new TestDescription("dEQP-GLES3.info", "vendor"), + new TestDescription("dEQP-GLES3.info", "renderer"), + new TestDescription("dEQP-GLES3.info", "version"), + new TestDescription("dEQP-GLES3.info", "shading_language_version"), + new TestDescription("dEQP-GLES3.info", "extensions"), + new TestDescription("dEQP-GLES3.info", "render_target") + }; + + final String expectedTrie + = "{dEQP-GLES3{info{vendor,renderer,version,shading_language_version,extensions,render_target}}}"; + + List<TestDescription> allTests = new ArrayList<TestDescription>(); + for (TestDescription id : testIds) { + allTests.add(id); + } + + List<TestDescription> activeTests = new ArrayList<TestDescription>(); + for (TestDescription id: testIds) { + activeTests.add(id); + } + + KhronosCTSRunner khronosCTSRunner = buildKhronosCTSRunner(allTests, mTestsDir); + + OptionSetter setter = new OptionSetter(khronosCTSRunner); + mLogData = false; + setter.setOptionValue("deqp-log-result-details", mLogData ? "true" : "false"); + + testFiltering(khronosCTSRunner, expectedTrie, activeTests); + } + + public void testRun_trivialIncludeFilter() throws Exception { + final TestDescription[] testIds = { + new TestDescription("dEQP-GLES3.missing", "no"), + new TestDescription("dEQP-GLES3.missing", "nope"), + new TestDescription("dEQP-GLES3.missing", "donotwant"), + new TestDescription("dEQP-GLES3.pick_me", "yes"), + new TestDescription("dEQP-GLES3.pick_me", "ok"), + new TestDescription("dEQP-GLES3.pick_me", "accepted"), + }; + + List<TestDescription> allTests = new ArrayList<TestDescription>(); + for (TestDescription id : testIds) { + allTests.add(id); + } + + List<TestDescription> activeTests = new ArrayList<TestDescription>(); + activeTests.add(testIds[3]); + activeTests.add(testIds[4]); + activeTests.add(testIds[5]); + + String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}"; + + KhronosCTSRunner khronosCTSRunner = buildKhronosCTSRunner(allTests, mTestsDir); + OptionSetter setter = new OptionSetter(khronosCTSRunner); + setter.setOptionValue("include-filter", "dEQP-GLES3.pick_me#*"); + mLogData = false; + setter.setOptionValue("deqp-log-result-details", mLogData ? "true" : "false"); + + testFiltering(khronosCTSRunner, expectedTrie, activeTests); + } + + public void testRun_trivialExcludeFilter() throws Exception { + final TestDescription[] testIds = { + new TestDescription("dEQP-GLES3.missing", "no"), + new TestDescription("dEQP-GLES3.missing", "nope"), + new TestDescription("dEQP-GLES3.missing", "donotwant"), + new TestDescription("dEQP-GLES3.pick_me", "yes"), + new TestDescription("dEQP-GLES3.pick_me", "ok"), + new TestDescription("dEQP-GLES3.pick_me", "accepted"), + }; + + List<TestDescription> allTests = new ArrayList<TestDescription>(); + for (TestDescription id : testIds) { + allTests.add(id); + } + + List<TestDescription> activeTests = new ArrayList<TestDescription>(); + activeTests.add(testIds[3]); + activeTests.add(testIds[4]); + activeTests.add(testIds[5]); + + String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}"; + + KhronosCTSRunner khronosCTSRunner = buildKhronosCTSRunner(allTests, mTestsDir); + OptionSetter setter = new OptionSetter(khronosCTSRunner); + setter.setOptionValue("exclude-filter", "dEQP-GLES3.missing#*"); + mLogData = false; + setter.setOptionValue("deqp-log-result-details", mLogData ? "true" : "false"); + + testFiltering(khronosCTSRunner, expectedTrie, activeTests); + } + + public void testRun_includeAndExcludeFilter() throws Exception { + final TestDescription[] testIds = { + new TestDescription("dEQP-GLES3.group1", "foo"), + new TestDescription("dEQP-GLES3.group1", "nope"), + new TestDescription("dEQP-GLES3.group1", "donotwant"), + new TestDescription("dEQP-GLES3.group2", "foo"), + new TestDescription("dEQP-GLES3.group2", "yes"), + new TestDescription("dEQP-GLES3.group2", "thoushallnotpass"), + }; + + List<TestDescription> allTests = new ArrayList<TestDescription>(); + for (TestDescription id : testIds) { + allTests.add(id); + } + + List<TestDescription> activeTests = new ArrayList<TestDescription>(); + activeTests.add(testIds[4]); + + String expectedTrie = "{dEQP-GLES3{group2{yes}}}"; + + KhronosCTSRunner khronosCTSRunner = buildKhronosCTSRunner(allTests, mTestsDir); + + OptionSetter setter = new OptionSetter(khronosCTSRunner); + mLogData = false; + setter.setOptionValue("deqp-log-result-details", mLogData ? "true" : "false"); + + Set<String> includes = new HashSet<>(); + includes.add("dEQP-GLES3.group2#*"); + khronosCTSRunner.addAllIncludeFilters(includes); + + Set<String> excludes = new HashSet<>(); + excludes.add("*foo"); + excludes.add("*thoushallnotpass"); + khronosCTSRunner.addAllExcludeFilters(excludes); + testFiltering(khronosCTSRunner, expectedTrie, activeTests); + } + + public void testRun_includeAll() throws Exception { + final TestDescription[] testIds = { + new TestDescription("dEQP-GLES3.group1", "mememe"), + new TestDescription("dEQP-GLES3.group1", "yeah"), + new TestDescription("dEQP-GLES3.group1", "takeitall"), + new TestDescription("dEQP-GLES3.group2", "jeba"), + new TestDescription("dEQP-GLES3.group2", "yes"), + new TestDescription("dEQP-GLES3.group2", "granted"), + }; + + List<TestDescription> allTests = new ArrayList<TestDescription>(); + for (TestDescription id : testIds) { + allTests.add(id); + } + + String expectedTrie = "{dEQP-GLES3{group1{mememe,yeah,takeitall},group2{jeba,yes,granted}}}"; + + KhronosCTSRunner khronosCTSRunner = buildKhronosCTSRunner(allTests, mTestsDir); + + OptionSetter setter = new OptionSetter(khronosCTSRunner); + mLogData = false; + setter.setOptionValue("deqp-log-result-details", mLogData ? "true" : "false"); + + khronosCTSRunner.addIncludeFilter("*"); + testFiltering(khronosCTSRunner, expectedTrie, allTests); + } + + public void testRun_excludeAll() throws Exception { + final TestDescription[] testIds = { + new TestDescription("dEQP-GLES3.group1", "no"), + new TestDescription("dEQP-GLES3.group1", "nope"), + new TestDescription("dEQP-GLES3.group1", "nottoday"), + new TestDescription("dEQP-GLES3.group2", "banned"), + new TestDescription("dEQP-GLES3.group2", "notrecognized"), + new TestDescription("dEQP-GLES3.group2", "-2"), + }; + + List<TestDescription> allTests = new ArrayList<TestDescription>(); + for (TestDescription id : testIds) { + allTests.add(id); + } + + KhronosCTSRunner khronosCTSRunner = buildKhronosCTSRunner(allTests, mTestsDir); + OptionSetter setter = new OptionSetter(khronosCTSRunner); + mLogData = false; + setter.setOptionValue("deqp-log-result-details", mLogData ? "true" : "false"); + khronosCTSRunner.addExcludeFilter("*"); + String expectedTrie = ""; + + List<TestDescription> activeTests = new ArrayList<TestDescription>(); + testFiltering(khronosCTSRunner, expectedTrie, activeTests); + } + + public void testDotToHashConversionInFilters() throws Exception { + final TestDescription[] testIds = { + new TestDescription("dEQP-GLES3.missing", "no"), + new TestDescription("dEQP-GLES3.pick_me", "donotwant"), + new TestDescription("dEQP-GLES3.pick_me", "yes") + }; + + List<TestDescription> allTests = new ArrayList<TestDescription>(); + for (TestDescription id : testIds) { + allTests.add(id); + } + + List<TestDescription> activeTests = new ArrayList<TestDescription>(); + activeTests.add(testIds[2]); + + String expectedTrie = "{dEQP-GLES3{pick_me{yes}}}"; + + KhronosCTSRunner khronosCTSRunner = buildKhronosCTSRunner(allTests, mTestsDir); + OptionSetter setter = new OptionSetter(khronosCTSRunner); + mLogData = false; + setter.setOptionValue("deqp-log-result-details", mLogData ? "true" : "false"); + khronosCTSRunner.addIncludeFilter("dEQP-GLES3.pick_me.yes"); + testFiltering(khronosCTSRunner, expectedTrie, activeTests); + } + + /** + * Test running a unexecutable test. + */ + public void testRun_unexecutableTests() throws Exception { + final String instrumentationAnswerNoExecs = + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" + + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" + + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" + + "INSTRUMENTATION_STATUS_CODE: 0\r\n" + + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" + + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" + + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" + + "INSTRUMENTATION_STATUS_CODE: 0\r\n" + + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" + + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" + + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" + + "INSTRUMENTATION_STATUS_CODE: 0\r\n" + + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" + + "INSTRUMENTATION_STATUS_CODE: 0\r\n" + + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" + + "INSTRUMENTATION_STATUS_CODE: 0\r\n" + + "INSTRUMENTATION_CODE: 0\r\n"; + + final TestDescription[] testIds = { + new TestDescription("dEQP-GLES3.missing", "no"), + new TestDescription("dEQP-GLES3.missing", "nope"), + new TestDescription("dEQP-GLES3.missing", "donotwant"), + }; + + final String[] testPaths = { + "dEQP-GLES3.missing.no", + "dEQP-GLES3.missing.nope", + "dEQP-GLES3.missing.donotwant", + }; + + ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); + ITestInvocationListener mockListener + = EasyMock.createStrictMock(ITestInvocationListener.class); + IDevice mockIDevice = EasyMock.createMock(IDevice.class); + + Collection<TestDescription> allTests = new ArrayList<TestDescription>(); + + for (TestDescription id : testIds) { + allTests.add(id); + } + + KhronosCTSRunner khronosCTSRunner = buildKhronosCTSRunner(allTests, mTestsDir); + OptionSetter setter = new OptionSetter(khronosCTSRunner); + mLogData = false; + setter.setOptionValue("deqp-log-result-details", mLogData ? "true" : "false"); + + // first try + runInstrumentationLineAndAnswer(mockDevice, mockIDevice, + "{dEQP-GLES3{missing{no,nope,donotwant}}}", null, instrumentationAnswerNoExecs); + + // splitting begins + runInstrumentationLineAndAnswer(mockDevice, mockIDevice, + "{dEQP-GLES3{missing{no}}}", null, instrumentationAnswerNoExecs); + runInstrumentationLineAndAnswer(mockDevice, mockIDevice, + "{dEQP-GLES3{missing{nope,donotwant}}}", null, instrumentationAnswerNoExecs); + runInstrumentationLineAndAnswer(mockDevice, mockIDevice, + "{dEQP-GLES3{missing{nope}}}", null, instrumentationAnswerNoExecs); + runInstrumentationLineAndAnswer(mockDevice, mockIDevice, + "{dEQP-GLES3{missing{donotwant}}}", null, instrumentationAnswerNoExecs); + + mockListener.testRunStarted(getTestId(), testPaths.length); + EasyMock.expectLastCall().once(); + + // Expect the calls twice: setupTestEnvironment() and teardownTestEnvironment() + EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_pkgs"))). + andReturn("").once(); + EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_values"))). + andReturn("").once(); + EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_pkgs"))). + andReturn("").once(); + EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_values"))). + andReturn("").once(); + + // Expect the runGetTestsParamsActivity() is called once, no matter if there is test to be ran + runGetTestsParamsInstrumentation(mockDevice, mockIDevice); + + for (int i = 0; i < testPaths.length; i++) { + mockListener.testStarted(EasyMock.eq(testIds[i])); + EasyMock.expectLastCall().once(); + + mockListener.testFailed(EasyMock.eq(testIds[i]), + EasyMock.eq(TEST_FAILURE_MESSAGE_CONFIG+"\n" + + "Abort: Test cannot be executed")); + EasyMock.expectLastCall().once(); + + mockListener.testEnded(EasyMock.eq(testIds[i]), + EasyMock.<HashMap<String, Metric>>notNull()); + EasyMock.expectLastCall().once(); + } + + mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull()); + EasyMock.expectLastCall().once(); + + EasyMock.replay(mockDevice, mockIDevice); + EasyMock.replay(mockListener); + + khronosCTSRunner.setDevice(mockDevice); + khronosCTSRunner.run(mockListener); + + EasyMock.verify(mockListener); + EasyMock.verify(mockDevice, mockIDevice); + } + + /** + * Test that result code produces correctly pass or fail. + */ + private void testResultCode(final String resultCode, boolean pass) throws Exception { + final TestDescription testId = new TestDescription("dEQP-GLES3.info", "version"); + final String testPath = "dEQP-GLES3.info.version"; + final String testTrie = "{dEQP-GLES3{info{version}}}"; + + /* MultiLineReceiver expects "\r\n" line ending. */ + final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" + + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" + + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" + + "INSTRUMENTATION_STATUS_CODE: 0\r\n" + + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" + + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" + + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" + + "INSTRUMENTATION_STATUS_CODE: 0\r\n" + + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" + + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" + + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" + + "INSTRUMENTATION_STATUS_CODE: 0\r\n" + + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" + + "INSTRUMENTATION_STATUS_CODE: 0\r\n" + + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" + + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n" + + "INSTRUMENTATION_STATUS_CODE: 0\r\n" + + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=" + resultCode + "\r\n" + + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Detail" + resultCode + "\r\n" + + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" + + "INSTRUMENTATION_STATUS_CODE: 0\r\n" + + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" + + "INSTRUMENTATION_STATUS_CODE: 0\r\n" + + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" + + "INSTRUMENTATION_STATUS_CODE: 0\r\n" + + "INSTRUMENTATION_CODE: 0\r\n"; + + ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); + ITestInvocationListener mockListener + = EasyMock.createStrictMock(ITestInvocationListener.class); + IDevice mockIDevice = EasyMock.createMock(IDevice.class); + + Collection<TestDescription> allTests = new ArrayList<TestDescription>(); + allTests.add(testId); + + KhronosCTSRunner khronosCTSRunner = buildKhronosCTSRunner(allTests, mTestsDir); + + runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, null, output); + + mockListener.testRunStarted(getTestId(), 1); + EasyMock.expectLastCall().once(); + + // Expect the runGetTestsParamsActivity() is called once, no matter if there is test to be ran + runGetTestsParamsInstrumentation(mockDevice, mockIDevice); + + // Expect the calls twice: setupTestEnvironment() and teardownTestEnvironment() + EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_pkgs"))). + andReturn("").once(); + EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_values"))). + andReturn("").once(); + EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_pkgs"))). + andReturn("").once(); + EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_values"))). + andReturn("").once(); + + mockListener.testStarted(EasyMock.eq(testId)); + EasyMock.expectLastCall().once(); + + if (!pass) { + mockListener.testFailed(testId, + TEST_FAILURE_MESSAGE_CONFIG+"\n" + + resultCode + ": Detail" + resultCode); + + EasyMock.expectLastCall().once(); + } + + mockListener.testEnded(EasyMock.eq(testId), EasyMock.<HashMap<String, Metric>>notNull()); + EasyMock.expectLastCall().once(); + + mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull()); + EasyMock.expectLastCall().once(); + + EasyMock.replay(mockDevice, mockIDevice); + EasyMock.replay(mockListener); + + khronosCTSRunner.setDevice(mockDevice); + khronosCTSRunner.run(mockListener); + + EasyMock.verify(mockListener); + EasyMock.verify(mockDevice, mockIDevice); + } + + /** + * Test Pass result code + */ + public void testRun_resultPass() throws Exception { + testResultCode("Pass", true); + } + + /** + * Test dEQP Fail result code. + */ + public void testRun_resultFail() throws Exception { + testResultCode("Fail", false); + } + + /** + * Test dEQP NotSupported result code. + */ + public void testRun_resultNotSupported() throws Exception { + testResultCode("NotSupported", true); + } + + /** + * Test dEQP QualityWarning result code. + */ + public void testRun_resultQualityWarning() throws Exception { + testResultCode("QualityWarning", true); + } + + /** + * Test dEQP CompatibilityWarning result code. + */ + public void testRun_resultCompatibilityWarning() throws Exception { + testResultCode("CompatibilityWarning", true); + } + + /** + * Test dEQP ResourceError result code. + */ + public void testRun_resultResourceError() throws Exception { + testResultCode("ResourceError", false); + } + + /** + * Test dEQP InternalError result code. + */ + public void testRun_resultInternalError() throws Exception { + testResultCode("InternalError", false); + } + + /** + * Test dEQP Crash result code. + */ + public void testRun_resultCrash() throws Exception { + testResultCode("Crash", false); + } + + /** + * Test dEQP Timeout result code. + */ + public void testRun_resultTimeout() throws Exception { + testResultCode("Timeout", false); + } + + /** + * Test interface to mock Tradefed device types. + */ + public static interface RecoverableTestDevice extends ITestDevice, IManagedTestDevice { + } + + private static enum RecoveryEvent { + PROGRESS, + FAIL_CONNECTION_REFUSED, + FAIL_LINK_KILLED, + } + + private void runRecoveryWithPattern(KhronosCTSRunner.Recovery recovery, RecoveryEvent[] events) + throws DeviceNotAvailableException { + for (RecoveryEvent event : events) { + switch (event) { + case PROGRESS: + recovery.onExecutionProgressed(); + break; + case FAIL_CONNECTION_REFUSED: + recovery.recoverConnectionRefused(); + break; + case FAIL_LINK_KILLED: + recovery.recoverComLinkKilled(); + break; + } + } + } + + private void setRecoveryExpectationWait(KhronosCTSRunner.ISleepProvider mockSleepProvider) { + mockSleepProvider.sleep(EasyMock.gt(0)); + EasyMock.expectLastCall().once(); + } + + private void setRecoveryExpectationKillProcess(RecoverableTestDevice mockDevice, + KhronosCTSRunner.ISleepProvider mockSleepProvider) throws DeviceNotAvailableException { + EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))). + andReturn("root 1234 org.khronos.cts").once(); + + EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("kill -9 1234"))). + andReturn("").once(); + + // Recovery checks if kill failed + mockSleepProvider.sleep(EasyMock.gt(0)); + EasyMock.expectLastCall().once(); + EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))). + andReturn("").once(); + } + + private void setRecoveryExpectationRecovery(RecoverableTestDevice mockDevice) + throws DeviceNotAvailableException { + EasyMock.expect(mockDevice.recoverDevice()).andReturn(true).once(); + } + + private void setRecoveryExpectationReboot(RecoverableTestDevice mockDevice) + throws DeviceNotAvailableException { + mockDevice.reboot(); + EasyMock.expectLastCall().once(); + } + + + private int setRecoveryExpectationOfAConnFailure(RecoverableTestDevice mockDevice, int numConsecutiveErrors) + throws DeviceNotAvailableException { + switch (numConsecutiveErrors) { + case 0: + case 1: + setRecoveryExpectationRecovery(mockDevice); + return 2; + case 2: + setRecoveryExpectationReboot(mockDevice); + return 3; + default: + return 4; + } + } + + private int setRecoveryExpectationOfAComKilled(RecoverableTestDevice mockDevice, + KhronosCTSRunner.ISleepProvider mockSleepProvider, int numConsecutiveErrors) + throws DeviceNotAvailableException { + switch (numConsecutiveErrors) { + case 0: + setRecoveryExpectationWait(mockSleepProvider); + setRecoveryExpectationKillProcess(mockDevice, mockSleepProvider); + return 1; + case 1: + setRecoveryExpectationRecovery(mockDevice); + setRecoveryExpectationKillProcess(mockDevice, mockSleepProvider); + return 2; + case 2: + setRecoveryExpectationReboot(mockDevice); + return 3; + default: + return 4; + } + } + + private void setRecoveryExpectationsOfAPattern(RecoverableTestDevice mockDevice, + KhronosCTSRunner.ISleepProvider mockSleepProvider, RecoveryEvent[] events) + throws DeviceNotAvailableException { + int numConsecutiveErrors = 0; + for (RecoveryEvent event : events) { + switch (event) { + case PROGRESS: + numConsecutiveErrors = 0; + break; + case FAIL_CONNECTION_REFUSED: + numConsecutiveErrors = setRecoveryExpectationOfAConnFailure(mockDevice, numConsecutiveErrors); + break; + case FAIL_LINK_KILLED: + numConsecutiveErrors = setRecoveryExpectationOfAComKilled(mockDevice, + mockSleepProvider, numConsecutiveErrors); + break; + } + } + } + + /** + * Test dEQP runner recovery state machine. + */ + private void testRecoveryWithPattern(boolean expectSuccess, RecoveryEvent...pattern) + throws Exception { + KhronosCTSRunner.Recovery recovery = new KhronosCTSRunner.Recovery(); + IMocksControl orderedControl = EasyMock.createStrictControl(); + RecoverableTestDevice mockDevice = orderedControl.createMock(RecoverableTestDevice.class); + EasyMock.expect(mockDevice.getSerialNumber()).andStubReturn("SERIAL"); + KhronosCTSRunner.ISleepProvider mockSleepProvider = + orderedControl.createMock(KhronosCTSRunner.ISleepProvider.class); + + setRecoveryExpectationsOfAPattern(mockDevice, mockSleepProvider, pattern); + + orderedControl.replay(); + + recovery.setDevice(mockDevice); + recovery.setSleepProvider(mockSleepProvider); + try { + runRecoveryWithPattern(recovery, pattern); + if (!expectSuccess) { + fail("Expected DeviceNotAvailableException"); + } + } catch (DeviceNotAvailableException ex) { + if (expectSuccess) { + fail("Did not expect DeviceNotAvailableException"); + } + } + + orderedControl.verify(); + } + + public void testRecovery_NoEvents() throws Exception { + testRecoveryWithPattern(true); + } + + public void testRecovery_AllOk() throws Exception { + testRecoveryWithPattern(true, RecoveryEvent.PROGRESS, RecoveryEvent.PROGRESS); + } + + // conn fail patterns + + public void testRecovery_OneConnectionFailureBegin() throws Exception { + testRecoveryWithPattern(true, RecoveryEvent.FAIL_CONNECTION_REFUSED, + RecoveryEvent.PROGRESS); + } + + public void testRecovery_TwoConnectionFailuresBegin() throws Exception { + testRecoveryWithPattern(true, RecoveryEvent.FAIL_CONNECTION_REFUSED, + RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.PROGRESS); + } + + public void testRecovery_ThreeConnectionFailuresBegin() throws Exception { + testRecoveryWithPattern(false, RecoveryEvent.FAIL_CONNECTION_REFUSED, + RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_CONNECTION_REFUSED); + } + + public void testRecovery_OneConnectionFailureMid() throws Exception { + testRecoveryWithPattern(true, RecoveryEvent.PROGRESS, + RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.PROGRESS); + } + + public void testRecovery_TwoConnectionFailuresMid() throws Exception { + testRecoveryWithPattern(true, RecoveryEvent.PROGRESS, + RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_CONNECTION_REFUSED, + RecoveryEvent.PROGRESS); + } + + public void testRecovery_ThreeConnectionFailuresMid() throws Exception { + testRecoveryWithPattern(false, RecoveryEvent.PROGRESS, + RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_CONNECTION_REFUSED, + RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.PROGRESS); + } + + // link fail patterns + + public void testRecovery_OneLinkFailureBegin() throws Exception { + testRecoveryWithPattern(true, RecoveryEvent.FAIL_LINK_KILLED, + RecoveryEvent.PROGRESS); + } + + public void testRecovery_TwoLinkFailuresBegin() throws Exception { + testRecoveryWithPattern(true, RecoveryEvent.FAIL_LINK_KILLED, + RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.PROGRESS); + } + + public void testRecovery_ThreeLinkFailuresBegin() throws Exception { + testRecoveryWithPattern(true, RecoveryEvent.FAIL_LINK_KILLED, + RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED, + RecoveryEvent.PROGRESS); + } + + public void testRecovery_FourLinkFailuresBegin() throws Exception { + testRecoveryWithPattern(false, RecoveryEvent.FAIL_LINK_KILLED, + RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED, + RecoveryEvent.FAIL_LINK_KILLED); + } + + public void testRecovery_OneLinkFailureMid() throws Exception { + testRecoveryWithPattern(true, RecoveryEvent.PROGRESS, + RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.PROGRESS); + } + + public void testRecovery_TwoLinkFailuresMid() throws Exception { + testRecoveryWithPattern(true, RecoveryEvent.PROGRESS, + RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED, + RecoveryEvent.PROGRESS); + } + + public void testRecovery_ThreeLinkFailuresMid() throws Exception { + testRecoveryWithPattern(true, RecoveryEvent.PROGRESS, + RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED, + RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.PROGRESS); + } + + public void testRecovery_FourLinkFailuresMid() throws Exception { + testRecoveryWithPattern(false, RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED, + RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED, + RecoveryEvent.FAIL_LINK_KILLED); + } + + // mixed patterns + + public void testRecovery_MixedFailuresProgressBetween() throws Exception { + testRecoveryWithPattern(true, + RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED, + RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_CONNECTION_REFUSED, + RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED, + RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_CONNECTION_REFUSED, + RecoveryEvent.PROGRESS); + } + + public void testRecovery_MixedFailuresNoProgressBetween() throws Exception { + testRecoveryWithPattern(true, + RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED, + RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_LINK_KILLED, + RecoveryEvent.PROGRESS); + } + + /** + * Test recovery if process cannot be killed + */ + public void testRecovery_unkillableProcess () throws Exception { + KhronosCTSRunner.Recovery recovery = new KhronosCTSRunner.Recovery(); + IMocksControl orderedControl = EasyMock.createStrictControl(); + RecoverableTestDevice mockDevice = orderedControl.createMock(RecoverableTestDevice.class); + KhronosCTSRunner.ISleepProvider mockSleepProvider = + orderedControl.createMock(KhronosCTSRunner.ISleepProvider.class); + + // recovery attempts to kill the process after a timeout + mockSleepProvider.sleep(EasyMock.gt(0)); + EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))). + andReturn("root 1234 com.drawelement.deqp").once(); + EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("kill -9 1234"))). + andReturn("").once(); + + // Recovery checks if kill failed + mockSleepProvider.sleep(EasyMock.gt(0)); + EasyMock.expectLastCall().once(); + EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))). + andReturn("root 1234 com.drawelement.deqp").once(); + + // Recovery resets the connection + EasyMock.expect(mockDevice.recoverDevice()).andReturn(true); + + // and attempts to kill the process again + EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))). + andReturn("root 1234 com.drawelement.deqp").once(); + EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("kill -9 1234"))). + andReturn("").once(); + + // Recovery checks if kill failed + mockSleepProvider.sleep(EasyMock.gt(0)); + EasyMock.expectLastCall().once(); + EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))). + andReturn("root 1234 com.drawelement.deqp").once(); + + // recovery reboots the device + mockDevice.reboot(); + EasyMock.expectLastCall().once(); + + orderedControl.replay(); + recovery.setDevice(mockDevice); + recovery.setSleepProvider(mockSleepProvider); + recovery.recoverComLinkKilled(); + orderedControl.verify(); + } + + /** + * {@inheritDoc} + */ + @Override + protected void setUp() throws Exception { + super.setUp(); + mTestsDir = FileUtil.createTempDir("khronos-cts-runner-test-cases"); + } + + /** + * {@inheritDoc} + */ + @Override + protected void tearDown() throws Exception { + FileUtil.recursiveDelete(mTestsDir); + super.tearDown(); + } +}
diff --git a/android/openglcts/src/org/khronos/cts/testercore/KhronosCTSInstrumentation.java b/android/openglcts/src/org/khronos/cts/testercore/KhronosCTSInstrumentation.java new file mode 100644 index 0000000..6ac2a8d --- /dev/null +++ b/android/openglcts/src/org/khronos/cts/testercore/KhronosCTSInstrumentation.java
@@ -0,0 +1,286 @@ + +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.khronos.cts.testercore; +import android.app.Instrumentation; +import android.os.Bundle; +import java.lang.Boolean; +import java.lang.Thread; +import java.io.File; +public class KhronosCTSInstrumentation extends Instrumentation{ + private static final String LOG_TAG = "KhronosCTS"; + private static final long LAUNCH_TIMEOUT_MS = 10000; + private static final long NO_DATA_TIMEOUT_MS = 5000; + private static final long NO_ACTIVITY_SLEEP_MS = 100; + private static final long REMOTE_DEAD_SLEEP_MS = 100; + private String m_cmdLine; + private String m_logFileName; + private String m_testParamFileName; + private String m_testName; + private boolean m_logData; + private void deleteDirectory(File directoryName) + { + // delete the files and sub-directories under the directoryName + for (File file : directoryName.listFiles()) + { + if (file.isDirectory()) + { + deleteDirectory(file); + } + else + { + file.delete(); + } + } + // check the directoryName is an empty directory + assert directoryName.listFiles().length == 0; + // delete the directoryName + directoryName.delete(); + } + @Override + public void onCreate (Bundle arguments) { + super.onCreate(arguments); + m_cmdLine = arguments.getString("khronosCTSCmdLine"); + m_logFileName = arguments.getString("khronosCTSLogFileName"); + m_testParamFileName = arguments.getString("khronosCTSTestParamFileName"); + m_testName = arguments.getString("khronosCTSTestName"); + if (m_cmdLine == null) + m_cmdLine = ""; + if (m_logFileName == null) + m_logFileName = ""; + if (m_testParamFileName == null) + m_testParamFileName = ""; + if (m_testName == null) + m_testName = "org.khronos.gl_cts/android.app.NativeActivity"; + if(arguments.getString("khronosCTSLogData") != null){ + if (arguments.getString("khronosCTSLogData").compareToIgnoreCase("true") == 0) + m_logData = true; + else + m_logData = false; + } + else{ + m_logData = false; + } + start(); + } + @Override + public void onStart() { + super.onStart(); + final KhronosCTSRemoteAPI khronosCTSRemoteApi = new KhronosCTSRemoteAPI(getTargetContext(), m_logFileName, m_testParamFileName); + //final KhronosCTSTestLogFileManager testLogFileManager = new KhronosCTSTestLogFileManager(); + final KhronosCTSTestLogParser khronosCTSTestLogParser = new KhronosCTSTestLogParser(); + try + { + KhronosCTSLog.d(LOG_TAG, "onStart"); + String fileToParse = ""; + if (m_testName.equals("org.khronos.gl_cts/org.khronos.cts.ES32GetTestParamActivity")) + { + if (m_testParamFileName.isEmpty()) + { + throw new Exception ("activity org.khronos.cts.ES32GetTestParamActivity requires khronosCTSTestParamFileName arg"); + } + fileToParse = m_testParamFileName; + } + else + { + if (m_logFileName.isEmpty()) + { + throw new Exception ("activity android.app.NativeActivity requires khronosCTSLogFileName arg"); + } + fileToParse = m_logFileName; + } + final File logFile = new File(fileToParse); + if (logFile.exists()) + logFile.delete(); + khronosCTSRemoteApi.start(m_testName, m_cmdLine); + { + final long startTimeMs = System.currentTimeMillis(); + while (true) + { + final long timeSinceStartMs = System.currentTimeMillis() - startTimeMs; + if (logFile.exists()) + { + break; + } + else if (timeSinceStartMs > LAUNCH_TIMEOUT_MS) + { + khronosCTSRemoteApi.kill(); + throw new Exception ("Timeout while waiting for log file directory"); + } + else + { + Thread.sleep(NO_ACTIVITY_SLEEP_MS); + } + } + } + khronosCTSTestLogParser.init(this, fileToParse, m_logData); + // parse until tester dies + { + while (true) + { + if (!khronosCTSTestLogParser.parse()) + { + Thread.sleep(NO_ACTIVITY_SLEEP_MS); + if(!khronosCTSRemoteApi.isRunning()) + break; + } + } + } + // parse remaining messages + { + long lastDataMs = System.currentTimeMillis(); + while (true) + { + if (khronosCTSTestLogParser.parse()) + { + lastDataMs = System.currentTimeMillis(); + } + else + { + final long timeSinceLastDataMs = System.currentTimeMillis()-lastDataMs; + if (timeSinceLastDataMs > NO_DATA_TIMEOUT_MS) + break; // Assume no data is available for reading any more + // Remote is dead, wait a bit until trying to read again + Thread.sleep(REMOTE_DEAD_SLEEP_MS); + } + } + } + finish(0, new Bundle()); + } + catch (Exception e) + { + KhronosCTSLog.e(LOG_TAG, "Exception", e); + Bundle info = new Bundle(); + info.putString("Exception", e.getMessage()); + finish(1, info); + } + finally{ + try { + khronosCTSTestLogParser.deinit(); + } catch (Exception e) { + KhronosCTSLog.w(LOG_TAG, "Got exception while closing log", e); + } + khronosCTSRemoteApi.kill(); + } + } + public void testCaseResult (String code, String details) + { + Bundle info = new Bundle(); + info.putString("dEQP-EventType", "TestCaseResult"); + info.putString("dEQP-TestCaseResult-Code", code); + info.putString("dEQP-TestCaseResult-Details", details); + sendStatus(0, info); + } + public void beginTestCase (String testCase) + { + Bundle info = new Bundle(); + info.putString("dEQP-EventType", "BeginTestCase"); + info.putString("dEQP-BeginTestCase-TestCasePath", testCase); + sendStatus(0, info); + } + public void endTestCase () + { + Bundle info = new Bundle(); + info.putString("dEQP-EventType", "EndTestCase"); + sendStatus(0, info); + } + public void testLogData (String log) throws InterruptedException + { + if (m_logData) + { + final int chunkSize = 4*1024; + while (log != null) + { + String message; + if (log.length() > chunkSize) + { + message = log.substring(0, chunkSize); + log = log.substring(chunkSize); + } + else + { + message = log; + log = null; + } + Bundle info = new Bundle(); + info.putString("dEQP-EventType", "TestLogData"); + info.putString("dEQP-TestLogData-Log", message); + sendStatus(0, info); + if (log != null) + { + Thread.sleep(1); // 1ms + } + } + } + } + public void beginTestRunParamsCollection() + { + Bundle info = new Bundle(); + info.putString("dEQP-EventType", "BeginTestRunParamsCollection"); + sendStatus(0, info); + } + public void endTestRunParamsCollection() + { + Bundle info = new Bundle(); + info.putString("dEQP-EventType", "EndTestRunParamsCollection"); + sendStatus(0, info); + } + public void beginTestRunParams(String testRunParams) + { + Bundle info = new Bundle(); + info.putString("dEQP-EventType", "BeginTestRunParams"); + info.putString("dEQP-TestRunParam", testRunParams); + sendStatus(0, info); + } + public void endTestRunParams() + { + Bundle info = new Bundle(); + info.putString("dEQP-EventType", "EndTestRunParams"); + sendStatus(0, info); + } + public void beginSession () + { + Bundle info = new Bundle(); + info.putString("dEQP-EventType", "BeginSession"); + sendStatus(0, info); + } + public void endSession () + { + Bundle info = new Bundle(); + info.putString("dEQP-EventType", "EndSession"); + sendStatus(0, info); + } + public void sessionInfo (String name, String value) + { + Bundle info = new Bundle(); + info.putString("dEQP-EventType", "SessionInfo"); + info.putString("dEQP-SessionInfo-Name", name); + info.putString("dEQP-SessionInfo-Value", value); + sendStatus(0, info); + } + public void terminateTestCase (String reason) + { + Bundle info = new Bundle(); + info.putString("dEQP-EventType", "TerminateTestCase"); + info.putString("dEQP-TerminateTestCase-Reason", reason); + sendStatus(0, info); + } + @Override + public void onDestroy() { + KhronosCTSLog.e(LOG_TAG, "onDestroy"); + super.onDestroy(); + } +}
diff --git a/android/openglcts/src/org/khronos/cts/testercore/KhronosCTSLog.java b/android/openglcts/src/org/khronos/cts/testercore/KhronosCTSLog.java new file mode 100644 index 0000000..df0a25e --- /dev/null +++ b/android/openglcts/src/org/khronos/cts/testercore/KhronosCTSLog.java
@@ -0,0 +1,48 @@ + +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.khronos.cts.testercore; +import android.util.Log; +public class KhronosCTSLog { + private static final boolean LOG_DEBUG = android.util.Log.isLoggable("KhronosCTS", android.util.Log.DEBUG); + private static final boolean LOG_INFO = true; + private static final boolean LOG_WARNING = true; + private static final boolean LOG_ERROR = true; + public static void d (String tag, String msg) { + if (LOG_DEBUG) + android.util.Log.d(tag, msg); + } + public static void i (String tag, String msg) { + if (LOG_INFO) + android.util.Log.i(tag, msg); + } + public static void w (String tag, String msg) { + if (LOG_WARNING) + android.util.Log.w(tag, msg); + } + public static void w (String tag, String msg, Throwable tr) { + if (LOG_WARNING) + android.util.Log.w(tag, msg, tr); + } + public static void e (String tag, String msg) { + if (LOG_ERROR) + android.util.Log.e(tag, msg); + } + public static void e (String tag, String msg, Throwable tr) { + if (LOG_ERROR) + android.util.Log.e(tag, msg, tr); + } +}
diff --git a/android/openglcts/src/org/khronos/cts/testercore/KhronosCTSRemoteAPI.java b/android/openglcts/src/org/khronos/cts/testercore/KhronosCTSRemoteAPI.java new file mode 100644 index 0000000..3adaafd --- /dev/null +++ b/android/openglcts/src/org/khronos/cts/testercore/KhronosCTSRemoteAPI.java
@@ -0,0 +1,127 @@ + +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.khronos.cts.testercore; +import android.app.ActivityManager; +import android.content.Context; +import android.content.ComponentName; +import android.content.Intent; +import android.os.Process; +import java.util.List; +public class KhronosCTSRemoteAPI{ + private static final String LOG_TAG = "KhronosCTS"; + private Context m_context; + private String m_processName; + private String m_logFileName; + private String m_testParamFileName; + private boolean m_isTestProcessStarted; + public KhronosCTSRemoteAPI (Context context, String logFileName, String testParamFileName) { + m_context = context; + m_processName = m_context.getPackageName() + ":testercore"; + m_logFileName = logFileName; + m_testParamFileName = testParamFileName; + m_isTestProcessStarted = false; + } + private ComponentName getDefaultTestComponent () { + return new ComponentName(m_context.getPackageName(), "android.app.NativeActivity"); + } + private ComponentName getTestComponent(String testName) { + if (testName != null && !testName.equals("")) { + ComponentName component = ComponentName.unflattenFromString(testName); + if (component == null) { + KhronosCTSLog.e(LOG_TAG, "Invalid component name supplied (" + testName + "), using default"); + component = getDefaultTestComponent(); + } + return component; + } + else { + return getDefaultTestComponent(); + } + } + public boolean start (String testerName, String cmdLine) { + KhronosCTSLog.w(LOG_TAG, "KhronosCTSRemoteAPI start: " + testerName); + // Choose component + ComponentName component = getTestComponent(testerName); + Intent testIntent = new Intent(); + testIntent.setComponent(component); + testIntent.setFlags(Intent.FLAG_ACTIVITY_NEW_TASK); + // Add all data to cmdLine + cmdLine = testerName + " " + cmdLine; + if (!m_logFileName.isEmpty()) + { + cmdLine += " --deqp-log-filename=" + m_logFileName; + } + cmdLine = cmdLine.replaceAll(" ", " "); + testIntent.putExtra("cmdLine", cmdLine); + if (!m_testParamFileName.isEmpty()) + { + testIntent.putExtra("khronosCTSTestParamFileName", m_testParamFileName); + } + // Try to resolve intent. + boolean isActivity = m_context.getPackageManager().resolveActivity(testIntent, 0) != null; + if (!isActivity) { + KhronosCTSLog.e(LOG_TAG, "Can't resolve component as activity (" + component.flattenToString() + "), using default"); + component = getDefaultTestComponent(); + } + try { + m_context.startActivity(testIntent); + } catch (Exception e) { + KhronosCTSLog.e(LOG_TAG, "Failed to start test", e); + return false; + } + m_isTestProcessStarted = true; + return true; + } + public boolean kill() { + ActivityManager.RunningAppProcessInfo processInfo = findProcess(m_processName); + // \note not mutating m_isTestProcessStarted yet since process does not die immediately + if (processInfo != null) { + KhronosCTSLog.d(LOG_TAG, "Killing " + m_processName); + Process.killProcess(processInfo.pid); + return true; + } else { + return false; + } + } + public boolean isRunning() { + if (!m_isTestProcessStarted) { + return false; + } else if (isProcessRunning(m_processName)) { + return true; + } else { + // Cache result. Safe, because only start() can spawn the process + m_isTestProcessStarted = false; + return false; + } + } + public String getLogFileName() { + return m_logFileName; + } + private ActivityManager.RunningAppProcessInfo findProcess (String name) { + ActivityManager activityMgr = (ActivityManager)m_context.getSystemService(Context.ACTIVITY_SERVICE); + List<ActivityManager.RunningAppProcessInfo> processes = activityMgr.getRunningAppProcesses(); + for (ActivityManager.RunningAppProcessInfo info : processes) { + KhronosCTSLog.d(LOG_TAG, "Found proc : " + info.processName + " " + info.pid + + " name of the target process: " + name ); + if (info.processName.equals(name)) + return info; + } + return null; + } + private boolean isProcessRunning (String processName) { + return (findProcess(processName) != null); + } +}
diff --git a/android/openglcts/src/org/khronos/cts/testercore/KhronosCTSTestLogParser.java b/android/openglcts/src/org/khronos/cts/testercore/KhronosCTSTestLogParser.java new file mode 100644 index 0000000..6bf804e --- /dev/null +++ b/android/openglcts/src/org/khronos/cts/testercore/KhronosCTSTestLogParser.java
@@ -0,0 +1,88 @@ + +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.khronos.cts.testercore; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +public class KhronosCTSTestLogParser{ + static { + System.loadLibrary("khronosopenglcts"); + } + private long m_nativePointer; + private KhronosCTSInstrumentation m_instrumentation; + private FileInputStream m_log; + private String m_logFileName; + private byte[] m_buffer; + private long m_logRead; + public KhronosCTSTestLogParser() + { + m_nativePointer = 0; + m_instrumentation = null; + m_log = null; + m_logRead = 0; + m_buffer = null; + } + public void init (KhronosCTSInstrumentation instrumentation, String logFileName, boolean logData) throws FileNotFoundException + { + assert instrumentation != null; + assert m_instrumentation == null; + assert m_nativePointer == 0; + assert m_log == null; + m_logFileName = logFileName; + m_instrumentation = instrumentation; + m_nativePointer = nativeCreate(logData); + m_buffer = new byte[4*1024*1024]; + m_log = new FileInputStream(m_logFileName); + } + public void deinit () throws IOException + { + assert m_nativePointer != 0; + assert m_instrumentation != null; + assert m_log != null; + nativeDestroy(m_nativePointer); + if (m_log != null) + { + m_log.close(); + } + m_nativePointer = 0; + m_instrumentation = null; + m_log = null; + m_buffer = null; + } + public boolean parse() throws IOException + { + assert m_nativePointer != 0; + assert m_instrumentation != null; + assert m_log != null; + boolean gotData = false; + while (true) + { + final int numAvailable = m_log.available(); + if (numAvailable <= 0) + break; + final int numRead = m_log.read(m_buffer, 0, Math.min(numAvailable, m_buffer.length)); + assert numRead > 0; + m_logRead += numRead; + gotData = true; + nativeParse(m_nativePointer, m_instrumentation, m_buffer, numRead); + } + return gotData; + } + private static native long nativeCreate (boolean logData); + private static native void nativeDestroy (long nativePointer); + private static native void nativeParse (long nativePointer, KhronosCTSInstrumentation instrumentation, byte[] buffer, int size); +}
diff --git a/external/vulkancts/framework/vulkan/vkApiVersion.cpp b/external/vulkancts/framework/vulkan/vkApiVersion.cpp index f77b0ce..ae81eb4 100644 --- a/external/vulkancts/framework/vulkan/vkApiVersion.cpp +++ b/external/vulkancts/framework/vulkan/vkApiVersion.cpp
@@ -65,6 +65,7 @@ {VK_MAKE_API_VERSION(0, 1, 2, 0), VK_MAKE_API_VERSION(0, 1, 1, 0)}, {VK_MAKE_API_VERSION(1, 1, 0, 0), VK_MAKE_API_VERSION(0, 1, 2, 0)}, {VK_MAKE_API_VERSION(0, 1, 3, 0), VK_MAKE_API_VERSION(0, 1, 2, 0)}, + {VK_MAKE_API_VERSION(0, 1, 4, 0), VK_MAKE_API_VERSION(0, 1, 3, 0)}, }; bool isApiVersionEqual(uint32_t lhs, uint32_t rhs)
diff --git a/framework/delibs/deutil/deProcess.c b/framework/delibs/deutil/deProcess.c index 78ee9a3..6bce702 100644 --- a/framework/delibs/deutil/deProcess.c +++ b/framework/delibs/deutil/deProcess.c
@@ -138,7 +138,7 @@ { deProcess *process = (deProcess *)deCalloc(sizeof(deProcess)); if (!process) - return false; + return NULL; process->state = PROCESSSTATE_NOT_STARTED;
diff --git a/scripts/build_android_mustpass.py b/scripts/build_android_mustpass.py index e9159da..bc44fe1 100644 --- a/scripts/build_android_mustpass.py +++ b/scripts/build_android_mustpass.py
@@ -58,8 +58,7 @@ MAIN_EGL_COMMON_FILTERS = [include("egl-main.txt"), exclude("egl-test-issues.txt"), exclude("egl-manual-robustness.txt"), - exclude("egl-driver-issues.txt"), - exclude("egl-temp-excluded.txt")] + exclude("egl-driver-issues.txt")] # Android CTS is not using EGL test list for year 2021 MAIN_EGL_PKG = Package(module = EGL_MODULE, configurations = [ @@ -89,16 +88,15 @@ rotation = "unspecified", surfacetype = "window", required = True, - filters = MAIN_EGL_COMMON_FILTERS + [exclude("egl-main-2020-03-01.txt", "egl-main-2022-03-01.txt", "egl-main-2023-03-01.txt")], + filters = [include("egl-main-2024-03-01.txt")], runtime = "5m"), - # Risky subset - Configuration(name = "main-risky", + Configuration(name = "main-2025-03-01", glconfig = "rgba8888d24s8ms0", rotation = "unspecified", surfacetype = "window", required = True, - filters = [include("egl-temp-excluded.txt")], - runtime = "2m"), + filters = MAIN_EGL_COMMON_FILTERS + [exclude("egl-main-2020-03-01.txt", "egl-main-2022-03-01.txt", "egl-main-2023-03-01.txt", "egl-main-2024-03-01.txt")], + runtime = "5m"), # Note: There are no incremental deqp testlists for EGL since these tests do not work with # deqp-binary. @@ -108,7 +106,6 @@ include("gles2-main.txt"), exclude("gles2-test-issues.txt"), exclude("gles2-failures.txt"), - exclude("gles2-temp-excluded.txt"), ] MAIN_GLES2_PKG = Package(module = GLES2_MODULE, configurations = [ Configuration(name = "main-2020-03-01", @@ -144,7 +141,14 @@ rotation = "unspecified", surfacetype = "window", required = True, - filters = MAIN_GLES2_COMMON_FILTERS + [exclude("gles2-main-2020-03-01.txt", "gles2-main-2021-03-01.txt", "gles2-main-2022-03-01.txt", "gles2-main-2023-03-01.txt")], + filters = [include("gles2-main-2024-03-01.txt")], + runtime = "10m"), + Configuration(name = "main-2025-03-01", + glconfig = "rgba8888d24s8ms0", + rotation = "unspecified", + surfacetype = "window", + required = True, + filters = MAIN_GLES2_COMMON_FILTERS + [exclude("gles2-main-2020-03-01.txt", "gles2-main-2021-03-01.txt", "gles2-main-2022-03-01.txt", "gles2-main-2023-03-01.txt", "gles2-main-2024-03-01.txt")], runtime = "10m"), # Incremental deqp baseline @@ -160,7 +164,6 @@ exclude("gles3-driver-issues.txt"), exclude("gles3-test-issues.txt"), exclude("gles3-spec-issues.txt"), - exclude("gles3-temp-excluded.txt"), exclude("gles3-waivers.txt"), ] MAIN_GLES3_PKG = Package(module = GLES3_MODULE, configurations = [ @@ -198,7 +201,14 @@ rotation = "unspecified", surfacetype = "window", required = True, - filters = MAIN_GLES3_COMMON_FILTERS + [exclude("gles3-main-2020-03-01.txt", "gles3-main-2021-03-01.txt", "gles3-main-2022-03-01.txt", "gles3-main-2023-03-01.txt")], + filters = [include("gles3-main-2024-03-01.txt")], + runtime = "10m"), + Configuration(name = "main-2025-03-01", + glconfig = "rgba8888d24s8ms0", + rotation = "unspecified", + surfacetype = "window", + required = True, + filters = MAIN_GLES3_COMMON_FILTERS + [exclude("gles3-main-2020-03-01.txt", "gles3-main-2021-03-01.txt", "gles3-main-2022-03-01.txt", "gles3-main-2023-03-01.txt", "gles3-main-2024-03-01.txt")], runtime = "10m"), # Rotations Configuration(name = "rotate-portrait", @@ -260,7 +270,6 @@ exclude("gles31-driver-issues.txt"), exclude("gles31-test-issues.txt"), exclude("gles31-spec-issues.txt"), - exclude("gles31-temp-excluded.txt"), exclude("gles31-waivers.txt"), ] MAIN_GLES31_PKG = Package(module = GLES31_MODULE, configurations = [ @@ -297,7 +306,14 @@ rotation = "unspecified", surfacetype = "window", required = True, - filters = MAIN_GLES31_COMMON_FILTERS + [exclude("gles31-main-2020-03-01.txt", "gles31-main-2021-03-01.txt", "gles31-main-2022-03-01.txt", "gles31-main-2023-03-01.txt")], + filters = [include("gles31-main-2024-03-01.txt")], + runtime = "10m"), + Configuration(name = "main-2025-03-01", + glconfig = "rgba8888d24s8ms0", + rotation = "unspecified", + surfacetype = "window", + required = True, + filters = MAIN_GLES31_COMMON_FILTERS + [exclude("gles31-main-2020-03-01.txt", "gles31-main-2021-03-01.txt", "gles31-main-2022-03-01.txt", "gles31-main-2023-03-01.txt", "gles31-main-2024-03-01.txt")], runtime = "10m"), # Rotations Configuration(name = "rotate-portrait", @@ -354,7 +370,6 @@ exclude("vk-excluded-tests.txt"), exclude("vk-test-issues.txt"), exclude("vk-waivers.txt"), - exclude("vk-temp-excluded.txt"), ] MAIN_VULKAN_PKG = Package(module = VULKAN_MODULE, configurations = [ Configuration(name = "main-2019-03-01", @@ -378,7 +393,11 @@ runtime = "10m", listOfGroupsToSplit = ["dEQP-VK", "dEQP-VK.pipeline", "dEQP-VK.image", "dEQP-VK.shader_object"]), Configuration(name = "main-2024-03-01", - filters = MAIN_VULKAN_FILTERS + [exclude("vk-main-2019-03-01.txt", "vk-main-2020-03-01.txt", "vk-main-2021-03-01.txt", "vk-main-2022-03-01.txt", "vk-main-2023-03-01-part1.txt", "vk-main-2023-03-01-part2.txt")], + filters = [include("vk-main-2024-03-01.txt")], + runtime = "10m", + listOfGroupsToSplit = ["dEQP-VK", "dEQP-VK.pipeline", "dEQP-VK.image", "dEQP-VK.shader_object"]), + Configuration(name = "main-2025-03-01", + filters = MAIN_VULKAN_FILTERS + [exclude("vk-main-2019-03-01.txt", "vk-main-2020-03-01.txt", "vk-main-2021-03-01.txt", "vk-main-2022-03-01.txt", "vk-main-2023-03-01-part1.txt", "vk-main-2023-03-01-part2.txt", "vk-main-2024-03-01.txt")], runtime = "10m", listOfGroupsToSplit = ["dEQP-VK", "dEQP-VK.pipeline", "dEQP-VK.image", "dEQP-VK.shader_object"]), Configuration(name = "incremental-deqp",