Merge "Upgrade libyuv to 5f284054cb806f82e737f738003d764277970d45" into main
diff --git a/.vpython b/.vpython
deleted file mode 100644
index 4a64fd2..0000000
--- a/.vpython
+++ /dev/null
@@ -1,52 +0,0 @@
-# This is a vpython "spec" file.
-#
-# It describes patterns for python wheel dependencies of the python scripts in
-# the chromium repo, particularly for dependencies that have compiled components
-# (since pure-python dependencies can be easily vendored into third_party).
-#
-# When vpython is invoked, it finds this file and builds a python VirtualEnv,
-# containing all of the dependencies described in this file, fetching them from
-# CIPD (the "Chrome Infrastructure Package Deployer" service). Unlike `pip`,
-# this never requires the end-user machine to have a working python extension
-# compilation environment. All of these packages are built using:
-# https://chromium.googlesource.com/infra/infra/+/master/infra/tools/dockerbuild/
-#
-# All python scripts in the repo share this same spec, to avoid dependency
-# fragmentation.
-#
-# If you have depot_tools installed in your $PATH, you can invoke python scripts
-# in this repo by running them as you normally would run them, except
-# substituting `vpython` instead of `python` on the command line, e.g.:
-# vpython path/to/script.py some --arguments
-#
-# Read more about `vpython` and how to modify this file here:
-# https://chromium.googlesource.com/infra/infra/+/master/doc/users/vpython.md
-
-python_version: "2.7"
-
-# Used by:
-# third_party/catapult
-wheel: <
- name: "infra/python/wheels/psutil/${platform}_${py_python}_${py_abi}"
- version: "version:5.2.2"
->
-
-# Used by:
-# third_party/catapult
-wheel: <
- name: "infra/python/wheels/pypiwin32/${vpython_platform}"
- version: "version:219"
- match_tag: <
- platform: "win32"
- >
- match_tag: <
- platform: "win_amd64"
- >
->
-
-# Used by:
-# build/android
-wheel: <
- name: "infra/python/wheels/requests-py2_py3"
- version: "version:2.13.0"
->
diff --git a/.vpython3 b/.vpython3
index 28d819e..4670786 100644
--- a/.vpython3
+++ b/.vpython3
@@ -42,8 +42,8 @@
# build/android/pylib/local/emulator/avd.py
# components/policy/test_support/policy_testserver.py
wheel: <
- name: "infra/python/wheels/protobuf-py2_py3"
- version: "version:3.15.8"
+ name: "infra/python/wheels/protobuf-py3"
+ version: "version:4.25.1"
>
# TODO(https://crbug.com/898348): Add in necessary wheels as Python3 versions
@@ -184,7 +184,11 @@
>
wheel: <
name: "infra/python/wheels/urllib3-py2_py3"
- version: "version:1.24.3"
+ version: "version:1.26.6"
+>
+wheel: <
+ name: "infra/python/wheels/brotli/${vpython_platform}"
+ version: "version:1.0.9"
>
wheel: <
name: "infra/python/wheels/blessings-py2_py3"
diff --git a/AUTHORS b/AUTHORS
index 28c0895..b272adf 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -3,4 +3,5 @@
Google Inc.
+Ho Cheung <uioptt24@gmail.com>
Ivan Pavlotskiy <ivan.pavlotskiy@lgepartner.com>
diff --git a/Android.bp b/Android.bp
index e93b4b1..bb7026b 100644
--- a/Android.bp
+++ b/Android.bp
@@ -76,12 +76,26 @@
"-fexceptions",
"-DHAVE_JPEG",
"-DLIBYUV_UNLIMITED_DATA",
+ // TODO: enable these...
+ "-DLIBYUV_DISABLE_SME",
+ "-DLIBYUV_DISABLE_SVE",
],
arch: {
arm: {
cflags: ["-mfpu=neon"],
},
+ arm64: {
+ cflags: [
+ "-Xclang -target-feature",
+ // This comment prevents bpfmt from sorting the flags incorrectly.
+ "-Xclang +dotprod",
+ // This comment prevents bpfmt from sorting the flags incorrectly.
+ "-Xclang -target-feature",
+ // This comment prevents bpfmt from sorting the flags incorrectly.
+ "-Xclang +i8mm",
+ ],
+ },
},
shared_libs: ["libjpeg"],
@@ -99,6 +113,10 @@
lto: {
never: true,
},
+ // TODO: enable this...
+// sanitize: {
+// cfi: true,
+// },
}
// compatibilty static library until all uses of libyuv_static are replaced
diff --git a/BUILD.gn b/BUILD.gn
index 2c600b2..649249d 100644
--- a/BUILD.gn
+++ b/BUILD.gn
@@ -29,10 +29,18 @@
ldflags = [ "-Wl,--dynamic-linker,/system/bin/linker64" ]
}
}
- defines = []
+
+ # Define CHROMIUM to tell cpu_id to avoid sandbox unsafe system calls.
+ defines = [ "CHROMIUM" ]
if (!libyuv_use_neon) {
defines += [ "LIBYUV_DISABLE_NEON" ]
}
+ if (!libyuv_use_sve) {
+ defines += [ "LIBYUV_DISABLE_SVE" ]
+ }
+ if (!libyuv_use_sme) {
+ defines += [ "LIBYUV_DISABLE_SME" ]
+ }
if (libyuv_disable_rvv) {
defines += [ "LIBYUV_DISABLE_RVV" ]
}
@@ -76,6 +84,14 @@
deps += [ ":libyuv_neon" ]
}
+ if (libyuv_use_sve) {
+ deps += [ ":libyuv_sve" ]
+ }
+
+ if (libyuv_use_sme) {
+ deps += [ ":libyuv_sme" ]
+ }
+
if (libyuv_use_msa) {
deps += [ ":libyuv_msa" ]
}
@@ -104,11 +120,14 @@
"include/libyuv.h",
"include/libyuv/basic_types.h",
"include/libyuv/compare.h",
+ "include/libyuv/compare_row.h",
"include/libyuv/convert.h",
"include/libyuv/convert_argb.h",
"include/libyuv/convert_from.h",
"include/libyuv/convert_from_argb.h",
"include/libyuv/cpu_id.h",
+ "include/libyuv/loongson_intrinsics.h",
+ "include/libyuv/macros_msa.h",
"include/libyuv/mjpeg_decoder.h",
"include/libyuv/planar_functions.h",
"include/libyuv/rotate.h",
@@ -217,23 +236,45 @@
public_configs = [ ":libyuv_config" ]
- # Always enable optimization for Release and NaCl builds (to workaround
- # crbug.com/538243).
- if (!is_debug) {
- configs -= [ "//build/config/compiler:default_optimization" ]
-
- # Enable optimize for speed (-O2) over size (-Os).
- # TODO(fbarchard): Consider optimize_speed which is O3.
- configs += [ "//build/config/compiler:optimize_max" ]
- }
-
- if (current_cpu != "arm64") {
+ if (current_cpu == "arm64") {
+ cflags = [ "-march=armv8-a+dotprod+i8mm" ]
+ } else {
configs -= [ "//build/config/compiler:compiler_arm_fpu" ]
cflags = [ "-mfpu=neon" ]
}
}
}
+if (libyuv_use_sve) {
+ static_library("libyuv_sve") {
+ sources = [ "source/row_sve.cc" ]
+
+ deps = [ ":libyuv_internal" ]
+
+ public_configs = [ ":libyuv_config" ]
+
+ # SVE2 is an Armv9-A feature.
+ cflags = [ "-march=armv9-a+sve2" ]
+ }
+}
+
+if (libyuv_use_sme) {
+ static_library("libyuv_sme") {
+ sources = [
+ "source/rotate_sme.cc",
+ "source/row_sme.cc",
+ "source/scale_sme.cc",
+ ]
+
+ deps = [ ":libyuv_internal" ]
+
+ public_configs = [ ":libyuv_config" ]
+
+ # SME is an Armv9-A feature.
+ cflags = [ "-march=armv9-a+sme" ]
+ }
+}
+
if (libyuv_use_msa) {
static_library("libyuv_msa") {
sources = [
@@ -354,6 +395,9 @@
if (is_linux || is_chromeos) {
cflags = [ "-fexceptions" ]
+
+ # For enabling ASLR.
+ ldflags = [ "-pie" ]
}
if (is_ios) {
configs -= [ "//build/config/compiler:default_symbols" ]
@@ -363,9 +407,6 @@
if (!is_ios && !libyuv_disable_jpeg) {
defines += [ "HAVE_JPEG" ]
}
- if (is_android) {
- deps += [ "//testing/android/native_test:native_test_native_code" ]
- }
# TODO(YangZhang): These lines can be removed when high accuracy
# YUV to RGB to Neon is ported.
diff --git a/CM_linux_packages.cmake b/CM_linux_packages.cmake
index a073edf..4caf156 100644
--- a/CM_linux_packages.cmake
+++ b/CM_linux_packages.cmake
@@ -1,6 +1,7 @@
# determine the version number from the #define in libyuv/version.h
EXECUTE_PROCESS (
- COMMAND grep --perl-regex --only-matching "(?<=LIBYUV_VERSION )[0-9]+" include/libyuv/version.h
+ COMMAND grep -Eo "LIBYUV_VERSION\ [0-9]+" include/libyuv/version.h
+ COMMAND grep -Eo "[0-9]+"
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
OUTPUT_VARIABLE YUV_VERSION_NUMBER
OUTPUT_STRIP_TRAILING_WHITESPACE )
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 9abfa74..5df7685 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1,53 +1,200 @@
# CMakeLists for libyuv
# Originally created for "roxlu build system" to compile libyuv on windows
-# Run with -DTEST=ON to build unit tests
+# Run with -DUNIT_TEST=ON to build unit tests
-PROJECT ( YUV C CXX ) # "C" is required even for C++ projects
-CMAKE_MINIMUM_REQUIRED( VERSION 2.8.12 )
-OPTION( UNIT_TEST "Built unit tests" OFF )
+include(CheckCSourceCompiles)
-SET ( ly_base_dir ${PROJECT_SOURCE_DIR} )
-SET ( ly_src_dir ${ly_base_dir}/source )
-SET ( ly_inc_dir ${ly_base_dir}/include )
-SET ( ly_tst_dir ${ly_base_dir}/unit_test )
-SET ( ly_lib_name yuv )
-SET ( ly_lib_static ${ly_lib_name} )
-SET ( ly_lib_shared ${ly_lib_name}_shared )
+project ( YUV C CXX ) # "C" is required even for C++ projects
+cmake_minimum_required( VERSION 2.8.12 )
+option( UNIT_TEST "Built unit tests" OFF )
-FILE ( GLOB_RECURSE ly_source_files ${ly_src_dir}/*.cc )
-LIST ( SORT ly_source_files )
+set ( ly_base_dir ${PROJECT_SOURCE_DIR} )
+set ( ly_src_dir ${ly_base_dir}/source )
+set ( ly_inc_dir ${ly_base_dir}/include )
+set ( ly_tst_dir ${ly_base_dir}/unit_test )
+set ( ly_lib_name yuv )
+set ( ly_lib_static ${ly_lib_name} )
+set ( ly_lib_shared ${ly_lib_name}_shared )
-FILE ( GLOB_RECURSE ly_unittest_sources ${ly_tst_dir}/*.cc )
-LIST ( SORT ly_unittest_sources )
+# We cannot use GLOB here since we want to be able to separate out files that
+# need particular flags to enable architecture extensions like AArch64's SVE.
+# TODO: More of these files could be separated out for different architectures.
+set ( ly_common_source_files
+ ${ly_src_dir}/compare.cc
+ ${ly_src_dir}/compare_common.cc
+ ${ly_src_dir}/compare_gcc.cc
+ ${ly_src_dir}/compare_msa.cc
+ ${ly_src_dir}/compare_win.cc
+ ${ly_src_dir}/convert_argb.cc
+ ${ly_src_dir}/convert.cc
+ ${ly_src_dir}/convert_from_argb.cc
+ ${ly_src_dir}/convert_from.cc
+ ${ly_src_dir}/convert_jpeg.cc
+ ${ly_src_dir}/convert_to_argb.cc
+ ${ly_src_dir}/convert_to_i420.cc
+ ${ly_src_dir}/cpu_id.cc
+ ${ly_src_dir}/mjpeg_decoder.cc
+ ${ly_src_dir}/mjpeg_validate.cc
+ ${ly_src_dir}/planar_functions.cc
+ ${ly_src_dir}/rotate_any.cc
+ ${ly_src_dir}/rotate_argb.cc
+ ${ly_src_dir}/rotate.cc
+ ${ly_src_dir}/rotate_common.cc
+ ${ly_src_dir}/rotate_gcc.cc
+ ${ly_src_dir}/rotate_lsx.cc
+ ${ly_src_dir}/rotate_msa.cc
+ ${ly_src_dir}/rotate_win.cc
+ ${ly_src_dir}/row_any.cc
+ ${ly_src_dir}/row_common.cc
+ ${ly_src_dir}/row_gcc.cc
+ ${ly_src_dir}/row_lasx.cc
+ ${ly_src_dir}/row_lsx.cc
+ ${ly_src_dir}/row_msa.cc
+ ${ly_src_dir}/row_rvv.cc
+ ${ly_src_dir}/row_win.cc
+ ${ly_src_dir}/scale_any.cc
+ ${ly_src_dir}/scale_argb.cc
+ ${ly_src_dir}/scale.cc
+ ${ly_src_dir}/scale_common.cc
+ ${ly_src_dir}/scale_gcc.cc
+ ${ly_src_dir}/scale_lsx.cc
+ ${ly_src_dir}/scale_msa.cc
+ ${ly_src_dir}/scale_rgb.cc
+ ${ly_src_dir}/scale_rvv.cc
+ ${ly_src_dir}/scale_uv.cc
+ ${ly_src_dir}/scale_win.cc
+ ${ly_src_dir}/video_common.cc)
-INCLUDE_DIRECTORIES( BEFORE ${ly_inc_dir} )
+file ( GLOB_RECURSE ly_unittest_sources ${ly_tst_dir}/*.cc )
+list ( SORT ly_unittest_sources )
+
+include_directories( BEFORE ${ly_inc_dir} )
if(MSVC)
- ADD_DEFINITIONS ( -D_CRT_SECURE_NO_WARNINGS )
+ add_definitions ( -D_CRT_SECURE_NO_WARNINGS )
+endif()
+
+# Need to set PIC to allow creating shared libraries from object file libraries.
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+
+# Build the set of objects that do not need to be compiled with flags to enable
+# particular architecture features.
+add_library( ${ly_lib_name}_common_objects OBJECT ${ly_common_source_files} )
+set(ly_lib_parts $<TARGET_OBJECTS:${ly_lib_name}_common_objects>)
+
+string(TOLOWER "${CMAKE_SYSTEM_PROCESSOR}" SYSPROC)
+set(LOONGARCH64_ALIASES loongarch64)
+list(FIND LOONGARCH64_ALIASES "${SYSPROC}" LOONGARCH64MATCH)
+
+if(LOONGARCH64MATCH GREATER "-1")
+ set(LOONGARCH64 1)
+endif()
+
+if(NOT MSVC)
+ string(TOLOWER "${CMAKE_SYSTEM_PROCESSOR}" arch_lowercase)
+
+ if(arch_lowercase MATCHES "^arm" AND NOT arch_lowercase STREQUAL "arm64")
+ # Enable Arm Neon kernels.
+ add_definitions(-DLIBYUV_NEON=1)
+ add_library(${ly_lib_name}_neon OBJECT
+ ${ly_src_dir}/compare_neon.cc
+ ${ly_src_dir}/rotate_neon.cc
+ ${ly_src_dir}/row_neon.cc
+ ${ly_src_dir}/scale_neon.cc)
+ target_compile_options(${ly_lib_name}_neon PRIVATE -mfpu=neon)
+ list(APPEND ly_lib_parts $<TARGET_OBJECTS:${ly_lib_name}_neon>)
+ endif()
+
+ if(arch_lowercase STREQUAL "aarch64" OR arch_lowercase STREQUAL "arm64")
+ # Enable AArch64 Neon dot-product and i8mm kernels.
+ add_library(${ly_lib_name}_neon64 OBJECT
+ ${ly_src_dir}/compare_neon64.cc
+ ${ly_src_dir}/rotate_neon64.cc
+ ${ly_src_dir}/row_neon64.cc
+ ${ly_src_dir}/scale_neon64.cc)
+ target_compile_options(${ly_lib_name}_neon64 PRIVATE -march=armv8-a+dotprod+i8mm)
+ list(APPEND ly_lib_parts $<TARGET_OBJECTS:${ly_lib_name}_neon64>)
+
+ # Enable AArch64 SVE kernels.
+ add_library(${ly_lib_name}_sve OBJECT
+ ${ly_src_dir}/row_sve.cc)
+ target_compile_options(${ly_lib_name}_sve PRIVATE -march=armv9-a+sve2)
+ list(APPEND ly_lib_parts $<TARGET_OBJECTS:${ly_lib_name}_sve>)
+
+ set(OLD_CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS})
+ set(OLD_CMAKE_TRY_COMPILE_TARGET_TYPE ${CMAKE_TRY_COMPILE_TARGET_TYPE})
+ set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -march=armv9-a+sme")
+ set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)
+ # Check whether the compiler can compile SME functions; this fails
+ # with Clang for Windows.
+ check_c_source_compiles("
+__arm_locally_streaming void func(void) { }
+int main(void) { return 0; }
+ " CAN_COMPILE_SME)
+ set(CMAKE_REQUIRED_FLAGS ${OLD_CMAKE_REQUIRED_FLAGS})
+ set(CMAKE_TRY_COMPILE_TARGET_TYPE ${OLD_CMAKE_TRY_COMPILE_TARGET_TYPE})
+
+ if (CAN_COMPILE_SME)
+ # Enable AArch64 SME kernels.
+ add_library(${ly_lib_name}_sme OBJECT
+ ${ly_src_dir}/rotate_sme.cc
+ ${ly_src_dir}/row_sme.cc
+ ${ly_src_dir}/scale_sme.cc)
+ target_compile_options(${ly_lib_name}_sme PRIVATE -march=armv9-a+sme)
+ list(APPEND ly_lib_parts $<TARGET_OBJECTS:${ly_lib_name}_sme>)
+ else()
+ add_definitions(-DLIBYUV_DISABLE_SME)
+ endif()
+ endif()
+endif()
+
+if(LOONGARCH64)
+ include(CheckCXXSourceCompiles)
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wno-narrowing")
+ check_cxx_source_compiles("
+ int main(int argc, char **argv) {
+ __asm__ volatile (
+ \"vadd.w $vr0, $vr1, $vr1\"
+ );
+ return 0; }" SUPPORTS_LSX)
+
+ check_cxx_source_compiles("
+ int main(int argc, char **argv) {
+ __asm__ volatile (
+ \"xvadd.w $xr0, $xr1, $xr1\"
+ );
+ return 0; }" SUPPORTS_LASX)
+
+ if(SUPPORTS_LSX)
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mlsx")
+ endif()
+ if(SUPPORTS_LASX)
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mlasx")
+ endif()
endif()
# this creates the static library (.a)
-ADD_LIBRARY ( ${ly_lib_static} STATIC ${ly_source_files} )
+add_library( ${ly_lib_static} STATIC ${ly_lib_parts})
# this creates the shared library (.so)
-ADD_LIBRARY ( ${ly_lib_shared} SHARED ${ly_source_files} )
-SET_TARGET_PROPERTIES ( ${ly_lib_shared} PROPERTIES OUTPUT_NAME "${ly_lib_name}" )
-SET_TARGET_PROPERTIES ( ${ly_lib_shared} PROPERTIES PREFIX "lib" )
+add_library( ${ly_lib_shared} SHARED ${ly_lib_parts})
+set_target_properties( ${ly_lib_shared} PROPERTIES OUTPUT_NAME "${ly_lib_name}" )
+set_target_properties( ${ly_lib_shared} PROPERTIES PREFIX "lib" )
if(WIN32)
- SET_TARGET_PROPERTIES ( ${ly_lib_shared} PROPERTIES IMPORT_PREFIX "lib" )
+ set_target_properties( ${ly_lib_shared} PROPERTIES IMPORT_PREFIX "lib" )
endif()
# this creates the cpuid tool
-ADD_EXECUTABLE ( cpuid ${ly_base_dir}/util/cpuid.c )
-TARGET_LINK_LIBRARIES ( cpuid ${ly_lib_static} )
+add_executable ( cpuid ${ly_base_dir}/util/cpuid.c )
+target_link_libraries ( cpuid ${ly_lib_static} )
# this creates the conversion tool
-ADD_EXECUTABLE ( yuvconvert ${ly_base_dir}/util/yuvconvert.cc )
-TARGET_LINK_LIBRARIES ( yuvconvert ${ly_lib_static} )
+add_executable ( yuvconvert ${ly_base_dir}/util/yuvconvert.cc )
+target_link_libraries ( yuvconvert ${ly_lib_static} )
# this creates the yuvconstants tool
-ADD_EXECUTABLE ( yuvconstants ${ly_base_dir}/util/yuvconstants.c )
-TARGET_LINK_LIBRARIES ( yuvconstants ${ly_lib_static} )
+add_executable ( yuvconstants ${ly_base_dir}/util/yuvconstants.c )
+target_link_libraries ( yuvconstants ${ly_lib_static} )
find_package ( JPEG )
if (JPEG_FOUND)
@@ -59,10 +206,11 @@
if(UNIT_TEST)
find_library(GTEST_LIBRARY gtest)
if(GTEST_LIBRARY STREQUAL "GTEST_LIBRARY-NOTFOUND")
- set(GTEST_SRC_DIR /usr/src/gtest CACHE STRING "Location of gtest sources")
+ set(GTEST_SRC_DIR_DEFAULT /usr/src/gtest)
if (CMAKE_CROSSCOMPILING)
- set(GTEST_SRC_DIR third_party/googletest/src/googletest)
+ set(GTEST_SRC_DIR_DEFAULT ${CMAKE_SOURCE_DIR}/third_party/googletest/src/googletest)
endif()
+ set(GTEST_SRC_DIR ${GTEST_SRC_DIR_DEFAULT} CACHE STRING "Location of gtest sources")
if(EXISTS ${GTEST_SRC_DIR}/src/gtest-all.cc)
message(STATUS "building gtest from sources in ${GTEST_SRC_DIR}")
set(gtest_sources ${GTEST_SRC_DIR}/src/gtest-all.cc)
@@ -98,11 +246,11 @@
# install the conversion tool, .so, .a, and all the header files
-INSTALL ( PROGRAMS ${CMAKE_BINARY_DIR}/yuvconvert DESTINATION bin )
-INSTALL ( TARGETS ${ly_lib_static} DESTINATION lib )
-INSTALL ( TARGETS ${ly_lib_shared} LIBRARY DESTINATION lib RUNTIME DESTINATION bin )
-INSTALL ( DIRECTORY ${PROJECT_SOURCE_DIR}/include/ DESTINATION include )
+install ( TARGETS yuvconvert DESTINATION bin )
+install ( TARGETS ${ly_lib_static} DESTINATION lib )
+install ( TARGETS ${ly_lib_shared} LIBRARY DESTINATION lib RUNTIME DESTINATION bin ARCHIVE DESTINATION lib )
+install ( DIRECTORY ${PROJECT_SOURCE_DIR}/include/ DESTINATION include )
# create the .deb and .rpm packages using cpack
-INCLUDE ( CM_linux_packages.cmake )
+include ( CM_linux_packages.cmake )
diff --git a/DEPS b/DEPS
index 70ed1d5..ecc8001 100644
--- a/DEPS
+++ b/DEPS
@@ -5,39 +5,49 @@
vars = {
'chromium_git': 'https://chromium.googlesource.com',
- 'chromium_revision': 'af3d01376bec75a68f90160bfd38057d60510a2b',
- 'gn_version': 'git_revision:fae280eabe5d31accc53100137459ece19a7a295',
+ 'chromium_revision': '908f3898afdb1fca9352cf3827168a66ff914084',
+ 'gn_version': 'git_revision:c97a86a72105f3328a540f5a5ab17d11989ab7dd',
# ninja CIPD package version.
# https://chrome-infra-packages.appspot.com/p/infra/3pp/tools/ninja
- 'ninja_version': 'version:2@1.11.1.chromium.6',
+ 'ninja_version': 'version:3@1.12.1.chromium.4',
# reclient CIPD package version
- 'reclient_version': 're_client_version:0.110.0.43ec6b1-gomaip',
+ 'reclient_version': 're_client_version:0.172.0.3cf60ba5-gomaip',
+ # Fetch configuration files required for the 'use_remoteexec' gn arg
+ 'download_remoteexec_cfg': False,
+ # RBE instance to use for running remote builds
+ 'rbe_instance': Str('projects/rbe-webrtc-developer/instances/default_instance'),
+ # RBE project to download rewrapper config files for. Only needed if
+ # different from the project used in 'rbe_instance'
+ 'rewrapper_cfg_project': Str(''),
# Keep the Chromium default of generating location tags.
'generate_location_tags': True,
# By default, download the fuchsia sdk from the public sdk directory.
'fuchsia_sdk_cipd_prefix': 'fuchsia/sdk/core/',
- 'fuchsia_version': 'version:15.20230909.2.1',
+ 'fuchsia_version': 'version:26.20250103.4.1',
# By default, download the fuchsia images from the fuchsia GCS bucket.
'fuchsia_images_bucket': 'fuchsia',
'checkout_fuchsia': False,
# Since the images are hundreds of MB, default to only downloading the image
# most commonly useful for developers. Bots and developers that need to use
# other images can override this with additional images.
- 'checkout_fuchsia_boot_images': "terminal.qemu-x64,terminal.x64",
+ 'checkout_fuchsia_boot_images': "terminal.x64",
'checkout_fuchsia_product_bundles': '"{checkout_fuchsia_boot_images}" != ""',
+
+ # condition to allowlist deps for non-git-source processing.
+ 'non_git_source': 'True',
}
deps = {
'src/build':
- Var('chromium_git') + '/chromium/src/build' + '@' + '5885d3c24833ad72845a52a1b913a2b8bc651b56',
+ Var('chromium_git') + '/chromium/src/build' + '@' + 'f3e95cc9a07a76a7700eec429dacf0ea55cf9c01',
'src/buildtools':
- Var('chromium_git') + '/chromium/src/buildtools' + '@' + '79ab87fa54614258c4c95891e873223371194525',
+ Var('chromium_git') + '/chromium/src/buildtools' + '@' + 'dc741883264b58654396d30d5b8d077d4d7d758f',
'src/testing':
- Var('chromium_git') + '/chromium/src/testing' + '@' + '51e9a02297057cc0e917763a51e16680b7d16fb6',
+ Var('chromium_git') + '/chromium/src/testing' + '@' + '4341e4d7a272f95394094b48f1edf1ab7e33c61f',
'src/third_party':
- Var('chromium_git') + '/chromium/src/third_party' + '@' + '2dc4b18abd1003ce7b1eda509dc96f12d49a9667',
+ Var('chromium_git') + '/chromium/src/third_party' + '@' + 'f25a92da849d82fd0adfee562b04bccf1d9a9a03',
'src/buildtools/linux64': {
'packages': [
@@ -83,9 +93,9 @@
},
'src/third_party/catapult':
- Var('chromium_git') + '/catapult.git' + '@' + 'fa05d995e152efdae488a2aeba397cd609fdbc9d',
+ Var('chromium_git') + '/catapult.git' + '@' + '8491e07230d983b9eb55cdfbf075f0a9d4fb5d28',
'src/third_party/clang-format/script':
- Var('chromium_git') + '/external/github.com/llvm/llvm-project/clang/tools/clang-format.git' + '@' + 'f97059df7f8b205064625cdb5f97b56668a125ef',
+ Var('chromium_git') + '/external/github.com/llvm/llvm-project/clang/tools/clang-format.git' + '@' + '37f6e68a107df43b7d7e044fd36a13cbae3413f2',
'src/third_party/colorama/src':
Var('chromium_git') + '/external/colorama.git' + '@' + '3de9f013df4b470069d03d250224062e8cf15c49',
'src/third_party/cpu_features/src': {
@@ -93,29 +103,34 @@
'condition': 'checkout_android',
},
'src/third_party/depot_tools':
- Var('chromium_git') + '/chromium/tools/depot_tools.git' + '@' + 'd3e43dd4319ba169c0aaf44547eecf861f2fe5da',
- 'src/third_party/freetype/src':
- Var('chromium_git') + '/chromium/src/third_party/freetype2.git' + '@' + '9e3c5d7e183c1a8d5ed8868d7d28ef18d3ec9ec8',
+ Var('chromium_git') + '/chromium/tools/depot_tools.git' + '@' + '423f1e1914ab4aa7b2bdf804e216d4c097853ba2',
'third_party/fuchsia-gn-sdk': {
'url': Var('chromium_git') + '/chromium/src/third_party/fuchsia-gn-sdk.git' + '@' + '0d6902558d92fe3d49ba9a8f638ddea829be595b',
'condition': 'checkout_fuchsia',
},
'src/third_party/googletest/src':
- Var('chromium_git') + '/external/github.com/google/googletest.git' + '@' + 'af29db7ec28d6df1c7f0f745186884091e602e07',
+ Var('chromium_git') + '/external/github.com/google/googletest.git' + '@' + '7d76a231b0e29caf86e68d1df858308cd53b2a66',
'src/third_party/harfbuzz-ng/src':
- Var('chromium_git') + '/external/github.com/harfbuzz/harfbuzz.git' + '@' + 'db700b5670d9475cc8ed4880cc9447b232c5e432',
+ Var('chromium_git') + '/external/github.com/harfbuzz/harfbuzz.git' + '@' + '1c249be96e27eafd15eb86d832b67fbc3751634b',
+ 'src/third_party/instrumented_libs': {
+ 'url': Var('chromium_git') + '/chromium/third_party/instrumented_libraries.git' + '@' + '3cc43119a29158bcde39d288a8def4b8ec49baf8',
+ 'condition': 'checkout_instrumented_libraries',
+ },
'src/third_party/libc++/src':
- Var('chromium_git') + '/external/github.com/llvm/llvm-project/libcxx.git' + '@' + '84fb809dd6dae36d556dc0bb702c6cc2ce9d4b80',
+ Var('chromium_git') + '/external/github.com/llvm/llvm-project/libcxx.git' + '@' + '74dd760826063f15ce4d373b31bc34d1c65e41c2',
'src/third_party/libc++abi/src':
- Var('chromium_git') + '/external/github.com/llvm/llvm-project/libcxxabi.git' + '@' + '8d21803b9076b16d46c32e2f10da191ee758520c',
+ Var('chromium_git') + '/external/github.com/llvm/llvm-project/libcxxabi.git' + '@' + '7681005c6233e8a21b97e24c1a3c5c6979927d5a',
+ 'src/third_party/llvm-libc/src':
+ Var('chromium_git') + '/external/github.com/llvm/llvm-project/libc.git' + '@' + '2019a9e40b0bac10ad29f1a9e2288307b6ef16ce',
+
'src/third_party/libunwind/src':
- Var('chromium_git') + '/external/github.com/llvm/llvm-project/libunwind.git' + '@' + 'f1c687e0aaf0d70b9a53a150e9be5cb63af9215f',
+ Var('chromium_git') + '/external/github.com/llvm/llvm-project/libunwind.git' + '@' + 'd1e95b102f113ded38974cf06a65fe0457b6004b',
'src/third_party/libjpeg_turbo':
- Var('chromium_git') + '/chromium/deps/libjpeg_turbo.git' + '@' + '30bdb85e302ecfc52593636b2f44af438e05e784',
+ Var('chromium_git') + '/chromium/deps/libjpeg_turbo.git' + '@' + '927aabfcd26897abb9776ecf2a6c38ea5bb52ab6',
'src/third_party/nasm':
- Var('chromium_git') + '/chromium/deps/nasm.git' + '@' + '7fc833e889d1afda72c06220e5bed8fb43b2e5ce',
+ Var('chromium_git') + '/chromium/deps/nasm.git' + '@' + 'f477acb1049f5e043904b87b825c5915084a9a29',
'src/tools':
- Var('chromium_git') + '/chromium/src/tools' + '@' + 'a76c0dbb64c603a0d45e0c6dfae3a351b6e1adf1',
+ Var('chromium_git') + '/chromium/src/tools' + '@' + '09973d22d8f4110b35bbf133c613b4df2290d465',
# libyuv-only dependencies (not present in Chromium).
'src/third_party/gtest-parallel':
@@ -126,87 +141,54 @@
'condition': 'checkout_android or checkout_linux',
},
+ 'src/third_party/re2/src':
+ Var('chromium_git') + '/external/github.com/google/re2.git' + '@' + '6dcd83d60f7944926bfd308cc13979fc53dd69ca',
+
# Android deps:
- 'src/third_party/accessibility_test_framework': {
+ 'src/third_party/kotlin_stdlib/cipd': {
'packages': [
{
- 'package': 'chromium/third_party/accessibility-test-framework',
- 'version': 'b5ec1e56e58e56bc1a0c77d43111c37f9b512c8a',
+ 'package': 'chromium/third_party/kotlin_stdlib',
+ 'version': 'uguVAY3NvbfV4KgHrjjwvtTioMwPwSijfAgBPpbaYk0C',
},
],
'condition': 'checkout_android',
'dep_type': 'cipd',
},
- 'src/third_party/kotlin_stdlib': {
- 'packages': [
- {
- 'package': 'chromium/third_party/kotlin_stdlib',
- 'version': 'Z1gsqhL967kFQecxKrRwXHbl-vwQjpv0l7PMUZ0EVO8C',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
'src/third_party/kotlinc/current': {
'packages': [
{
'package': 'chromium/third_party/kotlinc',
- 'version': 'Rr02Gf2EkaeSs3EhSUHhPqDHSd1AzimrM6cRYUJCPjQC',
+ 'version': 'YrBSUjA4zjPf3DhU2SYlqamxAAQiM2WIeZftsDSjqTAC',
},
],
'condition': 'checkout_android',
'dep_type': 'cipd',
},
- 'src/third_party/boringssl/src':
- 'https://boringssl.googlesource.com/boringssl.git' + '@' + '20a06474c0b4a16779311bfe98ba69dc2402101d',
- 'src/base': {
- 'url': Var('chromium_git') + '/chromium/src/base' + '@' + 'd407b7061bce341bb6e11b539ea86c46c949ac4c',
- 'condition': 'checkout_android',
- },
- 'src/third_party/bazel': {
- 'packages': [
- {
- 'package': 'chromium/third_party/bazel',
- 'version': 'VjMsf48QUWw8n7XtJP2AuSjIGmbQeYdWdwyxVvIRLmAC',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
- 'src/third_party/bouncycastle': {
- 'packages': [
- {
- 'package': 'chromium/third_party/bouncycastle',
- 'version': 'c078e87552ba26e776566fdaf0f22cd8712743d0',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
- 'src/third_party/android_toolchain': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_toolchain/android_toolchain',
- 'version': 'R_8suM8m0oHbZ1awdxGXvKEFpAOETscbfZxkkMthyk8C',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/androidx': {
+ 'src/third_party/android_toolchain/ndk': {
'packages': [
{
- 'package': 'chromium/third_party/androidx',
- 'version': 'y7rF_rx56mD3FGhMiqnlbQ6HOqHJ95xUFNX1m-_a988C',
+ 'package': 'chromium/third_party/android_toolchain/android_toolchain',
+ 'version': 'Idl-vYnWGnM8K3XJhM3h6zjYVDXlnljVz3FE00V9IM8C',
},
],
'condition': 'checkout_android',
'dep_type': 'cipd',
},
+ 'src/third_party/androidx/cipd': {
+ 'packages': [
+ {
+ 'package': 'chromium/third_party/androidx',
+ 'version': 'gUjEawxv5mQO8yfbuC8W-rx4V3zYE-4LTWggXpZHI4sC',
+ },
+ ],
+ 'condition': 'checkout_android and non_git_source',
+ 'dep_type': 'cipd',
+ },
+
'src/third_party/android_support_test_runner': {
'packages': [
{
@@ -219,38 +201,26 @@
},
'src/third_party/android_sdk/public': {
'packages': [
- {
- 'package': 'chromium/third_party/android_sdk/public/build-tools/34.0.0',
- 'version': 'YK9Rzw3fDzMHVzatNN6VlyoD_81amLZpN1AbmkdOd6AC',
- },
- {
+ {
+ 'package': 'chromium/third_party/android_sdk/public/build-tools/35.0.0',
+ 'version': 'DxwAZ3hD551Neu6ycuW5CPnXFrdleRBd93oX1eB_m9YC'
+ },
+ {
'package': 'chromium/third_party/android_sdk/public/emulator',
- 'version': '9lGp8nTUCRRWGMnI_96HcKfzjnxEJKUcfvfwmA3wXNkC',
- },
- {
- 'package': 'chromium/third_party/android_sdk/public/patcher',
- 'version': 'I6FNMhrXlpB-E1lOhMlvld7xt9lBVNOO83KIluXDyA0C',
- },
- {
+ 'version': '9lGp8nTUCRRWGMnI_96HcKfzjnxEJKUcfvfwmA3wXNkC'
+ },
+ {
'package': 'chromium/third_party/android_sdk/public/platform-tools',
- 'version': 'HWVsGs2HCKgSVv41FsOcsfJbNcB0UFiNrF6Tc4yRArYC',
- },
- {
- 'package': 'chromium/third_party/android_sdk/public/platforms/android-34',
- 'version': 'u-bhWbTME6u-DjypTgr3ZikCyeAeU6txkR9ET6Uudc8C',
- },
- {
- 'package': 'chromium/third_party/android_sdk/public/platforms/android-tiramisuprivacysandbox',
- 'version': 'YWMYkzyxGBgVsty0GhXL1oxbY0pGXQIgFc0Rh7ZMRPYC',
- },
- {
- 'package': 'chromium/third_party/android_sdk/public/sources/android-31',
- 'version': '_a_BcnANjPYw5mSKlNHa7GFY8yc1kdqj2rmQgac7yUcC',
- },
- {
+ 'version': 'WihaseZR6cojZbkzIqwGhpTp92ztaGfqq8njBU8eTXYC'
+ },
+ {
+ 'package': 'chromium/third_party/android_sdk/public/platforms/android-35',
+ 'version': 'kIXA-9XuCfOESodXEdOBkW5f1ytrGWdbp3HFp1I8A_0C'
+ },
+ {
'package': 'chromium/third_party/android_sdk/public/cmdline-tools',
- 'version': 'EWnL2r7oV5GtE9Ef7GyohyFam42wtMtEKYU4dCb3U1YC',
- },
+ 'version': 'OCGHZKTdjXjIELVI6FMAgcDkfnd_1ybB0_MtljPtz-8C'
+ }
],
'condition': 'checkout_android',
'dep_type': 'cipd',
@@ -265,36 +235,19 @@
'condition': 'checkout_mac',
'dep_type': 'cipd',
},
- 'src/third_party/android_build_tools/aapt2': {
+
+
+ 'src/third_party/android_build_tools/aapt2/cipd': {
'packages': [
{
'package': 'chromium/third_party/android_build_tools/aapt2',
- 'version': 'STY0BXlZxsEhudnlXQFed-B5UpwehcoM0sYqor6qRqsC',
+ 'version': '_lNsOL_GGlXLOIMGtrbMOqNd7TQHabaP1q8SlvUpFbMC',
},
],
'condition': 'checkout_android',
'dep_type': 'cipd',
},
- 'src/third_party/byte_buddy': {
- 'packages': [
- {
- 'package': 'chromium/third_party/byte_buddy',
- 'version': 'c9b53316603fc2d997c899c7ca1707f809b918cd',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
- 'src/third_party/byte_buddy/android_sdk_build_tools_25_0_2': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_sdk/public/build-tools',
- 'version': 'kwIs2vdfTm93yEP8LG5aSnchN4BVEdVxbqQtF4XpPdkC',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
+
'src/third_party/ced/src': {
'url': Var('chromium_git') + '/external/github.com/google/compact_enc_det.git' + '@' + 'ba412eaaacd3186085babcd901679a48863c7dd5',
'condition': 'checkout_android',
@@ -317,40 +270,34 @@
'condition': 'checkout_android',
'dep_type': 'cipd',
},
- 'src/third_party/guava': {
- 'packages': [
- {
- 'package': 'chromium/third_party/guava',
- 'version': 'a6fba501f3a0de88b9be1daa2052632de5b96a46',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
- 'src/third_party/hamcrest': {
+
+ 'src/third_party/hamcrest/cipd': {
'packages': [
{
'package': 'chromium/third_party/hamcrest',
- 'version': '37eccfc658fe79695d6abb6dd497463c4372032f',
+ 'version': 'dBioOAmFJjqAr_DY7dipbXdVfAxUQwjOBNibMPtX8lQC',
},
],
'condition': 'checkout_android',
'dep_type': 'cipd',
},
+
'src/third_party/icu': {
- 'url': Var('chromium_git') + '/chromium/deps/icu.git' + '@' + 'e8c3bc9ea97d4423ad0515e5f1c064f486dae8b1',
+ 'url': Var('chromium_git') + '/chromium/deps/icu.git' + '@' + 'bbccc2f6efc1b825de5f2c903c48be685cd0cf22',
},
- 'src/third_party/icu4j': {
+
+ 'src/third_party/icu4j/cipd': {
'packages': [
{
'package': 'chromium/third_party/icu4j',
- 'version': 'e87e5bed2b4935913ee26a3ebd0b723ee2344354',
+ 'version': '8dV7WRVX0tTaNNqkLEnCA_dMofr2MJXFK400E7gOFygC',
},
],
'condition': 'checkout_android',
'dep_type': 'cipd',
},
+
'src/third_party/intellij': {
'packages': [
{
@@ -361,14 +308,14 @@
'condition': 'checkout_android',
'dep_type': 'cipd',
},
- 'src/third_party/jdk': {
+ 'src/third_party/jdk/current': {
'packages': [
{
'package': 'chromium/third_party/jdk',
- 'version': 'GCFtf5t6M4HlrHj6NXedHbpHp2xjgognF8ptNci4478C',
+ 'version': 'G-WIGWdFIBTPPhv2xPK_lTfI5N7WmfCBh0oTKqX-c5sC',
},
- ],
- 'condition': 'checkout_android',
+ ],
+ 'condition': 'host_os == "linux" and checkout_android',
'dep_type': 'cipd',
},
'src/third_party/jsr-305/src': {
@@ -376,11 +323,11 @@
'condition': 'checkout_android',
},
'src/third_party/junit/src': {
- 'url': Var('chromium_git') + '/external/junit.git' + '@' + '05fe2a64f59127c02135be22f416e91260d6ede6',
+ 'url': Var('chromium_git') + '/external/junit.git' + '@' + '0eb5ce72848d730da5bd6d42902fdd6a8a42055d',
'condition': 'checkout_android',
},
'src/third_party/libunwindstack': {
- 'url': Var('chromium_git') + '/chromium/src/third_party/libunwindstack.git' + '@' + '4dbfa0e8c844c8e243b297bc185e54a99ff94f9e',
+ 'url': Var('chromium_git') + '/chromium/src/third_party/libunwindstack.git' + '@' + '215bddfd8eecb7dd990494854a31b70a96d71b3c',
'condition': 'checkout_android',
},
'src/third_party/ninja': {
@@ -396,16 +343,6 @@
'url': Var('chromium_git') + '/external/mockito/mockito.git' + '@' + '7c3641bcef717ffa7d765f2c86b847d0aab1aac9',
'condition': 'checkout_android',
},
- 'src/third_party/objenesis': {
- 'packages': [
- {
- 'package': 'chromium/third_party/objenesis',
- 'version': 'tknDblENYi8IaJYyD6tUahUyHYZlzJ_Y74_QZSz4DpIC',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
'src/third_party/ow2_asm': {
'packages': [
{
@@ -416,11 +353,12 @@
'condition': 'checkout_android',
'dep_type': 'cipd',
},
- 'src/third_party/r8': {
+
+ 'src/third_party/r8/cipd': {
'packages': [
{
'package': 'chromium/third_party/r8',
- 'version': 'O1BBWiBTIeNUcraX8STMtQXVaCleu6SJJjWCcnfhPLkC',
+ 'version': 'TQJgBofMEzGILWhAM0LXeob_ZpAiDc8w8SBzU0d8o8YC',
},
],
'condition': 'checkout_android',
@@ -429,11 +367,11 @@
# This duplication is intentional, so we avoid updating the r8.jar used by
# dexing unless necessary, since each update invalidates all incremental
# dexing and unnecessarily slows down all bots.
- 'src/third_party/r8/d8': {
+ 'src/third_party/r8/d8/cipd': {
'packages': [
{
'package': 'chromium/third_party/r8',
- 'version': 'vw5kLlW3-suSlCKSO9OQpFWpR8oDnvQ8k1RgKNUapQYC',
+ 'version': 'U3Jf_ewWOZyxa6vyO3wjNIgm8XIz1yFk-4k3-wqDL44C',
},
],
'condition': 'checkout_android',
@@ -453,17 +391,19 @@
'url': Var('chromium_git') + '/external/github.com/kennethreitz/requests.git' + '@' + 'c7e0fc087ceeadb8b4c84a0953a422c474093d6d',
'condition': 'checkout_android',
},
- 'src/third_party/robolectric': {
+
+ 'src/third_party/robolectric/cipd': {
'packages': [
{
'package': 'chromium/third_party/robolectric',
- 'version': 'hzetqh1qFI32FOgQroZvGcGdomrgVBJ6WKRnl1KFw6EC',
+ 'version': 'G3VkWqTv1YWDvC6zCrL34iQREzrzdBmSL4GMboAIiAEC',
},
],
'condition': 'checkout_android',
'dep_type': 'cipd',
},
- 'src/third_party/sqlite4java': {
+
+ 'src/third_party/sqlite4java/cipd': {
'packages': [
{
'package': 'chromium/third_party/sqlite4java',
@@ -473,11 +413,11 @@
'condition': 'checkout_android',
'dep_type': 'cipd',
},
- 'src/third_party/turbine': {
+ 'src/third_party/turbine/cipd': {
'packages': [
{
'package': 'chromium/third_party/turbine',
- 'version': '2I2Nz480QsuCxpQ1lMfbigX8l5HAhX3_ykWU4TKRGo4C',
+ 'version': 'dz8pRLjwNlToJ0tS14T-TDQJNikmFXEDByMo-OzBbl0C',
},
],
'condition': 'checkout_android',
@@ -490,1841 +430,1314 @@
# iOS deps:
'src/ios': {
- 'url': Var('chromium_git') + '/chromium/src/ios' + '@' + 'ddd58e86cf4ebdc0db60a5d0f3c323de49bb295c',
+ 'url': Var('chromium_git') + '/chromium/src/ios' + '@' + '6e4e345fbb2b4f25bb4072d0d0b1cac4ee887126',
'condition': 'checkout_ios'
},
# Everything coming after this is automatically updated by the auto-roller.
# === ANDROID_DEPS Generated Code Start ===
# Generated by //third_party/android_deps/fetch_all.py
- 'src/third_party/android_deps/libs/android_arch_core_common': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/android_arch_core_common',
- 'version': 'version:2@1.1.1.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/android_arch_core_runtime': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/android_arch_core_runtime',
- 'version': 'version:2@1.1.1.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/android_arch_lifecycle_common': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/android_arch_lifecycle_common',
- 'version': 'version:2@1.1.1.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/android_arch_lifecycle_common_java8': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/android_arch_lifecycle_common_java8',
- 'version': 'version:2@1.1.1.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/android_arch_lifecycle_livedata': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/android_arch_lifecycle_livedata',
- 'version': 'version:2@1.1.1.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/android_arch_lifecycle_livedata_core': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/android_arch_lifecycle_livedata_core',
- 'version': 'version:2@1.1.1.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/android_arch_lifecycle_runtime': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/android_arch_lifecycle_runtime',
- 'version': 'version:2@1.1.1.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/android_arch_lifecycle_viewmodel': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/android_arch_lifecycle_viewmodel',
- 'version': 'version:2@1.1.1.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_animated_vector_drawable': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_animated_vector_drawable',
- 'version': 'version:2@28.0.0.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_appcompat_v7': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_appcompat_v7',
- 'version': 'version:2@28.0.0.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_asynclayoutinflater': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_asynclayoutinflater',
- 'version': 'version:2@28.0.0.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_cardview_v7': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_cardview_v7',
- 'version': 'version:2@28.0.0.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_collections': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_collections',
- 'version': 'version:2@28.0.0.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_coordinatorlayout': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_coordinatorlayout',
- 'version': 'version:2@28.0.0.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_cursoradapter': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_cursoradapter',
- 'version': 'version:2@28.0.0.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_customview': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_customview',
- 'version': 'version:2@28.0.0.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_design': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_design',
- 'version': 'version:2@28.0.0.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_documentfile': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_documentfile',
- 'version': 'version:2@28.0.0.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_drawerlayout': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_drawerlayout',
- 'version': 'version:2@28.0.0.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_interpolator': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_interpolator',
- 'version': 'version:2@28.0.0.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_loader': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_loader',
- 'version': 'version:2@28.0.0.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_localbroadcastmanager': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_localbroadcastmanager',
- 'version': 'version:2@28.0.0.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_multidex': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_multidex',
- 'version': 'version:2@1.0.0.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_print': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_print',
- 'version': 'version:2@28.0.0.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_recyclerview_v7': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_recyclerview_v7',
- 'version': 'version:2@28.0.0.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_slidingpanelayout': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_slidingpanelayout',
- 'version': 'version:2@28.0.0.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_support_annotations': {
+ 'src/third_party/android_deps/cipd/libs/com_android_support_support_annotations': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/com_android_support_support_annotations',
'version': 'version:2@28.0.0.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_android_support_support_compat': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_support_compat',
- 'version': 'version:2@28.0.0.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_support_core_ui': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_support_core_ui',
- 'version': 'version:2@28.0.0.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_support_core_utils': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_support_core_utils',
- 'version': 'version:2@28.0.0.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_support_fragment': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_support_fragment',
- 'version': 'version:2@28.0.0.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_support_media_compat': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_support_media_compat',
- 'version': 'version:2@28.0.0.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_support_v4': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_support_v4',
- 'version': 'version:2@28.0.0.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_support_vector_drawable': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_support_vector_drawable',
- 'version': 'version:2@28.0.0.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_swiperefreshlayout': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_swiperefreshlayout',
- 'version': 'version:2@28.0.0.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_transition': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_transition',
- 'version': 'version:2@28.0.0.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_versionedparcelable': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_versionedparcelable',
- 'version': 'version:2@28.0.0.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_support_viewpager': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_android_support_viewpager',
- 'version': 'version:2@28.0.0.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_android_tools_common': {
+ 'src/third_party/android_deps/cipd/libs/com_android_tools_common': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/com_android_tools_common',
'version': 'version:2@30.2.0-beta01.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_android_tools_layoutlib_layoutlib_api': {
+ 'src/third_party/android_deps/cipd/libs/com_android_tools_layoutlib_layoutlib_api': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/com_android_tools_layoutlib_layoutlib_api',
'version': 'version:2@30.2.0-beta01.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_android_tools_sdk_common': {
+ 'src/third_party/android_deps/cipd/libs/com_android_tools_sdk_common': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/com_android_tools_sdk_common',
'version': 'version:2@30.2.0-beta01.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_github_ben_manes_caffeine_caffeine': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_github_ben_manes_caffeine_caffeine',
- 'version': 'version:2@2.8.8.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_github_kevinstern_software_and_algorithms': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_github_kevinstern_software_and_algorithms',
- 'version': 'version:2@1.0.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_google_android_annotations': {
+ 'src/third_party/android_deps/cipd/libs/com_google_android_annotations': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/com_google_android_annotations',
'version': 'version:2@4.1.1.4.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_android_apps_common_testing_accessibility_framework_accessibility_test_framework': {
+ 'src/third_party/android_deps/cipd/libs/com_google_android_apps_common_testing_accessibility_framework_accessibility_test_framework': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/com_google_android_apps_common_testing_accessibility_framework_accessibility_test_framework',
'version': 'version:2@4.0.0.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_android_datatransport_transport_api': {
+ 'src/third_party/android_deps/cipd/libs/com_google_android_datatransport_transport_api': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/com_google_android_datatransport_transport_api',
'version': 'version:2@2.2.1.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_android_gms_play_services_auth': {
+ 'src/third_party/android_deps/cipd/libs/com_google_android_gms_play_services_auth': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/com_google_android_gms_play_services_auth',
- 'version': 'version:2@20.1.0.cr1',
+ 'version': 'version:2@21.1.1.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_android_gms_play_services_auth_api_phone': {
+ 'src/third_party/android_deps/cipd/libs/com_google_android_gms_play_services_auth_api_phone': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/com_google_android_gms_play_services_auth_api_phone',
- 'version': 'version:2@18.0.1.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_google_android_gms_play_services_auth_base': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_google_android_gms_play_services_auth_base',
'version': 'version:2@18.0.2.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_android_gms_play_services_base': {
+ 'src/third_party/android_deps/cipd/libs/com_google_android_gms_play_services_auth_base': {
+ 'packages': [
+ {
+ 'package': 'chromium/third_party/android_deps/libs/com_google_android_gms_play_services_auth_base',
+ 'version': 'version:2@18.0.10.cr1',
+ },
+ ],
+ 'condition': 'checkout_android and non_git_source',
+ 'dep_type': 'cipd',
+ },
+
+ 'src/third_party/android_deps/cipd/libs/com_google_android_gms_play_services_auth_blockstore': {
+ 'packages': [
+ {
+ 'package': 'chromium/third_party/android_deps/libs/com_google_android_gms_play_services_auth_blockstore',
+ 'version': 'version:2@16.4.0.cr1',
+ },
+ ],
+ 'condition': 'checkout_android and non_git_source',
+ 'dep_type': 'cipd',
+ },
+
+ 'src/third_party/android_deps/cipd/libs/com_google_android_gms_play_services_base': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/com_google_android_gms_play_services_base',
- 'version': 'version:2@18.0.1.cr1',
+ 'version': 'version:2@18.5.0.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_android_gms_play_services_basement': {
+ 'src/third_party/android_deps/cipd/libs/com_google_android_gms_play_services_basement': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/com_google_android_gms_play_services_basement',
- 'version': 'version:2@18.1.0.cr1',
+ 'version': 'version:2@18.4.0.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_android_gms_play_services_cast': {
+ 'src/third_party/android_deps/cipd/libs/com_google_android_gms_play_services_cast': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/com_google_android_gms_play_services_cast',
'version': 'version:2@17.0.0.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_android_gms_play_services_cast_framework': {
+ 'src/third_party/android_deps/cipd/libs/com_google_android_gms_play_services_cast_framework': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/com_google_android_gms_play_services_cast_framework',
'version': 'version:2@17.0.0.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_android_gms_play_services_clearcut': {
+ 'src/third_party/android_deps/cipd/libs/com_google_android_gms_play_services_clearcut': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/com_google_android_gms_play_services_clearcut',
'version': 'version:2@17.0.0.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_android_gms_play_services_cloud_messaging': {
+ 'src/third_party/android_deps/cipd/libs/com_google_android_gms_play_services_cloud_messaging': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/com_google_android_gms_play_services_cloud_messaging',
'version': 'version:2@16.0.0.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_android_gms_play_services_flags': {
+ 'src/third_party/android_deps/cipd/libs/com_google_android_gms_play_services_fido': {
+ 'packages': [
+ {
+ 'package': 'chromium/third_party/android_deps/libs/com_google_android_gms_play_services_fido',
+ 'version': 'version:2@21.1.0.cr1',
+ },
+ ],
+ 'condition': 'checkout_android and non_git_source',
+ 'dep_type': 'cipd',
+ },
+
+ 'src/third_party/android_deps/cipd/libs/com_google_android_gms_play_services_flags': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/com_google_android_gms_play_services_flags',
'version': 'version:2@17.0.0.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_android_gms_play_services_gcm': {
+ 'src/third_party/android_deps/cipd/libs/com_google_android_gms_play_services_gcm': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/com_google_android_gms_play_services_gcm',
'version': 'version:2@17.0.0.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_android_gms_play_services_iid': {
+ 'src/third_party/android_deps/cipd/libs/com_google_android_gms_play_services_identity_credentials': {
+ 'packages': [
+ {
+ 'package': 'chromium/third_party/android_deps/libs/com_google_android_gms_play_services_identity_credentials',
+ 'version': 'version:2@16.0.0-alpha02.cr1',
+ },
+ ],
+ 'condition': 'checkout_android and non_git_source',
+ 'dep_type': 'cipd',
+ },
+
+ 'src/third_party/android_deps/cipd/libs/com_google_android_gms_play_services_iid': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/com_google_android_gms_play_services_iid',
'version': 'version:2@17.0.0.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_android_gms_play_services_instantapps': {
+ 'src/third_party/android_deps/cipd/libs/com_google_android_gms_play_services_instantapps': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/com_google_android_gms_play_services_instantapps',
'version': 'version:2@18.0.1.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_android_gms_play_services_location': {
+ 'src/third_party/android_deps/cipd/libs/com_google_android_gms_play_services_location': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/com_google_android_gms_play_services_location',
- 'version': 'version:2@19.0.1.cr1',
+ 'version': 'version:2@21.0.1.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_android_gms_play_services_phenotype': {
+ 'src/third_party/android_deps/cipd/libs/com_google_android_gms_play_services_phenotype': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/com_google_android_gms_play_services_phenotype',
'version': 'version:2@17.0.0.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_android_gms_play_services_places_placereport': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_google_android_gms_play_services_places_placereport',
- 'version': 'version:2@17.0.0.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_google_android_gms_play_services_stats': {
+ 'src/third_party/android_deps/cipd/libs/com_google_android_gms_play_services_stats': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/com_google_android_gms_play_services_stats',
'version': 'version:2@17.0.0.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_android_gms_play_services_tasks': {
+ 'src/third_party/android_deps/cipd/libs/com_google_android_gms_play_services_tasks': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/com_google_android_gms_play_services_tasks',
- 'version': 'version:2@18.0.2.cr1',
+ 'version': 'version:2@18.2.0.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_android_gms_play_services_vision': {
+ 'src/third_party/android_deps/cipd/libs/com_google_android_gms_play_services_vision': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/com_google_android_gms_play_services_vision',
'version': 'version:2@20.1.3.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_android_gms_play_services_vision_common': {
+ 'src/third_party/android_deps/cipd/libs/com_google_android_gms_play_services_vision_common': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/com_google_android_gms_play_services_vision_common',
'version': 'version:2@19.1.3.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_android_material_material': {
+ 'src/third_party/android_deps/cipd/libs/com_google_android_libraries_identity_googleid_googleid': {
+ 'packages': [
+ {
+ 'package': 'chromium/third_party/android_deps/libs/com_google_android_libraries_identity_googleid_googleid',
+ 'version': 'version:2@1.1.1.cr1',
+ },
+ ],
+ 'condition': 'checkout_android and non_git_source',
+ 'dep_type': 'cipd',
+ },
+
+ 'src/third_party/android_deps/cipd/libs/com_google_android_material_material': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/com_google_android_material_material',
- 'version': 'version:2@1.7.0-alpha02.cr1',
+ 'version': 'version:2@1.13.0-alpha05.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_android_play_core_common': {
+ 'src/third_party/android_deps/cipd/libs/com_google_android_play_core_common': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/com_google_android_play_core_common',
'version': 'version:2@2.0.2.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_android_play_feature_delivery': {
+ 'src/third_party/android_deps/cipd/libs/com_google_android_play_feature_delivery': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/com_google_android_play_feature_delivery',
'version': 'version:2@2.0.1.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_auto_auto_common': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_google_auto_auto_common',
- 'version': 'version:2@1.2.1.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_google_auto_service_auto_service': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_google_auto_service_auto_service',
- 'version': 'version:2@1.0-rc6.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_google_auto_service_auto_service_annotations': {
+ 'src/third_party/android_deps/cipd/libs/com_google_auto_service_auto_service_annotations': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/com_google_auto_service_auto_service_annotations',
'version': 'version:2@1.0-rc6.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_auto_value_auto_value_annotations': {
+ 'src/third_party/android_deps/cipd/libs/com_google_auto_value_auto_value_annotations': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/com_google_auto_value_auto_value_annotations',
- 'version': 'version:2@1.10.1.cr1',
+ 'version': 'version:2@1.11.0.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_code_findbugs_jsr305': {
+ 'src/third_party/android_deps/cipd/libs/com_google_code_findbugs_jsr305': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/com_google_code_findbugs_jsr305',
'version': 'version:2@3.0.2.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_code_gson_gson': {
+ 'src/third_party/android_deps/cipd/libs/com_google_code_gson_gson': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/com_google_code_gson_gson',
'version': 'version:2@2.9.0.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_dagger_dagger': {
+ 'src/third_party/android_deps/cipd/libs/com_google_dagger_dagger': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/com_google_dagger_dagger',
- 'version': 'version:2@2.30.cr1',
+ 'version': 'version:2@2.52.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_dagger_dagger_compiler': {
+ 'src/third_party/android_deps/cipd/libs/com_google_dagger_hilt_core': {
'packages': [
{
- 'package': 'chromium/third_party/android_deps/libs/com_google_dagger_dagger_compiler',
- 'version': 'version:2@2.30.cr1',
+ 'package': 'chromium/third_party/android_deps/libs/com_google_dagger_hilt_core',
+ 'version': 'version:2@2.52.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_dagger_dagger_producers': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_google_dagger_dagger_producers',
- 'version': 'version:2@2.30.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_google_dagger_dagger_spi': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_google_dagger_dagger_spi',
- 'version': 'version:2@2.30.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_google_errorprone_error_prone_annotation': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_google_errorprone_error_prone_annotation',
- 'version': 'version:2@2.11.0.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_google_errorprone_error_prone_annotations': {
+ 'src/third_party/android_deps/cipd/libs/com_google_errorprone_error_prone_annotations': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/com_google_errorprone_error_prone_annotations',
- 'version': 'version:2@2.18.0.cr1',
+ 'version': 'version:2@2.30.0.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_errorprone_error_prone_check_api': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_google_errorprone_error_prone_check_api',
- 'version': 'version:2@2.11.0.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_google_errorprone_error_prone_core': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_google_errorprone_error_prone_core',
- 'version': 'version:2@2.11.0.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_google_errorprone_error_prone_type_annotations': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_google_errorprone_error_prone_type_annotations',
- 'version': 'version:2@2.11.0.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_google_errorprone_javac': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_google_errorprone_javac',
- 'version': 'version:2@9+181-r4173-1.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_google_errorprone_javac_shaded': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_google_errorprone_javac_shaded',
- 'version': 'version:2@9-dev-r4023-3.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_google_firebase_firebase_annotations': {
+ 'src/third_party/android_deps/cipd/libs/com_google_firebase_firebase_annotations': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/com_google_firebase_firebase_annotations',
'version': 'version:2@16.0.0.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_firebase_firebase_common': {
+ 'src/third_party/android_deps/cipd/libs/com_google_firebase_firebase_common': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/com_google_firebase_firebase_common',
'version': 'version:2@19.5.0.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_firebase_firebase_components': {
+ 'src/third_party/android_deps/cipd/libs/com_google_firebase_firebase_components': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/com_google_firebase_firebase_components',
'version': 'version:2@16.1.0.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_firebase_firebase_encoders': {
+ 'src/third_party/android_deps/cipd/libs/com_google_firebase_firebase_encoders': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/com_google_firebase_firebase_encoders',
'version': 'version:2@16.1.0.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_firebase_firebase_encoders_json': {
+ 'src/third_party/android_deps/cipd/libs/com_google_firebase_firebase_encoders_json': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/com_google_firebase_firebase_encoders_json',
'version': 'version:2@17.1.0.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_firebase_firebase_iid': {
+ 'src/third_party/android_deps/cipd/libs/com_google_firebase_firebase_iid': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/com_google_firebase_firebase_iid',
'version': 'version:2@21.0.1.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_firebase_firebase_iid_interop': {
+ 'src/third_party/android_deps/cipd/libs/com_google_firebase_firebase_iid_interop': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/com_google_firebase_firebase_iid_interop',
'version': 'version:2@17.0.0.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_firebase_firebase_installations': {
+ 'src/third_party/android_deps/cipd/libs/com_google_firebase_firebase_installations': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/com_google_firebase_firebase_installations',
'version': 'version:2@16.3.5.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_firebase_firebase_installations_interop': {
+ 'src/third_party/android_deps/cipd/libs/com_google_firebase_firebase_installations_interop': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/com_google_firebase_firebase_installations_interop',
'version': 'version:2@16.0.1.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_firebase_firebase_measurement_connector': {
+ 'src/third_party/android_deps/cipd/libs/com_google_firebase_firebase_measurement_connector': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/com_google_firebase_firebase_measurement_connector',
'version': 'version:2@18.0.0.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_firebase_firebase_messaging': {
+ 'src/third_party/android_deps/cipd/libs/com_google_firebase_firebase_messaging': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/com_google_firebase_firebase_messaging',
'version': 'version:2@21.0.1.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_googlejavaformat_google_java_format': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_google_googlejavaformat_google_java_format',
- 'version': 'version:2@1.5.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_google_guava_failureaccess': {
+ 'src/third_party/android_deps/cipd/libs/com_google_guava_failureaccess': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/com_google_guava_failureaccess',
- 'version': 'version:2@1.0.1.cr1',
+ 'version': 'version:2@1.0.2.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_guava_guava': {
+ 'src/third_party/android_deps/cipd/libs/com_google_guava_guava': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/com_google_guava_guava',
- 'version': 'version:2@31.1-jre.cr1',
+ 'version': 'version:2@33.3.1-jre.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_guava_guava_android': {
+ 'src/third_party/android_deps/cipd/libs/com_google_guava_guava_android': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/com_google_guava_guava_android',
- 'version': 'version:2@31.1-android.cr1',
+ 'version': 'version:2@33.3.1-android.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_guava_listenablefuture': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_google_guava_listenablefuture',
- 'version': 'version:2@1.0.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_google_j2objc_j2objc_annotations': {
+ 'src/third_party/android_deps/cipd/libs/com_google_j2objc_j2objc_annotations': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/com_google_j2objc_j2objc_annotations',
- 'version': 'version:2@1.3.cr1',
+ 'version': 'version:2@3.0.0.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_google_protobuf_protobuf_java': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/com_google_protobuf_protobuf_java',
- 'version': 'version:2@3.19.2.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/com_google_protobuf_protobuf_javalite': {
+ 'src/third_party/android_deps/cipd/libs/com_google_protobuf_protobuf_javalite': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/com_google_protobuf_protobuf_javalite',
- 'version': 'version:2@3.21.1.cr1',
+ 'version': 'version:2@4.28.0.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_googlecode_java_diff_utils_diffutils': {
+ 'src/third_party/android_deps/cipd/libs/com_google_testparameterinjector_test_parameter_injector': {
+ 'packages': [
+ {
+ 'package': 'chromium/third_party/android_deps/libs/com_google_testparameterinjector_test_parameter_injector',
+ 'version': 'version:2@1.18.cr1',
+ },
+ ],
+ 'condition': 'checkout_android and non_git_source',
+ 'dep_type': 'cipd',
+ },
+
+ 'src/third_party/android_deps/cipd/libs/com_googlecode_java_diff_utils_diffutils': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/com_googlecode_java_diff_utils_diffutils',
'version': 'version:2@1.3.0.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_squareup_javapoet': {
+ 'src/third_party/android_deps/cipd/libs/com_squareup_javapoet': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/com_squareup_javapoet',
'version': 'version:2@1.13.0.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_squareup_javawriter': {
+ 'src/third_party/android_deps/cipd/libs/com_squareup_javawriter': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/com_squareup_javawriter',
'version': 'version:2@2.1.1.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_squareup_okio_okio_jvm': {
+ 'src/third_party/android_deps/cipd/libs/com_squareup_moshi_moshi': {
+ 'packages': [
+ {
+ 'package': 'chromium/third_party/android_deps/libs/com_squareup_moshi_moshi',
+ 'version': 'version:2@1.15.0.cr1',
+ },
+ ],
+ 'condition': 'checkout_android and non_git_source',
+ 'dep_type': 'cipd',
+ },
+
+ 'src/third_party/android_deps/cipd/libs/com_squareup_moshi_moshi_adapters': {
+ 'packages': [
+ {
+ 'package': 'chromium/third_party/android_deps/libs/com_squareup_moshi_moshi_adapters',
+ 'version': 'version:2@1.15.0.cr1',
+ },
+ ],
+ 'condition': 'checkout_android and non_git_source',
+ 'dep_type': 'cipd',
+ },
+
+ 'src/third_party/android_deps/cipd/libs/com_squareup_okio_okio_jvm': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/com_squareup_okio_okio_jvm',
- 'version': 'version:2@3.3.0.cr1',
+ 'version': 'version:2@3.9.1.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/com_squareup_wire_wire_runtime_jvm': {
+ 'src/third_party/android_deps/cipd/libs/com_squareup_wire_wire_runtime_jvm': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/com_squareup_wire_wire_runtime_jvm',
- 'version': 'version:2@4.7.0.cr1',
+ 'version': 'version:2@5.0.0.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/io_github_java_diff_utils_java_diff_utils': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/io_github_java_diff_utils_java_diff_utils',
- 'version': 'version:2@4.0.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/io_grpc_grpc_api': {
+ 'src/third_party/android_deps/cipd/libs/io_grpc_grpc_api': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/io_grpc_grpc_api',
'version': 'version:2@1.49.0.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/io_grpc_grpc_binder': {
+ 'src/third_party/android_deps/cipd/libs/io_grpc_grpc_binder': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/io_grpc_grpc_binder',
'version': 'version:2@1.49.0.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/io_grpc_grpc_context': {
+ 'src/third_party/android_deps/cipd/libs/io_grpc_grpc_context': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/io_grpc_grpc_context',
'version': 'version:2@1.49.0.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/io_grpc_grpc_core': {
+ 'src/third_party/android_deps/cipd/libs/io_grpc_grpc_core': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/io_grpc_grpc_core',
'version': 'version:2@1.49.0.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/io_grpc_grpc_protobuf_lite': {
+ 'src/third_party/android_deps/cipd/libs/io_grpc_grpc_protobuf_lite': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/io_grpc_grpc_protobuf_lite',
'version': 'version:2@1.49.0.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/io_grpc_grpc_stub': {
+ 'src/third_party/android_deps/cipd/libs/io_grpc_grpc_stub': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/io_grpc_grpc_stub',
'version': 'version:2@1.49.0.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/io_perfmark_perfmark_api': {
+ 'src/third_party/android_deps/cipd/libs/io_perfmark_perfmark_api': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/io_perfmark_perfmark_api',
'version': 'version:2@0.25.0.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/javax_annotation_javax_annotation_api': {
+ 'src/third_party/android_deps/cipd/libs/jakarta_inject_jakarta_inject_api': {
+ 'packages': [
+ {
+ 'package': 'chromium/third_party/android_deps/libs/jakarta_inject_jakarta_inject_api',
+ 'version': 'version:2@2.0.1.cr1',
+ },
+ ],
+ 'condition': 'checkout_android and non_git_source',
+ 'dep_type': 'cipd',
+ },
+
+ 'src/third_party/android_deps/cipd/libs/javax_annotation_javax_annotation_api': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/javax_annotation_javax_annotation_api',
'version': 'version:2@1.3.2.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/javax_annotation_jsr250_api': {
+ 'src/third_party/android_deps/cipd/libs/javax_annotation_jsr250_api': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/javax_annotation_jsr250_api',
'version': 'version:2@1.0.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/javax_inject_javax_inject': {
+ 'src/third_party/android_deps/cipd/libs/javax_inject_javax_inject': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/javax_inject_javax_inject',
'version': 'version:2@1.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/net_bytebuddy_byte_buddy': {
+ 'src/third_party/android_deps/cipd/libs/net_bytebuddy_byte_buddy': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/net_bytebuddy_byte_buddy',
- 'version': 'version:2@1.14.5.cr1',
+ 'version': 'version:2@1.14.12.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/net_bytebuddy_byte_buddy_agent': {
+ 'src/third_party/android_deps/cipd/libs/net_bytebuddy_byte_buddy_agent': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/net_bytebuddy_byte_buddy_agent',
- 'version': 'version:2@1.14.5.cr1',
+ 'version': 'version:2@1.14.12.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/net_ltgt_gradle_incap_incap': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/net_ltgt_gradle_incap_incap',
- 'version': 'version:2@0.2.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/org_bouncycastle_bcprov_jdk18on': {
+ 'src/third_party/android_deps/cipd/libs/org_bouncycastle_bcprov_jdk18on': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/org_bouncycastle_bcprov_jdk18on',
- 'version': 'version:2@1.72.cr1',
+ 'version': 'version:2@1.78.1.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/org_ccil_cowan_tagsoup_tagsoup': {
+ 'src/third_party/android_deps/cipd/libs/org_ccil_cowan_tagsoup_tagsoup': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/org_ccil_cowan_tagsoup_tagsoup',
'version': 'version:2@1.2.1.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/org_checkerframework_checker_compat_qual': {
+ 'src/third_party/android_deps/cipd/libs/org_checkerframework_checker_compat_qual': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/org_checkerframework_checker_compat_qual',
'version': 'version:2@2.5.5.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/org_checkerframework_checker_qual': {
+ 'src/third_party/android_deps/cipd/libs/org_checkerframework_checker_qual': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/org_checkerframework_checker_qual',
- 'version': 'version:2@3.25.0.cr1',
+ 'version': 'version:2@3.43.0.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/org_checkerframework_checker_util': {
+ 'src/third_party/android_deps/cipd/libs/org_checkerframework_checker_util': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/org_checkerframework_checker_util',
'version': 'version:2@3.25.0.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/org_checkerframework_dataflow_errorprone': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/org_checkerframework_dataflow_errorprone',
- 'version': 'version:2@3.15.0.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/org_codehaus_mojo_animal_sniffer_annotations': {
+ 'src/third_party/android_deps/cipd/libs/org_codehaus_mojo_animal_sniffer_annotations': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/org_codehaus_mojo_animal_sniffer_annotations',
'version': 'version:2@1.21.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/org_conscrypt_conscrypt_openjdk_uber': {
+ 'src/third_party/android_deps/cipd/libs/org_conscrypt_conscrypt_openjdk_uber': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/org_conscrypt_conscrypt_openjdk_uber',
'version': 'version:2@2.5.2.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/org_eclipse_jgit_org_eclipse_jgit': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/org_eclipse_jgit_org_eclipse_jgit',
- 'version': 'version:2@4.4.1.201607150455-r.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/org_hamcrest_hamcrest': {
+ 'src/third_party/android_deps/cipd/libs/org_hamcrest_hamcrest': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/org_hamcrest_hamcrest',
'version': 'version:2@2.2.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/org_jetbrains_kotlin_kotlin_stdlib_jdk7': {
+ 'src/third_party/android_deps/cipd/libs/org_jetbrains_kotlin_kotlin_android_extensions_runtime': {
'packages': [
{
- 'package': 'chromium/third_party/android_deps/libs/org_jetbrains_kotlin_kotlin_stdlib_jdk7',
- 'version': 'version:2@1.8.20.cr1',
+ 'package': 'chromium/third_party/android_deps/libs/org_jetbrains_kotlin_kotlin_android_extensions_runtime',
+ 'version': 'version:2@1.9.22.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/org_jetbrains_kotlin_kotlin_stdlib_jdk8': {
+ 'src/third_party/android_deps/cipd/libs/org_jetbrains_kotlin_kotlin_parcelize_runtime': {
'packages': [
{
- 'package': 'chromium/third_party/android_deps/libs/org_jetbrains_kotlin_kotlin_stdlib_jdk8',
- 'version': 'version:2@1.8.20.cr1',
+ 'package': 'chromium/third_party/android_deps/libs/org_jetbrains_kotlin_kotlin_parcelize_runtime',
+ 'version': 'version:2@1.9.22.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/org_jetbrains_kotlinx_kotlinx_coroutines_android': {
+ 'src/third_party/android_deps/cipd/libs/org_jetbrains_kotlinx_atomicfu_jvm': {
+ 'packages': [
+ {
+ 'package': 'chromium/third_party/android_deps/libs/org_jetbrains_kotlinx_atomicfu_jvm',
+ 'version': 'version:2@0.23.2.cr1',
+ },
+ ],
+ 'condition': 'checkout_android and non_git_source',
+ 'dep_type': 'cipd',
+ },
+
+ 'src/third_party/android_deps/cipd/libs/org_jetbrains_kotlinx_kotlinx_coroutines_android': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/org_jetbrains_kotlinx_kotlinx_coroutines_android',
- 'version': 'version:2@1.6.4.cr1',
+ 'version': 'version:2@1.8.1.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/org_jetbrains_kotlinx_kotlinx_coroutines_core_jvm': {
+ 'src/third_party/android_deps/cipd/libs/org_jetbrains_kotlinx_kotlinx_coroutines_core_jvm': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/org_jetbrains_kotlinx_kotlinx_coroutines_core_jvm',
- 'version': 'version:2@1.6.4.cr1',
+ 'version': 'version:2@1.8.1.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/org_jetbrains_kotlinx_kotlinx_coroutines_guava': {
+ 'src/third_party/android_deps/cipd/libs/org_jetbrains_kotlinx_kotlinx_coroutines_guava': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/org_jetbrains_kotlinx_kotlinx_coroutines_guava',
- 'version': 'version:2@1.6.4.cr1',
+ 'version': 'version:2@1.8.1.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/org_jetbrains_kotlinx_kotlinx_metadata_jvm': {
+ 'src/third_party/android_deps/cipd/libs/org_jetbrains_kotlinx_kotlinx_coroutines_test_jvm': {
'packages': [
{
- 'package': 'chromium/third_party/android_deps/libs/org_jetbrains_kotlinx_kotlinx_metadata_jvm',
- 'version': 'version:2@0.1.0.cr1',
+ 'package': 'chromium/third_party/android_deps/libs/org_jetbrains_kotlinx_kotlinx_coroutines_test_jvm',
+ 'version': 'version:2@1.7.3.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/org_jsoup_jsoup': {
+ 'src/third_party/android_deps/cipd/libs/org_jetbrains_kotlinx_kotlinx_serialization_core_jvm': {
+ 'packages': [
+ {
+ 'package': 'chromium/third_party/android_deps/libs/org_jetbrains_kotlinx_kotlinx_serialization_core_jvm',
+ 'version': 'version:2@1.7.2.cr1',
+ },
+ ],
+ 'condition': 'checkout_android and non_git_source',
+ 'dep_type': 'cipd',
+ },
+
+ 'src/third_party/android_deps/cipd/libs/org_jsoup_jsoup': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/org_jsoup_jsoup',
'version': 'version:2@1.15.1.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/org_mockito_mockito_android': {
+ 'src/third_party/android_deps/cipd/libs/org_jspecify_jspecify': {
+ 'packages': [
+ {
+ 'package': 'chromium/third_party/android_deps/libs/org_jspecify_jspecify',
+ 'version': 'version:2@1.0.0.cr1',
+ },
+ ],
+ 'condition': 'checkout_android and non_git_source',
+ 'dep_type': 'cipd',
+ },
+
+ 'src/third_party/android_deps/cipd/libs/org_mockito_mockito_android': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/org_mockito_mockito_android',
- 'version': 'version:2@5.4.0.cr1',
+ 'version': 'version:2@5.11.0.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/org_mockito_mockito_core': {
+ 'src/third_party/android_deps/cipd/libs/org_mockito_mockito_core': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/org_mockito_mockito_core',
- 'version': 'version:2@5.4.0.cr1',
+ 'version': 'version:2@5.11.0.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/org_mockito_mockito_subclass': {
+ 'src/third_party/android_deps/cipd/libs/org_mockito_mockito_subclass': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/org_mockito_mockito_subclass',
- 'version': 'version:2@5.4.0.cr1',
+ 'version': 'version:2@5.11.0.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/org_objenesis_objenesis': {
+ 'src/third_party/android_deps/cipd/libs/org_objenesis_objenesis': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/org_objenesis_objenesis',
'version': 'version:2@3.3.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/org_ow2_asm_asm': {
+ 'src/third_party/android_deps/cipd/libs/org_ow2_asm_asm': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/org_ow2_asm_asm',
- 'version': 'version:2@9.5.cr1',
+ 'version': 'version:2@9.7.1.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/org_ow2_asm_asm_analysis': {
+ 'src/third_party/android_deps/cipd/libs/org_ow2_asm_asm_analysis': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/org_ow2_asm_asm_analysis',
- 'version': 'version:2@9.5.cr1',
+ 'version': 'version:2@9.7.1.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/org_ow2_asm_asm_commons': {
+ 'src/third_party/android_deps/cipd/libs/org_ow2_asm_asm_commons': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/org_ow2_asm_asm_commons',
- 'version': 'version:2@9.5.cr1',
+ 'version': 'version:2@9.7.1.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/org_ow2_asm_asm_tree': {
+ 'src/third_party/android_deps/cipd/libs/org_ow2_asm_asm_tree': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/org_ow2_asm_asm_tree',
- 'version': 'version:2@9.5.cr1',
+ 'version': 'version:2@9.7.1.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/org_ow2_asm_asm_util': {
+ 'src/third_party/android_deps/cipd/libs/org_ow2_asm_asm_util': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/org_ow2_asm_asm_util',
- 'version': 'version:2@9.5.cr1',
+ 'version': 'version:2@9.7.1.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/org_pcollections_pcollections': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/org_pcollections_pcollections',
- 'version': 'version:2@3.1.4.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/org_robolectric_annotations': {
+ 'src/third_party/android_deps/cipd/libs/org_robolectric_annotations': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/org_robolectric_annotations',
- 'version': 'version:2@4.10.3.cr1',
+ 'version': 'version:2@4.14.1.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/org_robolectric_junit': {
+ 'src/third_party/android_deps/cipd/libs/org_robolectric_junit': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/org_robolectric_junit',
- 'version': 'version:2@4.10.3.cr1',
+ 'version': 'version:2@4.14.1.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/org_robolectric_nativeruntime': {
+ 'src/third_party/android_deps/cipd/libs/org_robolectric_nativeruntime': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/org_robolectric_nativeruntime',
- 'version': 'version:2@4.10.3.cr1',
+ 'version': 'version:2@4.14.1.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/org_robolectric_nativeruntime_dist_compat': {
+ 'src/third_party/android_deps/cipd/libs/org_robolectric_nativeruntime_dist_compat': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/org_robolectric_nativeruntime_dist_compat',
- 'version': 'version:2@1.0.1.cr1',
+ 'version': 'version:2@1.0.16.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/org_robolectric_pluginapi': {
+ 'src/third_party/android_deps/cipd/libs/org_robolectric_pluginapi': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/org_robolectric_pluginapi',
- 'version': 'version:2@4.10.3.cr1',
+ 'version': 'version:2@4.14.1.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/org_robolectric_plugins_maven_dependency_resolver': {
+ 'src/third_party/android_deps/cipd/libs/org_robolectric_plugins_maven_dependency_resolver': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/org_robolectric_plugins_maven_dependency_resolver',
- 'version': 'version:2@4.10.3.cr1',
+ 'version': 'version:2@4.14.1.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/org_robolectric_resources': {
+ 'src/third_party/android_deps/cipd/libs/org_robolectric_resources': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/org_robolectric_resources',
- 'version': 'version:2@4.10.3.cr1',
+ 'version': 'version:2@4.14.1.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/org_robolectric_robolectric': {
+ 'src/third_party/android_deps/cipd/libs/org_robolectric_robolectric': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/org_robolectric_robolectric',
- 'version': 'version:2@4.10.3.cr1',
+ 'version': 'version:2@4.14.1.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/org_robolectric_sandbox': {
+ 'src/third_party/android_deps/cipd/libs/org_robolectric_sandbox': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/org_robolectric_sandbox',
- 'version': 'version:2@4.10.3.cr1',
+ 'version': 'version:2@4.14.1.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/org_robolectric_shadowapi': {
+ 'src/third_party/android_deps/cipd/libs/org_robolectric_shadowapi': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/org_robolectric_shadowapi',
- 'version': 'version:2@4.10.3.cr1',
+ 'version': 'version:2@4.14.1.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/org_robolectric_shadows_framework': {
+ 'src/third_party/android_deps/cipd/libs/org_robolectric_shadows_framework': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/org_robolectric_shadows_framework',
- 'version': 'version:2@4.10.3.cr1',
+ 'version': 'version:2@4.14.1.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/org_robolectric_shadows_playservices': {
- 'packages': [
- {
- 'package': 'chromium/third_party/android_deps/libs/org_robolectric_shadows_playservices',
- 'version': 'version:2@4.10.3.cr1',
- },
- ],
- 'condition': 'checkout_android',
- 'dep_type': 'cipd',
- },
-
- 'src/third_party/android_deps/libs/org_robolectric_utils': {
+ 'src/third_party/android_deps/cipd/libs/org_robolectric_utils': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/org_robolectric_utils',
- 'version': 'version:2@4.10.3.cr1',
+ 'version': 'version:2@4.14.1.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
- 'src/third_party/android_deps/libs/org_robolectric_utils_reflector': {
+ 'src/third_party/android_deps/cipd/libs/org_robolectric_utils_reflector': {
'packages': [
{
'package': 'chromium/third_party/android_deps/libs/org_robolectric_utils_reflector',
- 'version': 'version:2@4.10.3.cr1',
+ 'version': 'version:2@4.14.1.cr1',
},
],
- 'condition': 'checkout_android',
+ 'condition': 'checkout_android and non_git_source',
+ 'dep_type': 'cipd',
+ },
+
+ 'src/third_party/android_deps/cipd/libs/org_yaml_snakeyaml': {
+ 'packages': [
+ {
+ 'package': 'chromium/third_party/android_deps/libs/org_yaml_snakeyaml',
+ 'version': 'version:2@2.3.cr1',
+ },
+ ],
+ 'condition': 'checkout_android and non_git_source',
'dep_type': 'cipd',
},
# === ANDROID_DEPS Generated Code End ===
}
-pre_deps_hooks = [
- {
- # Remove any symlinks from before 177567c518b121731e507e9b9c4049c4dc96e4c8.
- # TODO(kjellander): Remove this in March 2017.
- 'name': 'cleanup_links',
- 'pattern': '.',
- 'action': ['python3', 'src/cleanup_links.py'],
- },
-]
-
hooks = [
{
# This clobbers when necessary (based on get_landmines.py). It should be
@@ -2391,54 +1804,6 @@
'condition': 'checkout_mac',
},
{
- 'name': 'msan_chained_origins_focal',
- 'pattern': '.',
- 'condition': 'checkout_instrumented_libraries',
- 'action': [ 'python3',
- 'src/third_party/depot_tools/download_from_google_storage.py',
- '--no_resume',
- '--no_auth',
- '--bucket', 'chromium-instrumented-libraries',
- '-s', 'src/third_party/instrumented_libraries/binaries/msan-chained-origins-focal.tgz.sha1',
- ],
- },
- {
- 'name': 'msan_no_origins_focal',
- 'pattern': '.',
- 'condition': 'checkout_instrumented_libraries',
- 'action': [ 'python3',
- 'src/third_party/depot_tools/download_from_google_storage.py',
- '--no_resume',
- '--no_auth',
- '--bucket', 'chromium-instrumented-libraries',
- '-s', 'src/third_party/instrumented_libraries/binaries/msan-no-origins-focal.tgz.sha1',
- ],
- },
- {
- 'name': 'msan_chained_origins_focal',
- 'pattern': '.',
- 'condition': 'checkout_instrumented_libraries',
- 'action': [ 'python3',
- 'src/third_party/depot_tools/download_from_google_storage.py',
- '--no_resume',
- '--no_auth',
- '--bucket', 'chromium-instrumented-libraries',
- '-s', 'src/third_party/instrumented_libraries/binaries/msan-chained-origins-focal.tgz.sha1',
- ],
- },
- {
- 'name': 'msan_no_origins_focal',
- 'pattern': '.',
- 'condition': 'checkout_instrumented_libraries',
- 'action': [ 'python3',
- 'src/third_party/depot_tools/download_from_google_storage.py',
- '--no_resume',
- '--no_auth',
- '--bucket', 'chromium-instrumented-libraries',
- '-s', 'src/third_party/instrumented_libraries/binaries/msan-no-origins-focal.tgz.sha1',
- ],
- },
- {
'name': 'Download Fuchsia SDK from GCS',
'pattern': '.',
'condition': 'checkout_fuchsia',
@@ -2473,60 +1838,6 @@
'action': ['python3', 'src/build/util/lastchange.py',
'-o', 'src/build/util/LASTCHANGE'],
},
- # Pull clang-format binaries using checked-in hashes.
- {
- 'name': 'clang_format_win',
- 'pattern': '.',
- 'condition': 'host_os == "win"',
- 'action': [ 'python3',
- 'src/third_party/depot_tools/download_from_google_storage.py',
- '--no_resume',
- '--platform=win32',
- '--no_auth',
- '--bucket', 'chromium-clang-format',
- '-s', 'src/buildtools/win/clang-format.exe.sha1',
- ],
- },
- {
- 'name': 'clang_format_mac_x64',
- 'pattern': '.',
- 'condition': 'host_os == "mac" and host_cpu == "x64"',
- 'action': [ 'python3',
- 'src/third_party/depot_tools/download_from_google_storage.py',
- '--no_resume',
- '--platform=darwin',
- '--no_auth',
- '--bucket', 'chromium-clang-format',
- '-s', 'src/buildtools/mac/clang-format.x64.sha1',
- '-o', 'src/buildtools/mac/clang-format',
- ],
- },
- {
- 'name': 'clang_format_mac_arm64',
- 'pattern': '.',
- 'condition': 'host_os == "mac" and host_cpu == "arm64"',
- 'action': [ 'python3',
- 'src/third_party/depot_tools/download_from_google_storage.py',
- '--no_resume',
- '--no_auth',
- '--bucket', 'chromium-clang-format',
- '-s', 'src/buildtools/mac/clang-format.arm64.sha1',
- '-o', 'src/buildtools/mac/clang-format',
- ],
- },
- {
- 'name': 'clang_format_linux',
- 'pattern': '.',
- 'condition': 'host_os == "linux"',
- 'action': [ 'python3',
- 'src/third_party/depot_tools/download_from_google_storage.py',
- '--no_resume',
- '--platform=linux*',
- '--no_auth',
- '--bucket', 'chromium-clang-format',
- '-s', 'src/buildtools/linux64/clang-format.sha1',
- ],
- },
# Pull luci-go binaries (isolate, swarming) using checked-in hashes.
{
'name': 'luci-go_win',
@@ -2571,6 +1882,25 @@
'src/testing/location_tags.json',
],
},
+ # Download remote exec cfg files
+ {
+ 'name': 'fetch_reclient_cfgs',
+ 'pattern': '.',
+ 'condition': 'download_remoteexec_cfg',
+ 'action': ['python3',
+ 'src/buildtools/reclient_cfgs/fetch_reclient_cfgs.py',
+ '--rbe_instance',
+ Var('rbe_instance'),
+ '--reproxy_cfg_template',
+ 'reproxy.cfg.template',
+ '--rewrapper_cfg_project',
+ Var('rewrapper_cfg_project'),
+ '--quiet',
+ ],
+ },
]
-recursedeps = []
+recursedeps = [
+ 'src/buildtools',
+ 'src/third_party/instrumented_libs',
+]
diff --git a/METADATA b/METADATA
index 19d0436..752e6f5 100644
--- a/METADATA
+++ b/METADATA
@@ -1,19 +1,19 @@
# This project was upgraded with external_updater.
-# Usage: tools/external_updater/updater.sh update libyuv
-# For more info, check https://cs.android.com/android/platform/superproject/+/main:tools/external_updater/README.md
+# Usage: tools/external_updater/updater.sh update external/<absolute path to project>
+# For more info, check https://cs.android.com/android/platform/superproject/main/+/main:tools/external_updater/README.md
name: "libyuv"
description: "libyuv is an open source project that includes YUV scaling and conversion functionality."
third_party {
license_type: NOTICE
last_upgrade_date {
- year: 2024
- month: 1
- day: 11
+ year: 2025
+ month: 3
+ day: 25
}
identifier {
type: "Git"
value: "https://chromium.googlesource.com/libyuv/libyuv/"
- version: "af6ac8265bbd07bcf977526458b60305c4304288"
+ version: "5f284054cb806f82e737f738003d764277970d45"
}
}
diff --git a/PRESUBMIT.py b/PRESUBMIT.py
index d3901ca..37df28a 100644
--- a/PRESUBMIT.py
+++ b/PRESUBMIT.py
@@ -9,44 +9,57 @@
# Runs PRESUBMIT.py in py3 mode by git cl presubmit.
USE_PYTHON3 = True
+
def _CommonChecks(input_api, output_api):
- """Checks common to both upload and commit."""
- results = []
- results.extend(input_api.canned_checks.RunPylint(input_api, output_api,
- files_to_skip=(r'^base[\\\/].*\.py$',
- r'^build[\\\/].*\.py$',
- r'^buildtools[\\\/].*\.py$',
- r'^ios[\\\/].*\.py$',
- r'^out.*[\\\/].*\.py$',
- r'^testing[\\\/].*\.py$',
- r'^third_party[\\\/].*\.py$',
- r'^tools[\\\/].*\.py$',
- # TODO(kjellander): should arguably be checked.
- r'^tools_libyuv[\\\/]valgrind[\\\/].*\.py$',
- r'^xcodebuild.*[\\\/].*\.py$',),
- disabled_warnings=['F0401', # Failed to import x
- 'E0611', # No package y in x
- 'W0232', # Class has no __init__ method
- ],
- pylintrc='pylintrc',
- version='2.7'))
- return results
+ """Checks common to both upload and commit."""
+ results = []
+ results.extend(
+ input_api.canned_checks.RunPylint(
+ input_api,
+ output_api,
+ files_to_skip=(
+ r'^base[\\\/].*\.py$',
+ r'^build[\\\/].*\.py$',
+ r'^buildtools[\\\/].*\.py$',
+ r'^ios[\\\/].*\.py$',
+ r'^out.*[\\\/].*\.py$',
+ r'^testing[\\\/].*\.py$',
+ r'^third_party[\\\/].*\.py$',
+ r'^tools[\\\/].*\.py$',
+ # TODO(kjellander): should arguably be checked.
+ r'^tools_libyuv[\\\/]valgrind[\\\/].*\.py$',
+ r'^xcodebuild.*[\\\/].*\.py$',
+ ),
+ disabled_warnings=[
+ 'F0401', # Failed to import x
+ 'E0611', # No package y in x
+ 'W0232', # Class has no __init__ method
+ ],
+ pylintrc='pylintrc',
+ version='2.7',
+ )
+ )
+ return results
def CheckChangeOnUpload(input_api, output_api):
- results = []
- results.extend(_CommonChecks(input_api, output_api))
- results.extend(
- input_api.canned_checks.CheckGNFormatted(input_api, output_api))
- return results
+ results = []
+ results.extend(_CommonChecks(input_api, output_api))
+ results.extend(
+ input_api.canned_checks.CheckGNFormatted(input_api, output_api)
+ )
+ return results
def CheckChangeOnCommit(input_api, output_api):
- results = []
- results.extend(_CommonChecks(input_api, output_api))
- results.extend(input_api.canned_checks.CheckOwners(input_api, output_api))
- results.extend(input_api.canned_checks.CheckChangeWasUploaded(
- input_api, output_api))
- results.extend(input_api.canned_checks.CheckChangeHasDescription(
- input_api, output_api))
- return results
+ results = []
+ results.extend(_CommonChecks(input_api, output_api))
+ results.extend(input_api.canned_checks.CheckOwners(input_api, output_api))
+ results.extend(
+ input_api.canned_checks.CheckChangeWasUploaded(input_api, output_api)
+ )
+ results.extend(
+ input_api.canned_checks.CheckChangeHasDescription(input_api,
+ output_api)
+ )
+ return results
diff --git a/README.chromium b/README.chromium
index 1389f28..46ed8e3 100644
--- a/README.chromium
+++ b/README.chromium
@@ -1,9 +1,10 @@
Name: libyuv
URL: https://chromium.googlesource.com/libyuv/libyuv/
-Version: 1883
-License: BSD
+Version: 1907
+License: BSD-3-Clause
License File: LICENSE
Shipped: yes
+Security Critical: yes
Description:
libyuv is an open source project that includes YUV conversion and scaling functionality.
diff --git a/README.md b/README.md
index 95eeb04..4baa69c 100644
--- a/README.md
+++ b/README.md
@@ -5,7 +5,7 @@
* Convert to RGB formats for rendering/effects.
* Rotate by 90/180/270 degrees to adjust for mobile devices in portrait mode.
* Optimized for SSSE3/AVX2 on x86/x64.
-* Optimized for Neon on Arm.
+* Optimized for Neon/SVE2/SME on Arm.
* Optimized for MSA on Mips.
* Optimized for RVV on RISC-V.
diff --git a/build_overrides/build.gni b/build_overrides/build.gni
index d9d01d5..1613440 100644
--- a/build_overrides/build.gni
+++ b/build_overrides/build.gni
@@ -35,12 +35,13 @@
# Use bundled hermetic Xcode installation maintained by Chromium,
# except for local iOS builds where it is unsupported.
-if (host_os == "mac") {
+# Allow for mac cross compile on linux machines.
+if (host_os == "mac" || host_os == "linux") {
_result = exec_script("//build/mac/should_use_hermetic_xcode.py",
[ target_os ],
"value")
assert(_result != 2,
- "Do not allow building targets with the default" +
+ "Do not allow building targets with the default " +
"hermetic toolchain if the minimum OS version is not met.")
use_system_xcode = _result == 0
}
diff --git a/cleanup_links.py b/cleanup_links.py
deleted file mode 100755
index 7d1eba9..0000000
--- a/cleanup_links.py
+++ /dev/null
@@ -1,108 +0,0 @@
-#!/usr/bin/env vpython3
-
-# Copyright 2017 The LibYuv Project Authors. All rights reserved.
-#
-# Use of this source code is governed by a BSD-style license
-# that can be found in the LICENSE file in the root of the source
-# tree. An additional intellectual property rights grant can be found
-# in the file PATENTS. All contributing project authors may
-# be found in the AUTHORS file in the root of the source tree.
-
-# This is a copy of the file from WebRTC in:
-# https://chromium.googlesource.com/external/webrtc/+/master/cleanup_links.py
-
-"""Script to cleanup symlinks created from setup_links.py.
-
-Before 177567c518b121731e507e9b9c4049c4dc96e4c8 (#15754) we had a Chromium
-checkout which we created symlinks into. In order to do clean syncs after
-landing that change, this script cleans up any old symlinks, avoiding annoying
-manual cleanup needed in order to complete gclient sync.
-"""
-
-import argparse
-import logging
-import os
-import shelve
-import subprocess
-import sys
-
-
-ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
-LINKS_DB = 'links'
-
-# Version management to make future upgrades/downgrades easier to support.
-SCHEMA_VERSION = 1
-
-class WebRTCLinkSetup():
- def __init__(self, links_db, dry_run=False):
- self._dry_run = dry_run
- self._links_db = links_db
-
- def CleanupLinks(self):
- logging.debug('CleanupLinks')
- for source, link_path in self._links_db.tems():
- if source == 'SCHEMA_VERSION':
- continue
- if os.path.islink(link_path) or sys.platform.startswith('win'):
- # os.path.islink() always returns false on Windows
- # See http://bugs.python.org/issue13143.
- logging.debug('Removing link to %s at %s', source, link_path)
- if not self._dry_run:
- if os.path.exists(link_path):
- if sys.platform.startswith('win') and os.path.isdir(link_path):
- subprocess.check_call(['rmdir', '/q', '/s', link_path],
- shell=True)
- else:
- os.remove(link_path)
- del self._links_db[source]
-
-
-def _initialize_database(filename):
- links_database = shelve.open(filename)
- # Wipe the database if this version of the script ends up looking at a
- # newer (future) version of the links db, just to be sure.
- version = links_database.get('SCHEMA_VERSION')
- if version and version != SCHEMA_VERSION:
- logging.info('Found database with schema version %s while this script only '
- 'supports %s. Wiping previous database contents.', version,
- SCHEMA_VERSION)
- links_database.clear()
- links_database['SCHEMA_VERSION'] = SCHEMA_VERSION
- return links_database
-
-
-def main():
- p = argparse.ArgumentParser()
- p.add_argument('-d', '--dry-run', action='store_true', default=False,
- help='Print what would be done, but don\'t perform any '
- 'operations. This will automatically set logging to '
- 'verbose.')
- p.add_argument('-v', '--verbose', action='store_const',
- const=logging.DEBUG, default=logging.INFO,
- help='Print verbose output for debugging.')
- options = p.parse_args()
-
- if options.dry_run:
- options.verbose = logging.DEBUG
- logging.basicConfig(format='%(message)s', level=options.verbose)
-
- # Work from the root directory of the checkout.
- script_dir = os.path.dirname(os.path.abspath(__file__))
- os.chdir(script_dir)
-
- # The database file gets .db appended on some platforms.
- db_filenames = [LINKS_DB, LINKS_DB + '.db']
- if any(os.path.isfile(f) for f in db_filenames):
- links_database = _initialize_database(LINKS_DB)
- try:
- symlink_creator = WebRTCLinkSetup(links_database, options.dry_run)
- symlink_creator.CleanupLinks()
- finally:
- for f in db_filenames:
- if os.path.isfile(f):
- os.remove(f)
- return 0
-
-
-if __name__ == '__main__':
- sys.exit(main())
diff --git a/docs/environment_variables.md b/docs/environment_variables.md
index 4eb0965..173c207 100644
--- a/docs/environment_variables.md
+++ b/docs/environment_variables.md
@@ -7,9 +7,11 @@
By default the cpu is detected and the most advanced form of SIMD is used. But you can disable instruction sets selectively, or completely, falling back on C code. Set the variable to 1 to disable the specified instruction set.
## All CPUs
+
LIBYUV_DISABLE_ASM
## Intel CPUs
+
LIBYUV_DISABLE_X86
LIBYUV_DISABLE_SSE2
LIBYUV_DISABLE_SSSE3
@@ -26,21 +28,32 @@
LIBYUV_DISABLE_AVX512VBMI
LIBYUV_DISABLE_AVX512VBMI2
LIBYUV_DISABLE_AVX512VBITALG
- LIBYUV_DISABLE_AVX512VPOPCNTDQ
- LIBYUV_DISABLE_GFNI
+ LIBYUV_DISABLE_AVX10
+ LIBYUV_DISABLE_AVX10_2
+ LIBYUV_DISABLE_AVXVNNI
+ LIBYUV_DISABLE_AVXVNNIINT8
+ LIBYUV_DISABLE_AMXINT8
-## ARM CPUs
+## Arm CPUs
LIBYUV_DISABLE_NEON
+ LIBYUV_DISABLE_NEON_DOTPROD
+ LIBYUV_DISABLE_NEON_I8MM
+ LIBYUV_DISABLE_SVE
+ LIBYUV_DISABLE_SVE2
+ LIBYUV_DISABLE_SME
## MIPS CPUs
+
LIBYUV_DISABLE_MSA
## LOONGARCH CPUs
+
LIBYUV_DISABLE_LSX
LIBYUV_DISABLE_LASX
## RISCV CPUs
+
LIBYUV_DISABLE_RVV
# Test Width/Height/Repeat
diff --git a/docs/feature_detection.md b/docs/feature_detection.md
new file mode 100644
index 0000000..d32e84b
--- /dev/null
+++ b/docs/feature_detection.md
@@ -0,0 +1,108 @@
+# Introduction
+
+Several routines in libyuv have multiple implementations specialized for a
+variety of CPU architecture extensions. Libyuv will automatically detect and
+use the latest architecture extension present on a machine for which a kernel
+implementation is available.
+
+# Feature detection on AArch64
+
+## Architecture extensions of interest
+
+The Arm 64-bit A-class architecture has a number of vector extensions which can
+be used to accelerate libyuv kernels.
+
+### Neon extensions
+
+Neon is available and mandatory in AArch64 from the base Armv8.0-A
+architecture. Neon can be used even if later extensions like the Scalable
+Vector Extension (SVE) are also present. The exception to this is if the CPU is
+currently operating in streaming mode as introduced by the Scalable Matrix
+Extension, described later.
+
+There are also a couple of architecture extensions present for Neon that we can
+take advantage of in libyuv:
+
+* The Neon DotProd extension is architecturally available from Armv8.1-A and
+ becomes mandatory from Armv8.4-A. This extension provides instructions to
+ perform a pairwise widening multiply of groups of four bytes from two source
+ vectors, taking the sum of the four widened multiply results within each
+ group to give a 32-bit result, accumulating into a destination vector.
+
+* The Neon I8MM extension extends the DotProd extension with support for
+ mixed-sign DotProds. The I8MM extension is architecturally available from
+ Armv8.1-A and becomes mandatory from Armv8.6-A. It does not strictly depend
+ on the DotProd extension being implemented, however at time of writing there
+ is no known micro-architecture implementation where I8MM is implemented
+ without the DotProd extension also being implemented.
+
+### The Scalable Vector Extension (SVE)
+
+The two Scalable Vector extensions (SVE and SVE2) provides equivalent
+functionality to most existing Neon instructions but with the ability to
+efficiently operate on vector registers with a run-time-determined vector
+length.
+
+The original version of SVE is architecturally available from Armv8.2-A and is
+primarily targeted at HPC applications. This focus means it does not include
+most of the DSP-style operations that are necessary for most libyuv
+color-conversion kernels, though it can still be used for many scaling or
+rotation kernels.
+
+SVE does not strictly depend on either of the Neon DotProd or I8MM extensions
+being implemented. The only micro-architecture at time of writing where SVE is
+implemented without these two extensions both also being implemented is the
+Fujitsu A64FX, which is not a CPU of interest for libyuv.
+
+SVE2 extends the base SVE extension with the remaining instructions from Neon,
+porting these instructions to operate on scalable vectors. SVE2 is
+architecturally available from Armv9.0-A. If SVE2 is implemented then SVE must
+also be implemented. Since Armv9.0-A is based on Armv8.5-A this implies that
+the Neon DotProd extension is also implemented. Interestingly this means that
+the I8MM extension is not mandatory since it only becomes mandatory from
+Armv8.6-A or Armv9.1-A, however there is no micro-architecture at time of
+writing where SVE2 is implemented without all previously-mentioned features
+also being implemented.
+
+### The Scalable Matrix Extension (SME)
+
+The Scalable Matrix Extension (SME) is an optional feature introduced from
+Armv9.2-A. SME exists alongside SVE and introduces new execution modes for
+applications performing extended periods of data processing. In particular SME
+introduces a few new components of interest:
+
+* Access to a scalable two-dimensional ZA tile register and new instructions to
+ interact with rows and columns of the ZA tiles. This can be useful for data
+ transformations like transposes.
+
+* A streaming SVE (SSVE) mode, during which the SVE vector length matches the
+ ZA tile register width. In typical systems where the ZA tile register width
+ is longer than the core SVE vector length, SSVE processing allows for faster
+ data processing, even if the ZA tile register is unused. While the CPU is
+ executing in streaming mode, Neon instructions are unavailable.
+
+* When both SSVE and the ZA tile registers are enabled there are additional
+ outer-product instructions accumulating into a whole ZA tile, suitable for
+ accelerating matrix arithmetic. This is likely less useful in libyuv.
+
+## Linux and Android
+
+On AArch64 running under Linux and Android, features are detected by inspecting
+the CPU auxiliary vector via `getauxval(AT_HWCAP)` and `getauxval(AT_HWCAP2)`,
+inspecting the returned bitmask.
+
+## Windows
+
+On Windows we detect features using the `IsProcessorFeaturePresent` interface
+and passing an enum parameter for the feature we want to check. More
+information on this can be found here:
+
+ https://learn.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-isprocessorfeaturepresent#parameters
+
+## Apple Silicon
+
+On Apple Silicon we detect features using the `sysctlbyname` interface and
+passing a string representing the feature we want to detect. More information
+on this can be found here:
+
+ https://developer.apple.com/documentation/kernel/1387446-sysctlbyname/determining_instruction_set_characteristics
diff --git a/docs/getting_started.md b/docs/getting_started.md
index f2f71b8..a052727 100644
--- a/docs/getting_started.md
+++ b/docs/getting_started.md
@@ -59,13 +59,13 @@
### Windows
- call gn gen out\Release "--args=is_debug=false target_cpu=\"x64\""
- call gn gen out\Debug "--args=is_debug=true target_cpu=\"x64\""
+ gn gen out\Release "--args=is_debug=false target_cpu=\"x64\""
+ gn gen out\Debug "--args=is_debug=true target_cpu=\"x64\""
ninja -v -C out\Release
ninja -v -C out\Debug
- call gn gen out\Release "--args=is_debug=false target_cpu=\"x86\""
- call gn gen out\Debug "--args=is_debug=true target_cpu=\"x86\""
+ gn gen out\Release "--args=is_debug=false target_cpu=\"x86\""
+ gn gen out\Debug "--args=is_debug=true target_cpu=\"x86\""
ninja -v -C out\Release
ninja -v -C out\Debug
diff --git a/download_vs_toolchain.py b/download_vs_toolchain.py
index 6bc086d..9fc89af 100644
--- a/download_vs_toolchain.py
+++ b/download_vs_toolchain.py
@@ -26,4 +26,4 @@
if __name__ == '__main__':
- sys.exit(vs_toolchain.main())
+ sys.exit(vs_toolchain.main())
diff --git a/fuzz/Android.bp b/fuzz/Android.bp
index a8d552b..1df5662 100644
--- a/fuzz/Android.bp
+++ b/fuzz/Android.bp
@@ -1,4 +1,3 @@
-
package {
// See: http://go/android-license-faq
// A large-scale-change added 'default_applicable_licenses' to import
diff --git a/include/libyuv/compare_row.h b/include/libyuv/compare_row.h
index 8293c91..ec0e331 100644
--- a/include/libyuv/compare_row.h
+++ b/include/libyuv/compare_row.h
@@ -12,45 +12,17 @@
#define INCLUDE_LIBYUV_COMPARE_ROW_H_
#include "libyuv/basic_types.h"
+#include "libyuv/cpu_support.h"
#ifdef __cplusplus
namespace libyuv {
extern "C" {
#endif
-#if defined(__pnacl__) || defined(__CLR_VER) || \
- (defined(__native_client__) && defined(__x86_64__)) || \
- (defined(__i386__) && !defined(__SSE__) && !defined(__clang__))
-#define LIBYUV_DISABLE_X86
-#endif
-#if defined(__native_client__)
-#define LIBYUV_DISABLE_NEON
-#endif
-// MemorySanitizer does not support assembly code yet. http://crbug.com/344505
-#if defined(__has_feature)
-#if __has_feature(memory_sanitizer) && !defined(LIBYUV_DISABLE_NEON)
-#define LIBYUV_DISABLE_NEON
-#endif
-#if __has_feature(memory_sanitizer) && !defined(LIBYUV_DISABLE_X86)
-#define LIBYUV_DISABLE_X86
-#endif
-#endif
-// Visual C 2012 required for AVX2.
-#if defined(_M_IX86) && !defined(__clang__) && defined(_MSC_VER) && \
- _MSC_VER >= 1700
-#define VISUALC_HAS_AVX2 1
-#endif // VisualStudio >= 2012
-
-// clang >= 3.4.0 required for AVX2.
-#if defined(__clang__) && (defined(__x86_64__) || defined(__i386__))
-#if (__clang_major__ > 3) || (__clang_major__ == 3 && (__clang_minor__ >= 4))
-#define CLANG_HAS_AVX2 1
-#endif // clang >= 3.4
-#endif // __clang__
-
// The following are available for Visual C and GCC:
-#if !defined(LIBYUV_DISABLE_X86) && \
- (defined(__x86_64__) || defined(__i386__) || defined(_M_IX86))
+#if !defined(LIBYUV_DISABLE_X86) && \
+ ((defined(__x86_64__) && !defined(LIBYUV_ENABLE_ROWWIN)) || \
+ defined(__i386__) || defined(_M_IX86))
#define HAS_HASHDJB2_SSE41
#define HAS_SUMSQUAREERROR_SSE2
#define HAS_HAMMINGDISTANCE_SSE42
@@ -65,21 +37,32 @@
#endif
// The following are available for GCC and clangcl:
-#if !defined(LIBYUV_DISABLE_X86) && (defined(__x86_64__) || defined(__i386__))
+#if !defined(LIBYUV_DISABLE_X86) && \
+ (defined(__x86_64__) || defined(__i386__)) && \
+ !defined(LIBYUV_ENABLE_ROWWIN)
#define HAS_HAMMINGDISTANCE_SSSE3
#endif
// The following are available for GCC and clangcl:
#if !defined(LIBYUV_DISABLE_X86) && defined(CLANG_HAS_AVX2) && \
- (defined(__x86_64__) || defined(__i386__))
+ (defined(__x86_64__) || defined(__i386__)) && \
+ !defined(LIBYUV_ENABLE_ROWWIN)
#define HAS_HAMMINGDISTANCE_AVX2
#endif
// The following are available for Neon:
#if !defined(LIBYUV_DISABLE_NEON) && \
(defined(__ARM_NEON__) || defined(LIBYUV_NEON) || defined(__aarch64__))
-#define HAS_SUMSQUAREERROR_NEON
#define HAS_HAMMINGDISTANCE_NEON
+#define HAS_SUMSQUAREERROR_NEON
+#endif
+
+// The following are available for AArch64 Neon:
+#if !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__)
+#define HAS_HASHDJB2_NEON
+
+#define HAS_HAMMINGDISTANCE_NEON_DOTPROD
+#define HAS_SUMSQUAREERROR_NEON_DOTPROD
#endif
#if !defined(LIBYUV_DISABLE_MSA) && defined(__mips_msa)
@@ -102,6 +85,9 @@
uint32_t HammingDistance_NEON(const uint8_t* src_a,
const uint8_t* src_b,
int count);
+uint32_t HammingDistance_NEON_DotProd(const uint8_t* src_a,
+ const uint8_t* src_b,
+ int count);
uint32_t HammingDistance_MSA(const uint8_t* src_a,
const uint8_t* src_b,
int count);
@@ -117,6 +103,9 @@
uint32_t SumSquareError_NEON(const uint8_t* src_a,
const uint8_t* src_b,
int count);
+uint32_t SumSquareError_NEON_DotProd(const uint8_t* src_a,
+ const uint8_t* src_b,
+ int count);
uint32_t SumSquareError_MSA(const uint8_t* src_a,
const uint8_t* src_b,
int count);
@@ -124,6 +113,7 @@
uint32_t HashDjb2_C(const uint8_t* src, int count, uint32_t seed);
uint32_t HashDjb2_SSE41(const uint8_t* src, int count, uint32_t seed);
uint32_t HashDjb2_AVX2(const uint8_t* src, int count, uint32_t seed);
+uint32_t HashDjb2_NEON(const uint8_t* src, int count, uint32_t seed);
#ifdef __cplusplus
} // extern "C"
diff --git a/include/libyuv/convert.h b/include/libyuv/convert.h
index 88619a4..750383a 100644
--- a/include/libyuv/convert.h
+++ b/include/libyuv/convert.h
@@ -418,6 +418,20 @@
int width,
int height);
+// Convert 10 bit P010 to 8 bit NV12.
+// dst_y can be NULL
+LIBYUV_API
+int P010ToNV12(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height);
+
#define I412ToI012 I410ToI010
#define H410ToH010 I410ToI010
#define H412ToH012 I410ToI010
@@ -511,6 +525,21 @@
int width,
int height);
+// Convert 10 bit YUV I010 to NV12
+LIBYUV_API
+int I010ToNV12(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height);
+
// Convert I210 to P210
LIBYUV_API
int I210ToP210(const uint16_t* src_y,
@@ -569,6 +598,23 @@
int width,
int height);
+// Convert J420 to I420.
+LIBYUV_API
+int J420ToI420(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height);
+
// Convert I400 (grey) to NV21.
LIBYUV_API
int I400ToNV21(const uint8_t* src_y,
@@ -878,6 +924,19 @@
int width,
int height);
+// RGB big endian (rgb in memory) to I444.
+LIBYUV_API
+int RAWToI444(const uint8_t* src_raw,
+ int src_stride_raw,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height);
+
// RGB big endian (rgb in memory) to J420.
LIBYUV_API
int RAWToJ420(const uint8_t* src_raw,
@@ -891,6 +950,19 @@
int width,
int height);
+// RGB big endian (rgb in memory) to J444.
+LIBYUV_API
+int RAWToJ444(const uint8_t* src_raw,
+ int src_stride_raw,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height);
+
// RGB16 (RGBP fourcc) little endian to I420.
LIBYUV_API
int RGB565ToI420(const uint8_t* src_rgb565,
diff --git a/include/libyuv/convert_argb.h b/include/libyuv/convert_argb.h
index 35eeac9..5b50567 100644
--- a/include/libyuv/convert_argb.h
+++ b/include/libyuv/convert_argb.h
@@ -1904,6 +1904,26 @@
int width,
int height);
+// Convert YUY2 to ARGB with matrix.
+LIBYUV_API
+int YUY2ToARGBMatrix(const uint8_t* src_yuy2,
+ int src_stride_yuy2,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height);
+
+// Convert UYVY to ARGB with matrix.
+LIBYUV_API
+int UYVYToARGBMatrix(const uint8_t* src_uyvy,
+ int src_stride_uyvy,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height);
+
// Convert Android420 to ARGB with matrix.
LIBYUV_API
int Android420ToARGBMatrix(const uint8_t* src_y,
diff --git a/include/libyuv/convert_from_argb.h b/include/libyuv/convert_from_argb.h
index ff2a581..54b7d99 100644
--- a/include/libyuv/convert_from_argb.h
+++ b/include/libyuv/convert_from_argb.h
@@ -229,6 +229,19 @@
int width,
int height);
+// Convert ARGB to J444.
+LIBYUV_API
+int ARGBToJ444(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_yj,
+ int dst_stride_yj,
+ uint8_t* dst_uj,
+ int dst_stride_uj,
+ uint8_t* dst_vj,
+ int dst_stride_vj,
+ int width,
+ int height);
+
// Convert ARGB to J400. (JPeg full range).
LIBYUV_API
int ARGBToJ400(const uint8_t* src_argb,
diff --git a/include/libyuv/cpu_id.h b/include/libyuv/cpu_id.h
index 5a81e7c..44ea8c0 100644
--- a/include/libyuv/cpu_id.h
+++ b/include/libyuv/cpu_id.h
@@ -21,45 +21,52 @@
// Internal flag to indicate cpuid requires initialization.
static const int kCpuInitialized = 0x1;
-// These flags are only valid on ARM processors.
+// These flags are only valid on Arm processors.
static const int kCpuHasARM = 0x2;
-static const int kCpuHasNEON = 0x4;
-// 0x8 reserved for future ARM flag.
-
-// These flags are only valid on x86 processors.
-static const int kCpuHasX86 = 0x10;
-static const int kCpuHasSSE2 = 0x20;
-static const int kCpuHasSSSE3 = 0x40;
-static const int kCpuHasSSE41 = 0x80;
-static const int kCpuHasSSE42 = 0x100;
-static const int kCpuHasAVX = 0x200;
-static const int kCpuHasAVX2 = 0x400;
-static const int kCpuHasERMS = 0x800;
-static const int kCpuHasFMA3 = 0x1000;
-static const int kCpuHasF16C = 0x2000;
-static const int kCpuHasAVX512BW = 0x4000;
-static const int kCpuHasAVX512VL = 0x8000;
-static const int kCpuHasAVX512VNNI = 0x10000;
-static const int kCpuHasAVX512VBMI = 0x20000;
-static const int kCpuHasAVX512VBMI2 = 0x40000;
-static const int kCpuHasAVX512VBITALG = 0x80000;
-static const int kCpuHasAVX10 = 0x100000;
-static const int kCpuHasAVXVNNI = 0x200000;
-static const int kCpuHasAVXVNNIINT8 = 0x400000;
-
-// These flags are only valid on MIPS processors.
-static const int kCpuHasMIPS = 0x800000;
-static const int kCpuHasMSA = 0x1000000;
-
-// These flags are only valid on LOONGARCH processors.
-static const int kCpuHasLOONGARCH = 0x2000000;
-static const int kCpuHasLSX = 0x4000000;
-static const int kCpuHasLASX = 0x8000000;
+static const int kCpuHasNEON = 0x100;
+static const int kCpuHasNeonDotProd = 0x200;
+static const int kCpuHasNeonI8MM = 0x400;
+static const int kCpuHasSVE = 0x800;
+static const int kCpuHasSVE2 = 0x1000;
+static const int kCpuHasSME = 0x2000;
// These flags are only valid on RISCV processors.
-static const int kCpuHasRISCV = 0x10000000;
-static const int kCpuHasRVV = 0x20000000;
-static const int kCpuHasRVVZVFH = 0x40000000;
+static const int kCpuHasRISCV = 0x4;
+static const int kCpuHasRVV = 0x100;
+static const int kCpuHasRVVZVFH = 0x200;
+
+// These flags are only valid on x86 processors.
+static const int kCpuHasX86 = 0x8;
+static const int kCpuHasSSE2 = 0x100;
+static const int kCpuHasSSSE3 = 0x200;
+static const int kCpuHasSSE41 = 0x400;
+static const int kCpuHasSSE42 = 0x800;
+static const int kCpuHasAVX = 0x1000;
+static const int kCpuHasAVX2 = 0x2000;
+static const int kCpuHasERMS = 0x4000;
+static const int kCpuHasFSMR = 0x8000;
+static const int kCpuHasFMA3 = 0x10000;
+static const int kCpuHasF16C = 0x20000;
+static const int kCpuHasAVX512BW = 0x40000;
+static const int kCpuHasAVX512VL = 0x80000;
+static const int kCpuHasAVX512VNNI = 0x100000;
+static const int kCpuHasAVX512VBMI = 0x200000;
+static const int kCpuHasAVX512VBMI2 = 0x400000;
+static const int kCpuHasAVX512VBITALG = 0x800000;
+static const int kCpuHasAVX10 = 0x1000000;
+static const int kCpuHasAVX10_2 = 0x2000000;
+static const int kCpuHasAVXVNNI = 0x4000000;
+static const int kCpuHasAVXVNNIINT8 = 0x8000000;
+static const int kCpuHasAMXINT8 = 0x10000000;
+
+// These flags are only valid on MIPS processors.
+static const int kCpuHasMIPS = 0x10;
+static const int kCpuHasMSA = 0x100;
+
+// These flags are only valid on LOONGARCH processors.
+static const int kCpuHasLOONGARCH = 0x20;
+static const int kCpuHasLSX = 0x100;
+static const int kCpuHasLASX = 0x200;
// Optional init function. TestCpuFlag does an auto-init.
// Returns cpu_info flags.
@@ -87,6 +94,15 @@
LIBYUV_API
int RiscvCpuCaps(const char* cpuinfo_name);
+#ifdef __linux__
+// On Linux, parse AArch64 features from getauxval(AT_HWCAP{,2}).
+LIBYUV_API
+int AArch64CpuCaps(unsigned long hwcap, unsigned long hwcap2);
+#else
+LIBYUV_API
+int AArch64CpuCaps();
+#endif
+
// For testing, allow CPU flags to be disabled.
// ie MaskCpuFlags(~kCpuHasSSSE3) to disable SSSE3.
// MaskCpuFlags(-1) to enable all cpu specific optimizations.
diff --git a/include/libyuv/cpu_support.h b/include/libyuv/cpu_support.h
new file mode 100644
index 0000000..9304ab2
--- /dev/null
+++ b/include/libyuv/cpu_support.h
@@ -0,0 +1,91 @@
+/*
+ * Copyright 2024 The LibYuv Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef INCLUDE_LIBYUV_CPU_SUPPORT_H_
+#define INCLUDE_LIBYUV_CPU_SUPPORT_H_
+
+#ifdef __cplusplus
+namespace libyuv {
+extern "C" {
+#endif
+
+#if defined(__pnacl__) || defined(__CLR_VER) || \
+ (defined(__native_client__) && defined(__x86_64__)) || \
+ (defined(__i386__) && !defined(__SSE__) && !defined(__clang__))
+#define LIBYUV_DISABLE_X86
+#endif
+
+#if defined(__native_client__)
+#define LIBYUV_DISABLE_NEON
+#endif
+
+// MemorySanitizer does not support assembly code yet. http://crbug.com/344505
+#if defined(__has_feature)
+#if __has_feature(memory_sanitizer) && !defined(LIBYUV_DISABLE_NEON)
+#define LIBYUV_DISABLE_NEON
+#endif
+#if __has_feature(memory_sanitizer) && !defined(LIBYUV_DISABLE_X86)
+#define LIBYUV_DISABLE_X86
+#endif
+#endif
+
+// clang >= 3.5.0 required for Arm64.
+#if defined(__clang__) && defined(__aarch64__) && !defined(LIBYUV_DISABLE_NEON)
+#if (__clang_major__ < 3) || (__clang_major__ == 3 && (__clang_minor__ < 5))
+#define LIBYUV_DISABLE_NEON
+#endif // clang >= 3.5
+#endif // __clang__
+
+// GCC >= 4.7.0 required for AVX2.
+#if defined(__GNUC__) && !defined(LIBYUV_ENABLE_ROWWIN) && \
+ (defined(__x86_64__) || defined(__i386__))
+#if (__GNUC__ > 4) || (__GNUC__ == 4 && (__GNUC_MINOR__ >= 7))
+#define GCC_HAS_AVX2 1
+#endif // GNUC >= 4.7
+#endif // __GNUC__
+
+// clang >= 3.4.0 required for AVX2.
+#if defined(__clang__) && !defined(LIBYUV_ENABLE_ROWWIN) && \
+ (defined(__x86_64__) || defined(__i386__))
+#if (__clang_major__ > 3) || (__clang_major__ == 3 && (__clang_minor__ >= 4))
+#define CLANG_HAS_AVX2 1
+#endif // clang >= 3.4
+#endif // __clang__
+
+// clang >= 6.0.0 required for AVX512.
+#if defined(__clang__) && !defined(LIBYUV_ENABLE_ROWWIN) && \
+ (defined(__x86_64__) || defined(__i386__))
+// clang in xcode follows a different versioning scheme.
+// TODO(fbarchard): fix xcode 9 ios b/789.
+#if (__clang_major__ >= 7) && !defined(__APPLE__)
+#define CLANG_HAS_AVX512 1
+#endif // clang >= 7
+#endif // __clang__
+
+// Visual C 2012 required for AVX2.
+#if defined(_M_IX86) && \
+ (!defined(__clang__) || defined(LIBYUV_ENABLE_ROWWIN)) && \
+ defined(_MSC_VER) && _MSC_VER >= 1700
+#define VISUALC_HAS_AVX2 1
+#endif // VisualStudio >= 2012
+
+// Clang 19 required for SME due to needing __arm_tpidr2_save from compiler-rt,
+// only enabled on Linux and Android (both define __linux__) for now.
+#if !defined(LIBYUV_DISABLE_SME) && defined(__aarch64__) && \
+ defined(__linux__) && defined(__clang__) && (__clang_major__ >= 19)
+#define CLANG_HAS_SME 1
+#endif
+
+#ifdef __cplusplus
+} // extern "C"
+} // namespace libyuv
+#endif
+
+#endif // INCLUDE_LIBYUV_CPU_SUPPORT_H_
diff --git a/include/libyuv/macros_msa.h b/include/libyuv/macros_msa.h
index b9a44fc..6434a4d 100644
--- a/include/libyuv/macros_msa.h
+++ b/include/libyuv/macros_msa.h
@@ -20,9 +20,9 @@
({ \
const uint8_t* psrc_lw_m = (const uint8_t*)(psrc); \
uint32_t val_m; \
- asm volatile("lw %[val_m], %[psrc_lw_m] \n" \
- : [val_m] "=r"(val_m) \
- : [psrc_lw_m] "m"(*psrc_lw_m)); \
+ asm("lw %[val_m], %[psrc_lw_m] \n" \
+ : [val_m] "=r"(val_m) \
+ : [psrc_lw_m] "m"(*psrc_lw_m)); \
val_m; \
})
@@ -31,9 +31,9 @@
({ \
const uint8_t* psrc_ld_m = (const uint8_t*)(psrc); \
uint64_t val_m = 0; \
- asm volatile("ld %[val_m], %[psrc_ld_m] \n" \
- : [val_m] "=r"(val_m) \
- : [psrc_ld_m] "m"(*psrc_ld_m)); \
+ asm("ld %[val_m], %[psrc_ld_m] \n" \
+ : [val_m] "=r"(val_m) \
+ : [psrc_ld_m] "m"(*psrc_ld_m)); \
val_m; \
})
#else // !(__mips == 64)
@@ -55,9 +55,9 @@
({ \
uint8_t* pdst_sw_m = (uint8_t*)(pdst); /* NOLINT */ \
uint32_t val_m = (val); \
- asm volatile("sw %[val_m], %[pdst_sw_m] \n" \
- : [pdst_sw_m] "=m"(*pdst_sw_m) \
- : [val_m] "r"(val_m)); \
+ asm("sw %[val_m], %[pdst_sw_m] \n" \
+ : [pdst_sw_m] "=m"(*pdst_sw_m) \
+ : [val_m] "r"(val_m)); \
})
#if (__mips == 64)
@@ -65,9 +65,9 @@
({ \
uint8_t* pdst_sd_m = (uint8_t*)(pdst); /* NOLINT */ \
uint64_t val_m = (val); \
- asm volatile("sd %[val_m], %[pdst_sd_m] \n" \
- : [pdst_sd_m] "=m"(*pdst_sd_m) \
- : [val_m] "r"(val_m)); \
+ asm("sd %[val_m], %[pdst_sd_m] \n" \
+ : [pdst_sd_m] "=m"(*pdst_sd_m) \
+ : [val_m] "r"(val_m)); \
})
#else // !(__mips == 64)
#define SD(val, pdst) \
@@ -86,8 +86,7 @@
uint8_t* psrc_lw_m = (uint8_t*)(psrc); \
uint32_t val_lw_m; \
\
- __asm__ volatile( \
- "lwr %[val_lw_m], 0(%[psrc_lw_m]) \n\t" \
+ asm("lwr %[val_lw_m], 0(%[psrc_lw_m]) \n\t" \
"lwl %[val_lw_m], 3(%[psrc_lw_m]) \n\t" \
\
: [val_lw_m] "=&r"(val_lw_m) \
@@ -102,8 +101,7 @@
uint8_t* psrc_ld_m = (uint8_t*)(psrc); \
uint64_t val_ld_m = 0; \
\
- __asm__ volatile( \
- "ldr %[val_ld_m], 0(%[psrc_ld_m]) \n\t" \
+ asm("ldr %[val_ld_m], 0(%[psrc_ld_m]) \n\t" \
"ldl %[val_ld_m], 7(%[psrc_ld_m]) \n\t" \
\
: [val_ld_m] "=&r"(val_ld_m) \
@@ -130,9 +128,9 @@
({ \
uint8_t* pdst_sw_m = (uint8_t*)(pdst); /* NOLINT */ \
uint32_t val_m = (val); \
- asm volatile("usw %[val_m], %[pdst_sw_m] \n" \
- : [pdst_sw_m] "=m"(*pdst_sw_m) \
- : [val_m] "r"(val_m)); \
+ asm("usw %[val_m], %[pdst_sw_m] \n" \
+ : [pdst_sw_m] "=m"(*pdst_sw_m) \
+ : [val_m] "r"(val_m)); \
})
#define SD(val, pdst) \
diff --git a/include/libyuv/planar_functions.h b/include/libyuv/planar_functions.h
index f934472..dbf51de 100644
--- a/include/libyuv/planar_functions.h
+++ b/include/libyuv/planar_functions.h
@@ -38,8 +38,10 @@
#endif
#endif
// The following are available on all x86 platforms:
-#if !defined(LIBYUV_DISABLE_X86) && \
- (defined(_M_IX86) || defined(__x86_64__) || defined(__i386__))
+#if !defined(LIBYUV_DISABLE_X86) && \
+ (defined(_M_IX86) || \
+ (defined(__x86_64__) && !defined(LIBYUV_ENABLE_ROWWIN)) || \
+ defined(__i386__))
#define HAS_ARGBAFFINEROW_SSE2
#endif
@@ -78,6 +80,16 @@
int width,
int height);
+LIBYUV_API
+void Convert8To8Plane(const uint8_t* src_y,
+ int src_stride_y,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ int scale, // 220 for Y, 225 for U,V
+ int bias, // 16
+ int width,
+ int height);
+
// Set a plane of data to a 32 bit value.
LIBYUV_API
void SetPlane(uint8_t* dst_y,
@@ -766,6 +778,10 @@
// Convert plane of 16 bit shorts to half floats.
// Source values are multiplied by scale before storing as half float.
+//
+// Note: Unlike other libyuv functions that operate on uint16_t buffers, the
+// src_stride_y and dst_stride_y parameters of HalfFloatPlane() are in bytes,
+// not in units of uint16_t.
LIBYUV_API
int HalfFloatPlane(const uint16_t* src_y,
int src_stride_y,
diff --git a/include/libyuv/rotate_row.h b/include/libyuv/rotate_row.h
index 3e6a2fe..c00d83c 100644
--- a/include/libyuv/rotate_row.h
+++ b/include/libyuv/rotate_row.h
@@ -12,30 +12,15 @@
#define INCLUDE_LIBYUV_ROTATE_ROW_H_
#include "libyuv/basic_types.h"
+#include "libyuv/cpu_support.h"
#ifdef __cplusplus
namespace libyuv {
extern "C" {
#endif
-#if defined(__pnacl__) || defined(__CLR_VER) || \
- (defined(__native_client__) && defined(__x86_64__)) || \
- (defined(__i386__) && !defined(__SSE__) && !defined(__clang__))
-#define LIBYUV_DISABLE_X86
-#endif
-#if defined(__native_client__)
-#define LIBYUV_DISABLE_NEON
-#endif
-// MemorySanitizer does not support assembly code yet. http://crbug.com/344505
-#if defined(__has_feature)
-#if __has_feature(memory_sanitizer) && !defined(LIBYUV_DISABLE_NEON)
-#define LIBYUV_DISABLE_NEON
-#endif
-#if __has_feature(memory_sanitizer) && !defined(LIBYUV_DISABLE_X86)
-#define LIBYUV_DISABLE_X86
-#endif
-#endif
// The following are available for Visual C 32 bit:
+// TODO - port to clangcl on rotate_win
#if !defined(LIBYUV_DISABLE_X86) && defined(_M_IX86) && defined(_MSC_VER) && \
!defined(__clang__)
#define HAS_TRANSPOSEWX8_SSSE3
@@ -43,25 +28,38 @@
#endif
// The following are available for GCC 32 or 64 bit:
-#if !defined(LIBYUV_DISABLE_X86) && (defined(__i386__) || defined(__x86_64__))
+#if !defined(LIBYUV_DISABLE_X86) && \
+ (defined(__i386__) || defined(__x86_64__)) && \
+ !defined(LIBYUV_ENABLE_ROWWIN)
#define HAS_TRANSPOSEWX8_SSSE3
#define HAS_TRANSPOSE4X4_32_SSE2
#define HAS_TRANSPOSE4X4_32_AVX2
#endif
// The following are available for 64 bit GCC:
-#if !defined(LIBYUV_DISABLE_X86) && defined(__x86_64__)
+#if !defined(LIBYUV_DISABLE_X86) && defined(__x86_64__) && \
+ !defined(LIBYUV_ENABLE_ROWWIN)
#define HAS_TRANSPOSEWX8_FAST_SSSE3
#define HAS_TRANSPOSEUVWX8_SSE2
#endif
#if !defined(LIBYUV_DISABLE_NEON) && \
(defined(__ARM_NEON__) || defined(LIBYUV_NEON) || defined(__aarch64__))
+#if defined(__aarch64__)
+#define HAS_TRANSPOSEWX16_NEON
+#else
#define HAS_TRANSPOSEWX8_NEON
+#endif
#define HAS_TRANSPOSEUVWX8_NEON
#define HAS_TRANSPOSE4X4_32_NEON
#endif
+#if !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && \
+ defined(__aarch64__)
+#define HAS_TRANSPOSEWXH_SME
+#define HAS_TRANSPOSEUVWXH_SME
+#endif
+
#if !defined(LIBYUV_DISABLE_MSA) && defined(__mips_msa)
#define HAS_TRANSPOSEWX16_MSA
#define HAS_TRANSPOSEUVWX16_MSA
@@ -94,6 +92,17 @@
uint8_t* dst,
int dst_stride,
int width);
+void TransposeWx16_NEON(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width);
+void TransposeWxH_SME(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width,
+ int height);
void TransposeWx8_SSSE3(const uint8_t* src,
int src_stride,
uint8_t* dst,
@@ -120,6 +129,11 @@
uint8_t* dst,
int dst_stride,
int width);
+void TransposeWx16_Any_NEON(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width);
void TransposeWx8_Any_SSSE3(const uint8_t* src,
int src_stride,
uint8_t* dst,
@@ -178,6 +192,14 @@
uint8_t* dst_b,
int dst_stride_b,
int width);
+void TransposeUVWxH_SME(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst_a,
+ int dst_stride_a,
+ uint8_t* dst_b,
+ int dst_stride_b,
+ int width,
+ int height);
void TransposeUVWx16_MSA(const uint8_t* src,
int src_stride,
uint8_t* dst_a,
diff --git a/include/libyuv/row.h b/include/libyuv/row.h
index 46685a5..6e8f46f 100644
--- a/include/libyuv/row.h
+++ b/include/libyuv/row.h
@@ -15,70 +15,19 @@
#include <stdlib.h> // For malloc
#include "libyuv/basic_types.h"
+#include "libyuv/cpu_support.h"
#ifdef __cplusplus
namespace libyuv {
extern "C" {
#endif
-#if defined(__pnacl__) || defined(__CLR_VER) || \
- (defined(__native_client__) && defined(__x86_64__)) || \
- (defined(__i386__) && !defined(__SSE__) && !defined(__clang__))
-#define LIBYUV_DISABLE_X86
-#endif
-#if defined(__native_client__)
-#define LIBYUV_DISABLE_NEON
-#endif
-// MemorySanitizer does not support assembly code yet. http://crbug.com/344505
-#if defined(__has_feature)
-#if __has_feature(memory_sanitizer) && !defined(LIBYUV_DISABLE_NEON)
-#define LIBYUV_DISABLE_NEON
-#endif
-#if __has_feature(memory_sanitizer) && !defined(LIBYUV_DISABLE_X86)
-#define LIBYUV_DISABLE_X86
-#endif
-#endif
-// clang >= 3.5.0 required for Arm64.
-#if defined(__clang__) && defined(__aarch64__) && !defined(LIBYUV_DISABLE_NEON)
-#if (__clang_major__ < 3) || (__clang_major__ == 3 && (__clang_minor__ < 5))
-#define LIBYUV_DISABLE_NEON
-#endif // clang >= 3.5
-#endif // __clang__
-
-// GCC >= 4.7.0 required for AVX2.
-#if defined(__GNUC__) && (defined(__x86_64__) || defined(__i386__))
-#if (__GNUC__ > 4) || (__GNUC__ == 4 && (__GNUC_MINOR__ >= 7))
-#define GCC_HAS_AVX2 1
-#endif // GNUC >= 4.7
-#endif // __GNUC__
-
-// clang >= 3.4.0 required for AVX2.
-#if defined(__clang__) && (defined(__x86_64__) || defined(__i386__))
-#if (__clang_major__ > 3) || (__clang_major__ == 3 && (__clang_minor__ >= 4))
-#define CLANG_HAS_AVX2 1
-#endif // clang >= 3.4
-#endif // __clang__
-
-// clang >= 6.0.0 required for AVX512.
-#if defined(__clang__) && (defined(__x86_64__) || defined(__i386__))
-// clang in xcode follows a different versioning scheme.
-// TODO(fbarchard): fix xcode 9 ios b/789.
-#if (__clang_major__ >= 7) && !defined(__APPLE__)
-#define CLANG_HAS_AVX512 1
-#endif // clang >= 7
-#endif // __clang__
-
-// Visual C 2012 required for AVX2.
-#if defined(_M_IX86) && !defined(__clang__) && defined(_MSC_VER) && \
- _MSC_VER >= 1700
-#define VISUALC_HAS_AVX2 1
-#endif // VisualStudio >= 2012
-
// The following are available on all x86 platforms:
-#if !defined(LIBYUV_DISABLE_X86) && \
- (defined(_M_IX86) || defined(__x86_64__) || defined(__i386__))
+#if !defined(LIBYUV_DISABLE_X86) && \
+ (defined(_M_IX86) || \
+ (defined(__x86_64__) && !defined(LIBYUV_ENABLE_ROWWIN)) || \
+ defined(__i386__))
// Conversions:
-#define HAS_ABGRTOYROW_SSSE3
#define HAS_ARGB1555TOARGBROW_SSE2
#define HAS_ARGB4444TOARGBROW_SSE2
#define HAS_ARGBEXTRACTALPHAROW_SSE2
@@ -90,9 +39,6 @@
#define HAS_ARGBTORGB24ROW_SSSE3
#define HAS_ARGBTORGB565DITHERROW_SSE2
#define HAS_ARGBTORGB565ROW_SSE2
-#define HAS_ARGBTOYJROW_SSSE3
-#define HAS_ARGBTOYROW_SSSE3
-#define HAS_BGRATOYROW_SSSE3
#define HAS_COPYROW_ERMS
#define HAS_COPYROW_SSE2
#define HAS_H422TOARGBROW_SSSE3
@@ -120,13 +66,8 @@
#define HAS_NV21TORGB24ROW_SSSE3
#define HAS_RAWTOARGBROW_SSSE3
#define HAS_RAWTORGB24ROW_SSSE3
-#define HAS_RAWTOYJROW_SSSE3
-#define HAS_RAWTOYROW_SSSE3
#define HAS_RGB24TOARGBROW_SSSE3
-#define HAS_RGB24TOYJROW_SSSE3
-#define HAS_RGB24TOYROW_SSSE3
#define HAS_RGB565TOARGBROW_SSE2
-#define HAS_RGBATOYROW_SSSE3
#define HAS_SETROW_ERMS
#define HAS_SETROW_X86
#define HAS_SPLITUVROW_SSE2
@@ -140,8 +81,6 @@
#define HAS_YUY2TOYROW_SSE2
#if !defined(LIBYUV_BIT_EXACT)
#define HAS_ABGRTOUVROW_SSSE3
-#define HAS_ARGBTOUV444ROW_SSSE3
-#define HAS_ARGBTOUVJROW_SSSE3
#define HAS_ARGBTOUVROW_SSSE3
#define HAS_BGRATOUVROW_SSSE3
#define HAS_RGBATOUVROW_SSSE3
@@ -158,7 +97,8 @@
#define HAS_ARGBGRAYROW_SSSE3
#define HAS_ARGBLUMACOLORTABLEROW_SSSE3
#define HAS_ARGBMIRRORROW_SSE2
-#define HAS_ARGBMULTIPLYROW_SSE2
+// TODO: Re-enable once rounding behaviour is fixed.
+// #define HAS_ARGBMULTIPLYROW_SSE2
#define HAS_ARGBPOLYNOMIALROW_SSE2
#define HAS_ARGBQUANTIZEROW_SSE2
#define HAS_ARGBSEPIAROW_SSSE3
@@ -176,13 +116,24 @@
// The following functions fail on gcc/clang 32 bit with fpic and framepointer.
// caveat: clangcl uses row_win.cc which works.
-#if defined(__x86_64__) || !defined(__pic__) || defined(__clang__) || \
- defined(_MSC_VER)
+#if (defined(__x86_64__) || !defined(__pic__) || defined(__clang__) || \
+ defined(_MSC_VER)) && \
+ !defined(LIBYUV_ENABLE_ROWWIN)
// TODO(fbarchard): fix build error on android_full_debug=1
// https://code.google.com/p/libyuv/issues/detail?id=517
#define HAS_I422ALPHATOARGBROW_SSSE3
#define HAS_I444ALPHATOARGBROW_SSSE3
#endif
+#if (defined(__x86_64__) || !defined(__pic__) || defined(__clang__) || \
+ defined(_MSC_VER)) && \
+ !defined(LIBYUV_ENABLE_ROWWIN)
+// TODO(fbarchard): fix build error on android_full_debug=1
+// https://code.google.com/p/libyuv/issues/detail?id=517
+// TODO(fbarchard): fix LIBYUV_ENABLE_ROWWIN with clang
+#define HAS_I422ALPHATOARGBROW_AVX2
+#define HAS_I444ALPHATOARGBROW_AVX2
+#endif
+
#endif
// The following are available on all x86 platforms, but
@@ -197,8 +148,6 @@
#define HAS_ARGBPOLYNOMIALROW_AVX2
#define HAS_ARGBSHUFFLEROW_AVX2
#define HAS_ARGBTORGB565DITHERROW_AVX2
-#define HAS_ARGBTOYJROW_AVX2
-#define HAS_ARGBTOYROW_AVX2
#define HAS_COPYROW_AVX
#define HAS_H422TOARGBROW_AVX2
#define HAS_HALFFLOATROW_AVX2
@@ -219,8 +168,6 @@
#define HAS_NV12TORGB565ROW_AVX2
#define HAS_NV21TOARGBROW_AVX2
#define HAS_NV21TORGB24ROW_AVX2
-#define HAS_RAWTOYJROW_AVX2
-#define HAS_RGB24TOYJROW_AVX2
#define HAS_SPLITUVROW_AVX2
#define HAS_UYVYTOARGBROW_AVX2
#define HAS_UYVYTOUV422ROW_AVX2
@@ -231,24 +178,13 @@
#define HAS_YUY2TOUVROW_AVX2
#define HAS_YUY2TOYROW_AVX2
// #define HAS_HALFFLOATROW_F16C // Enable to test half float cast
-#if !defined(LIBYUV_BIT_EXACT)
-#define HAS_ARGBTOUVJROW_AVX2
-#define HAS_ARGBTOUVROW_AVX2
-#endif
// Effects:
#define HAS_ARGBADDROW_AVX2
-#define HAS_ARGBMULTIPLYROW_AVX2
+// TODO: Re-enable once rounding behaviour is fixed.
+// #define HAS_ARGBMULTIPLYROW_AVX2
#define HAS_ARGBSUBTRACTROW_AVX2
#define HAS_BLENDPLANEROW_AVX2
-
-#if defined(__x86_64__) || !defined(__pic__) || defined(__clang__) || \
- defined(_MSC_VER)
-// TODO(fbarchard): fix build error on android_full_debug=1
-// https://code.google.com/p/libyuv/issues/detail?id=517
-#define HAS_I422ALPHATOARGBROW_AVX2
-#define HAS_I444ALPHATOARGBROW_AVX2
-#endif
#endif
// The following are available for AVX2 Visual C 32 bit:
@@ -275,7 +211,10 @@
// The following are available for gcc/clang x86 platforms:
// TODO(fbarchard): Port to Visual C
-#if !defined(LIBYUV_DISABLE_X86) && (defined(__x86_64__) || defined(__i386__))
+#if !defined(LIBYUV_DISABLE_X86) && \
+ (defined(__x86_64__) || defined(__i386__)) && \
+ !defined(LIBYUV_ENABLE_ROWWIN)
+#define HAS_RAWTOYJROW_SSSE3
#define HAS_AB64TOARGBROW_SSSE3
#define HAS_ABGRTOAR30ROW_SSSE3
#define HAS_ABGRTOYJROW_SSSE3
@@ -284,6 +223,8 @@
#define HAS_ARGBTOAB64ROW_SSSE3
#define HAS_ARGBTOAR30ROW_SSSE3
#define HAS_ARGBTOAR64ROW_SSSE3
+#define HAS_ARGBTOUV444ROW_SSSE3
+#define HAS_ARGBTOUVJ444ROW_SSSE3
#define HAS_ARGBUNATTENUATEROW_SSE2
#define HAS_CONVERT16TO8ROW_SSSE3
#define HAS_CONVERT8TO16ROW_SSE2
@@ -309,18 +250,32 @@
#define HAS_P210TOARGBROW_SSSE3
#define HAS_P410TOAR30ROW_SSSE3
#define HAS_P410TOARGBROW_SSSE3
+#define HAS_RAWTOARGBROW_AVX2
#define HAS_RAWTORGBAROW_SSSE3
#define HAS_RGB24MIRRORROW_SSSE3
#define HAS_RGBATOYJROW_SSSE3
#define HAS_SPLITARGBROW_SSE2
#define HAS_SPLITARGBROW_SSSE3
+#define HAS_SPLITRGBROW_SSE41
#define HAS_SPLITRGBROW_SSSE3
#define HAS_SPLITXRGBROW_SSE2
#define HAS_SPLITXRGBROW_SSSE3
#define HAS_SWAPUVROW_SSSE3
#define HAS_YUY2TONVUVROW_SSE2
+// TODO: port row_win to use 8 bit coefficients.
+#define HAS_ARGBTOYJROW_SSSE3
+#define HAS_ARGBTOYROW_SSSE3
+#define HAS_BGRATOYROW_SSSE3
+#define HAS_RAWTOYROW_SSSE3
+#define HAS_ABGRTOYROW_SSSE3
+#define HAS_RGB24TOYJROW_SSSE3
+#define HAS_RGB24TOYROW_SSSE3
+#define HAS_RGBATOYROW_SSSE3
+
#if !defined(LIBYUV_BIT_EXACT)
+// TODO: adjust row_win to use 8 bit negative coefficients.
#define HAS_ABGRTOUVJROW_SSSE3
+#define HAS_ARGBTOUVJROW_SSSE3
#endif
#if defined(__x86_64__) || !defined(__pic__)
@@ -333,9 +288,15 @@
// The following are available for AVX2 gcc/clang x86 platforms:
// TODO(fbarchard): Port to Visual C
-#if !defined(LIBYUV_DISABLE_X86) && \
- (defined(__x86_64__) || defined(__i386__)) && \
- (defined(CLANG_HAS_AVX2) || defined(GCC_HAS_AVX2))
+#if !defined(LIBYUV_DISABLE_X86) && \
+ (defined(__x86_64__) || defined(__i386__)) && \
+ (defined(CLANG_HAS_AVX2) || defined(GCC_HAS_AVX2)) && \
+ !defined(LIBYUV_ENABLE_ROWWIN)
+#define HAS_RAWTOYJROW_AVX2
+#define HAS_RGB24TOYJROW_AVX2
+
+#define HAS_ARGBTOYJROW_AVX2
+#define HAS_ARGBTOYROW_AVX2
#define HAS_AB64TOARGBROW_AVX2
#define HAS_ABGRTOAR30ROW_AVX2
#define HAS_ABGRTOYJROW_AVX2
@@ -347,9 +308,12 @@
#define HAS_ARGBTOAR64ROW_AVX2
#define HAS_ARGBTORAWROW_AVX2
#define HAS_ARGBTORGB24ROW_AVX2
+#define HAS_ARGBTOUV444ROW_AVX2
+#define HAS_ARGBTOUVJ444ROW_AVX2
#define HAS_ARGBUNATTENUATEROW_AVX2
#define HAS_CONVERT16TO8ROW_AVX2
#define HAS_CONVERT8TO16ROW_AVX2
+#define HAS_CONVERT8TO8ROW_AVX2
#define HAS_DETILEROW_16_AVX
#define HAS_DIVIDEROW_16_AVX2
#define HAS_HALFMERGEUVROW_AVX2
@@ -381,6 +345,7 @@
#define HAS_P410TOARGBROW_AVX2
#define HAS_RGBATOYJROW_AVX2
#define HAS_SPLITARGBROW_AVX2
+#define HAS_SPLITRGBROW_AVX2
#define HAS_SPLITUVROW_16_AVX2
#define HAS_SPLITXRGBROW_AVX2
#define HAS_SWAPUVROW_AVX2
@@ -388,6 +353,8 @@
#if !defined(LIBYUV_BIT_EXACT)
#define HAS_ABGRTOUVJROW_AVX2
#define HAS_ABGRTOUVROW_AVX2
+#define HAS_ARGBTOUVJROW_AVX2
+#define HAS_ARGBTOUVROW_AVX2
#endif
#if defined(__x86_64__) || !defined(__pic__)
@@ -400,10 +367,13 @@
// The following are available for AVX512 clang x86 platforms:
// TODO(fbarchard): Port to GCC and Visual C
-// TODO(fbarchard): re-enable HAS_ARGBTORGB24ROW_AVX512VBMI. Issue libyuv:789
-#if !defined(LIBYUV_DISABLE_X86) && \
- (defined(__x86_64__) || defined(__i386__)) && defined(CLANG_HAS_AVX512)
+// TODO(b/42280744): re-enable HAS_ARGBTORGB24ROW_AVX512VBMI.
+#if !defined(LIBYUV_DISABLE_X86) && \
+ (defined(__x86_64__) || defined(__i386__)) && defined(CLANG_HAS_AVX512) && \
+ !defined(LIBYUV_ENABLE_ROWWIN)
+#define HAS_COPYROW_AVX512BW
#define HAS_ARGBTORGB24ROW_AVX512VBMI
+#define HAS_CONVERT16TO8ROW_AVX512BW
#define HAS_MERGEUVROW_AVX512BW
#endif
@@ -440,6 +410,7 @@
#define HAS_ARGBTORGB565DITHERROW_NEON
#define HAS_ARGBTORGB565ROW_NEON
#define HAS_ARGBTOUV444ROW_NEON
+#define HAS_ARGBTOUVJ444ROW_NEON
#define HAS_ARGBTOUVJROW_NEON
#define HAS_ARGBTOUVROW_NEON
#define HAS_ARGBTOYJROW_NEON
@@ -451,6 +422,7 @@
#define HAS_BGRATOYROW_NEON
#define HAS_BYTETOFLOATROW_NEON
#define HAS_CONVERT16TO8ROW_NEON
+#define HAS_CONVERT8TO8ROW_NEON
#define HAS_COPYROW_NEON
#define HAS_DETILEROW_16_NEON
#define HAS_DETILEROW_NEON
@@ -553,6 +525,133 @@
// The following are available on AArch64 platforms:
#if !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__)
+#define HAS_ARGBTOAR30ROW_NEON
+#define HAS_ABGRTOAR30ROW_NEON
+#define HAS_I210ALPHATOARGBROW_NEON
+#define HAS_I410ALPHATOARGBROW_NEON
+#define HAS_I210TOARGBROW_NEON
+#define HAS_I410TOARGBROW_NEON
+#define HAS_I210TOAR30ROW_NEON
+#define HAS_I410TOAR30ROW_NEON
+#define HAS_I212TOARGBROW_NEON
+#define HAS_I212TOAR30ROW_NEON
+#define HAS_I422TOAR30ROW_NEON
+#define HAS_P210TOAR30ROW_NEON
+#define HAS_P210TOARGBROW_NEON
+#define HAS_P410TOAR30ROW_NEON
+#define HAS_P410TOARGBROW_NEON
+
+#define HAS_ABGRTOYJROW_NEON_DOTPROD
+#define HAS_ABGRTOYROW_NEON_DOTPROD
+#define HAS_ARGBTOYJROW_NEON_DOTPROD
+#define HAS_ARGBTOYROW_NEON_DOTPROD
+#define HAS_BGRATOYROW_NEON_DOTPROD
+#define HAS_RGBATOYJROW_NEON_DOTPROD
+#define HAS_RGBATOYROW_NEON_DOTPROD
+#define HAS_ARGBGRAYROW_NEON_DOTPROD
+#define HAS_ARGBSEPIAROW_NEON_DOTPROD
+
+#define HAS_ARGBCOLORMATRIXROW_NEON_I8MM
+#define HAS_ARGBTOUV444ROW_NEON_I8MM
+#define HAS_ARGBTOUVJ444ROW_NEON_I8MM
+#endif
+
+// The following are available on AArch64 SVE platforms:
+#if !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__)
+#define HAS_ABGRTOUVJROW_SVE2
+#define HAS_ABGRTOUVROW_SVE2
+#define HAS_ARGB1555TOARGBROW_SVE2
+#define HAS_ARGBTORAWROW_SVE2
+#define HAS_ARGBTORGB24ROW_SVE2
+#define HAS_ARGBTORGB565DITHERROW_SVE2
+#define HAS_ARGBTORGB565ROW_SVE2
+#define HAS_ARGBTOUVJROW_SVE2
+#define HAS_ARGBTOUVROW_SVE2
+#define HAS_AYUVTOUVROW_SVE2
+#define HAS_AYUVTOVUROW_SVE2
+#define HAS_BGRATOUVROW_SVE2
+#define HAS_CONVERT8TO8ROW_SVE2
+#define HAS_DIVIDEROW_16_SVE2
+#define HAS_HALFFLOATROW_SVE2
+#define HAS_I210ALPHATOARGBROW_SVE2
+#define HAS_I210TOAR30ROW_SVE2
+#define HAS_I210TOARGBROW_SVE2
+#define HAS_I212TOAR30ROW_SVE2
+#define HAS_I212TOARGBROW_SVE2
+#define HAS_I400TOARGBROW_SVE2
+#define HAS_I410ALPHATOARGBROW_SVE2
+#define HAS_I410TOAR30ROW_SVE2
+#define HAS_I410TOARGBROW_SVE2
+#define HAS_I422ALPHATOARGBROW_SVE2
+#define HAS_I422TOARGB1555ROW_SVE2
+#define HAS_I422TOARGB4444ROW_SVE2
+#define HAS_I422TOARGBROW_SVE2
+#define HAS_I422TORGB24ROW_SVE2
+#define HAS_I422TORGB565ROW_SVE2
+#define HAS_I422TORGBAROW_SVE2
+#define HAS_I444ALPHATOARGBROW_SVE2
+#define HAS_I444TOARGBROW_SVE2
+#define HAS_NV12TOARGBROW_SVE2
+#define HAS_NV12TORGB24ROW_SVE2
+#define HAS_NV21TOARGBROW_SVE2
+#define HAS_NV21TORGB24ROW_SVE2
+#define HAS_P210TOAR30ROW_SVE2
+#define HAS_P210TOARGBROW_SVE2
+#define HAS_P410TOAR30ROW_SVE2
+#define HAS_P410TOARGBROW_SVE2
+#define HAS_RAWTOARGBROW_SVE2
+#define HAS_RAWTORGB24ROW_SVE2
+#define HAS_RAWTORGBAROW_SVE2
+#define HAS_RGB24TOARGBROW_SVE2
+#define HAS_RGBATOUVROW_SVE2
+#define HAS_UYVYTOARGBROW_SVE2
+#define HAS_YUY2TOARGBROW_SVE2
+#endif
+
+// The following are available on AArch64 SME platforms:
+#if !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && \
+ defined(__aarch64__)
+#define HAS_ARGBMULTIPLYROW_SME
+#define HAS_CONVERT16TO8ROW_SME
+#define HAS_CONVERT8TO8ROW_SME
+#define HAS_COPYROW_SME
+#define HAS_I210ALPHATOARGBROW_SME
+#define HAS_I210TOAR30ROW_SME
+#define HAS_I210TOARGBROW_SME
+#define HAS_I212TOAR30ROW_SME
+#define HAS_I212TOARGBROW_SME
+#define HAS_I400TOARGBROW_SME
+#define HAS_I410ALPHATOARGBROW_SME
+#define HAS_I410TOAR30ROW_SME
+#define HAS_I410TOARGBROW_SME
+#define HAS_I422ALPHATOARGBROW_SME
+#define HAS_I422TOARGB1555ROW_SME
+#define HAS_I422TOARGB4444ROW_SME
+#define HAS_I422TOARGBROW_SME
+#define HAS_I422TORGB24ROW_SME
+#define HAS_I422TORGB565ROW_SME
+#define HAS_I422TORGBAROW_SME
+#define HAS_I444ALPHATOARGBROW_SME
+#define HAS_I444TOARGBROW_SME
+#define HAS_INTERPOLATEROW_16_SME
+#define HAS_INTERPOLATEROW_16TO8_SME
+#define HAS_INTERPOLATEROW_SME
+#define HAS_MERGEUVROW_16_SME
+#define HAS_MERGEUVROW_SME
+#define HAS_MULTIPLYROW_16_SME
+#define HAS_NV12TOARGBROW_SME
+#define HAS_NV12TORGB24ROW_SME
+#define HAS_NV21TOARGBROW_SME
+#define HAS_NV21TORGB24ROW_SME
+#define HAS_P210TOAR30ROW_SME
+#define HAS_P210TOARGBROW_SME
+#define HAS_P410TOAR30ROW_SME
+#define HAS_P410TOARGBROW_SME
+#define HAS_YUY2TOARGBROW_SME
+#endif
+
+// The following are available on AArch64 platforms:
+#if !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__)
#define HAS_GAUSSCOL_F32_NEON
#define HAS_GAUSSROW_F32_NEON
#define HAS_INTERPOLATEROW_16TO8_NEON
@@ -573,7 +672,8 @@
#define HAS_ARGBEXTRACTALPHAROW_MSA
#define HAS_ARGBGRAYROW_MSA
#define HAS_ARGBMIRRORROW_MSA
-#define HAS_ARGBMULTIPLYROW_MSA
+// TODO: Re-enable once rounding behaviour is fixed.
+// #define HAS_ARGBMULTIPLYROW_MSA
#define HAS_ARGBQUANTIZEROW_MSA
#define HAS_ARGBSEPIAROW_MSA
#define HAS_ARGBSETROW_MSA
@@ -672,7 +772,8 @@
#define HAS_ARGBTOUVROW_LSX
#define HAS_ARGBTOYJROW_LSX
#define HAS_ARGBMIRRORROW_LSX
-#define HAS_ARGBMULTIPLYROW_LSX
+// TODO: Re-enable once rounding behaviour is fixed.
+// #define HAS_ARGBMULTIPLYROW_LSX
#define HAS_BGRATOUVROW_LSX
#define HAS_BGRATOYROW_LSX
#define HAS_I400TOARGBROW_LSX
@@ -739,7 +840,8 @@
#define HAS_ARGBATTENUATEROW_LASX
#define HAS_ARGBGRAYROW_LASX
#define HAS_ARGBMIRRORROW_LASX
-#define HAS_ARGBMULTIPLYROW_LASX
+// TODO: Re-enable once rounding behaviour is fixed.
+// #define HAS_ARGBMULTIPLYROW_LASX
#define HAS_ARGBSEPIAROW_LASX
#define HAS_ARGBSHADEROW_LASX
#define HAS_ARGBSHUFFLEROW_LASX
@@ -793,29 +895,73 @@
#define HAS_RAWTOYJROW_LASX
#endif
+#if defined(__riscv_v_intrinsic) && __riscv_v_intrinsic >= 12000 && \
+ !defined(LIBYUV_DISABLE_RVV)
+#define LIBYUV_DISABLE_RVV 1
+#endif
+
#if !defined(LIBYUV_DISABLE_RVV) && defined(__riscv_vector)
-#define HAS_COPYROW_RVV
-#if __riscv_v_intrinsic == 11000
-#define HAS_AB64TOARGBROW_RVV
+#if defined(__riscv_v_intrinsic) && __riscv_v_intrinsic > 11000
+// Since v0.12, TUPLE_TYPE is introduced for segment load and store.
+#define LIBYUV_RVV_HAS_TUPLE_TYPE
+// Since v0.12, VXRM(fixed-point rounding mode) is included in arguments of
+// fixed-point intrinsics.
+#define LIBYUV_RVV_HAS_VXRM_ARG
+#endif
+#endif
+
+// The following are available for RVV 1.2
+#if !defined(LIBYUV_DISABLE_RVV) && defined(__riscv_vector)
#define HAS_ABGRTOYJROW_RVV
#define HAS_ABGRTOYROW_RVV
#define HAS_AR64TOARGBROW_RVV
-#define HAS_AR64TOAB64ROW_RVV
-#define HAS_ARGBATTENUATEROW_RVV
-#define HAS_ARGBBLENDROW_RVV
#define HAS_ARGBCOPYYTOALPHAROW_RVV
#define HAS_ARGBEXTRACTALPHAROW_RVV
-#define HAS_ARGBTOAB64ROW_RVV
-#define HAS_ARGBTOABGRROW_RVV
#define HAS_ARGBTOAR64ROW_RVV
-#define HAS_ARGBTOBGRAROW_RVV
-#define HAS_ARGBTORAWROW_RVV
-#define HAS_ARGBTORGB24ROW_RVV
-#define HAS_ARGBTORGBAROW_RVV
#define HAS_ARGBTOYJROW_RVV
#define HAS_ARGBTOYMATRIXROW_RVV
#define HAS_ARGBTOYROW_RVV
#define HAS_BGRATOYROW_RVV
+#define HAS_COPYROW_RVV
+#define HAS_INTERPOLATEROW_RVV
+#define HAS_RAWTOYJROW_RVV
+#define HAS_RAWTOYROW_RVV
+#define HAS_RGB24TOYJROW_RVV
+#define HAS_RGB24TOYROW_RVV
+#define HAS_RGBATOYJROW_RVV
+#define HAS_RGBATOYMATRIXROW_RVV
+#define HAS_RGBATOYROW_RVV
+#define HAS_RGBTOYMATRIXROW_RVV
+#define HAS_SPLITARGBROW_RVV
+#define HAS_SPLITRGBROW_RVV
+#define HAS_SPLITUVROW_RVV
+#define HAS_SPLITXRGBROW_RVV
+#endif
+
+// The following are available for RVV 1.1
+// TODO(fbarchard): Port to RVV 1.2 (tuple)
+// missing support for vcreate_v:
+// __riscv_vcreate_v_u16m2x2
+// __riscv_vcreate_v_u16m2x4
+// __riscv_vcreate_v_u16m4x2
+// __riscv_vcreate_v_u8m1x3
+// __riscv_vcreate_v_u8m1x4
+// __riscv_vcreate_v_u8m2x2
+// __riscv_vcreate_v_u8m2x3
+// __riscv_vcreate_v_u8m2x4
+// __riscv_vcreate_v_u8m4x2
+#if !defined(LIBYUV_DISABLE_RVV) && defined(__riscv_vector) && \
+ !defined(LIBYUV_RVV_HAS_TUPLE_TYPE)
+#define HAS_AB64TOARGBROW_RVV
+#define HAS_AR64TOAB64ROW_RVV
+#define HAS_ARGBATTENUATEROW_RVV
+#define HAS_ARGBBLENDROW_RVV
+#define HAS_ARGBTOAB64ROW_RVV
+#define HAS_ARGBTOABGRROW_RVV
+#define HAS_ARGBTOBGRAROW_RVV
+#define HAS_ARGBTORAWROW_RVV
+#define HAS_ARGBTORGB24ROW_RVV
+#define HAS_ARGBTORGBAROW_RVV
#define HAS_BLENDPLANEROW_RVV
#define HAS_I400TOARGBROW_RVV
#define HAS_I422ALPHATOARGBROW_RVV
@@ -825,7 +971,6 @@
#define HAS_I444ALPHATOARGBROW_RVV
#define HAS_I444TOARGBROW_RVV
#define HAS_I444TORGB24ROW_RVV
-#define HAS_INTERPOLATEROW_RVV
#define HAS_J400TOARGBROW_RVV
#define HAS_MERGEARGBROW_RVV
#define HAS_MERGERGBROW_RVV
@@ -838,21 +983,8 @@
#define HAS_RAWTOARGBROW_RVV
#define HAS_RAWTORGB24ROW_RVV
#define HAS_RAWTORGBAROW_RVV
-#define HAS_RAWTOYJROW_RVV
-#define HAS_RAWTOYROW_RVV
#define HAS_RGB24TOARGBROW_RVV
-#define HAS_RGB24TOYJROW_RVV
-#define HAS_RGB24TOYROW_RVV
#define HAS_RGBATOARGBROW_RVV
-#define HAS_RGBATOYJROW_RVV
-#define HAS_RGBATOYMATRIXROW_RVV
-#define HAS_RGBATOYROW_RVV
-#define HAS_RGBTOYMATRIXROW_RVV
-#define HAS_SPLITARGBROW_RVV
-#define HAS_SPLITRGBROW_RVV
-#define HAS_SPLITUVROW_RVV
-#define HAS_SPLITXRGBROW_RVV
-#endif
#endif
#if defined(_MSC_VER) && !defined(__CLR_VER) && !defined(__clang__)
@@ -1013,24 +1145,204 @@
IACA_UD_BYTES \
}
+void I210AlphaToARGBRow_NEON(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ const uint16_t* src_a,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I210AlphaToARGBRow_SVE2(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ const uint16_t* src_a,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I210AlphaToARGBRow_SME(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ const uint16_t* src_a,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I410AlphaToARGBRow_NEON(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ const uint16_t* src_a,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I410AlphaToARGBRow_SVE2(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ const uint16_t* src_a,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I410AlphaToARGBRow_SME(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ const uint16_t* src_a,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width);
void I444ToARGBRow_NEON(const uint8_t* src_y,
const uint8_t* src_u,
const uint8_t* src_v,
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width);
+void I444ToARGBRow_SVE2(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I444ToARGBRow_SME(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
void I444ToRGB24Row_NEON(const uint8_t* src_y,
const uint8_t* src_u,
const uint8_t* src_v,
uint8_t* dst_rgb24,
const struct YuvConstants* yuvconstants,
int width);
+void I210ToARGBRow_NEON(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I210ToARGBRow_SVE2(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I210ToARGBRow_SME(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I410ToARGBRow_NEON(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I410ToARGBRow_SVE2(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I410ToARGBRow_SME(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I210ToAR30Row_NEON(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I210ToAR30Row_SVE2(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I210ToAR30Row_SME(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I410ToAR30Row_NEON(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I410ToAR30Row_SVE2(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I410ToAR30Row_SME(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I212ToARGBRow_NEON(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I212ToARGBRow_SVE2(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I212ToARGBRow_SME(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I212ToAR30Row_NEON(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I212ToAR30Row_SVE2(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I212ToAR30Row_SME(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* rgb_buf,
+ const struct YuvConstants* yuvconstants,
+ int width);
void I422ToARGBRow_NEON(const uint8_t* src_y,
const uint8_t* src_u,
const uint8_t* src_v,
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width);
+void I422ToARGBRow_SVE2(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToARGBRow_SME(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToAR30Row_NEON(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
void I444AlphaToARGBRow_NEON(const uint8_t* src_y,
const uint8_t* src_u,
const uint8_t* src_v,
@@ -1038,6 +1350,20 @@
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width);
+void I444AlphaToARGBRow_SVE2(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ const uint8_t* src_a,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I444AlphaToARGBRow_SME(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ const uint8_t* src_a,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
void I422AlphaToARGBRow_NEON(const uint8_t* src_y,
const uint8_t* src_u,
const uint8_t* src_v,
@@ -1045,41 +1371,125 @@
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width);
+void I422AlphaToARGBRow_SVE2(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ const uint8_t* src_a,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422AlphaToARGBRow_SME(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ const uint8_t* src_a,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
void I422ToRGBARow_NEON(const uint8_t* src_y,
const uint8_t* src_u,
const uint8_t* src_v,
uint8_t* dst_rgba,
const struct YuvConstants* yuvconstants,
int width);
+void I422ToRGBARow_SVE2(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_rgba,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToRGBARow_SME(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_rgba,
+ const struct YuvConstants* yuvconstants,
+ int width);
void I422ToRGB24Row_NEON(const uint8_t* src_y,
const uint8_t* src_u,
const uint8_t* src_v,
uint8_t* dst_rgb24,
const struct YuvConstants* yuvconstants,
int width);
+void I422ToRGB24Row_SVE2(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToRGB24Row_SME(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width);
void I422ToRGB565Row_NEON(const uint8_t* src_y,
const uint8_t* src_u,
const uint8_t* src_v,
uint8_t* dst_rgb565,
const struct YuvConstants* yuvconstants,
int width);
+void I422ToRGB565Row_SVE2(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_rgb565,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToRGB565Row_SME(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_rgb565,
+ const struct YuvConstants* yuvconstants,
+ int width);
void I422ToARGB1555Row_NEON(const uint8_t* src_y,
const uint8_t* src_u,
const uint8_t* src_v,
uint8_t* dst_argb1555,
const struct YuvConstants* yuvconstants,
int width);
+void I422ToARGB1555Row_SVE2(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb1555,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToARGB1555Row_SME(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb1555,
+ const struct YuvConstants* yuvconstants,
+ int width);
void I422ToARGB4444Row_NEON(const uint8_t* src_y,
const uint8_t* src_u,
const uint8_t* src_v,
uint8_t* dst_argb4444,
const struct YuvConstants* yuvconstants,
int width);
+void I422ToARGB4444Row_SVE2(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb4444,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I422ToARGB4444Row_SME(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb4444,
+ const struct YuvConstants* yuvconstants,
+ int width);
void NV12ToARGBRow_NEON(const uint8_t* src_y,
const uint8_t* src_uv,
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width);
+void NV12ToARGBRow_SVE2(const uint8_t* src_y,
+ const uint8_t* src_uv,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void NV12ToARGBRow_SME(const uint8_t* src_y,
+ const uint8_t* src_uv,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
void NV12ToRGB565Row_NEON(const uint8_t* src_y,
const uint8_t* src_uv,
uint8_t* dst_rgb565,
@@ -1090,16 +1500,46 @@
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width);
+void NV21ToARGBRow_SVE2(const uint8_t* src_y,
+ const uint8_t* src_vu,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void NV21ToARGBRow_SME(const uint8_t* src_y,
+ const uint8_t* src_vu,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
void NV12ToRGB24Row_NEON(const uint8_t* src_y,
const uint8_t* src_uv,
uint8_t* dst_rgb24,
const struct YuvConstants* yuvconstants,
int width);
+void NV12ToRGB24Row_SVE2(const uint8_t* src_y,
+ const uint8_t* src_uv,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void NV12ToRGB24Row_SME(const uint8_t* src_y,
+ const uint8_t* src_uv,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width);
void NV21ToRGB24Row_NEON(const uint8_t* src_y,
const uint8_t* src_vu,
uint8_t* dst_rgb24,
const struct YuvConstants* yuvconstants,
int width);
+void NV21ToRGB24Row_SVE2(const uint8_t* src_y,
+ const uint8_t* src_vu,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void NV21ToRGB24Row_SME(const uint8_t* src_y,
+ const uint8_t* src_vu,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width);
void NV21ToYUV24Row_NEON(const uint8_t* src_y,
const uint8_t* src_vu,
uint8_t* dst_yuv24,
@@ -1108,10 +1548,26 @@
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width);
+void YUY2ToARGBRow_SVE2(const uint8_t* src_yuy2,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void YUY2ToARGBRow_SME(const uint8_t* src_yuy2,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
void UYVYToARGBRow_NEON(const uint8_t* src_uyvy,
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width);
+void UYVYToARGBRow_SVE2(const uint8_t* src_uyvy,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void UYVYToARGBRow_SME(const uint8_t* src_uyvy,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
void I444ToARGBRow_RVV(const uint8_t* src_y,
const uint8_t* src_u,
const uint8_t* src_v,
@@ -1409,6 +1865,18 @@
void ARGBToYJRow_NEON(const uint8_t* src_argb, uint8_t* dst_yj, int width);
void ABGRToYJRow_NEON(const uint8_t* src_abgr, uint8_t* dst_yj, int width);
void RGBAToYJRow_NEON(const uint8_t* src_rgba, uint8_t* dst_yj, int width);
+void ARGBToYRow_NEON_DotProd(const uint8_t* src_argb,
+ uint8_t* dst_y,
+ int width);
+void ARGBToYJRow_NEON_DotProd(const uint8_t* src_argb,
+ uint8_t* dst_yj,
+ int width);
+void ABGRToYJRow_NEON_DotProd(const uint8_t* src_abgr,
+ uint8_t* dst_yj,
+ int width);
+void RGBAToYJRow_NEON_DotProd(const uint8_t* src_rgba,
+ uint8_t* dst_yj,
+ int width);
void ARGBToYRow_RVV(const uint8_t* src_argb, uint8_t* dst_y, int width);
void ARGBToYJRow_RVV(const uint8_t* src_argb, uint8_t* dst_yj, int width);
void ABGRToYJRow_RVV(const uint8_t* src_rgba, uint8_t* dst_yj, int width);
@@ -1427,11 +1895,28 @@
uint8_t* dst_u,
uint8_t* dst_v,
int width);
+void ARGBToUV444Row_NEON_I8MM(const uint8_t* src_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
+void ARGBToUVJ444Row_NEON(const uint8_t* src_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
+void ARGBToUVJ444Row_NEON_I8MM(const uint8_t* src_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
void ARGBToUVRow_NEON(const uint8_t* src_argb,
int src_stride_argb,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
+void ARGBToUVRow_SVE2(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
void ARGBToUV444Row_MSA(const uint8_t* src_argb,
uint8_t* dst_u,
uint8_t* dst_v,
@@ -1464,26 +1949,51 @@
uint8_t* dst_u,
uint8_t* dst_v,
int width);
+void ARGBToUVJRow_SVE2(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
void ABGRToUVJRow_NEON(const uint8_t* src_abgr,
int src_stride_abgr,
uint8_t* dst_uj,
uint8_t* dst_vj,
int width);
+void ABGRToUVJRow_SVE2(const uint8_t* src_abgr,
+ int src_stride_abgr,
+ uint8_t* dst_uj,
+ uint8_t* dst_vj,
+ int width);
void BGRAToUVRow_NEON(const uint8_t* src_bgra,
int src_stride_bgra,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
+void BGRAToUVRow_SVE2(const uint8_t* src_bgra,
+ int src_stride_bgra,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
void ABGRToUVRow_NEON(const uint8_t* src_abgr,
int src_stride_abgr,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
+void ABGRToUVRow_SVE2(const uint8_t* src_abgr,
+ int src_stride_abgr,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
void RGBAToUVRow_NEON(const uint8_t* src_rgba,
int src_stride_rgba,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
+void RGBAToUVRow_SVE2(const uint8_t* src_rgba,
+ int src_stride_rgba,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
void RGB24ToUVRow_NEON(const uint8_t* src_rgb24,
int src_stride_rgb24,
uint8_t* dst_u,
@@ -1632,6 +2142,15 @@
void BGRAToYRow_NEON(const uint8_t* src_bgra, uint8_t* dst_y, int width);
void ABGRToYRow_NEON(const uint8_t* src_abgr, uint8_t* dst_y, int width);
void RGBAToYRow_NEON(const uint8_t* src_rgba, uint8_t* dst_y, int width);
+void BGRAToYRow_NEON_DotProd(const uint8_t* src_bgra,
+ uint8_t* dst_y,
+ int width);
+void ABGRToYRow_NEON_DotProd(const uint8_t* src_abgr,
+ uint8_t* dst_y,
+ int width);
+void RGBAToYRow_NEON_DotProd(const uint8_t* src_rgba,
+ uint8_t* dst_y,
+ int width);
void RGB24ToYRow_NEON(const uint8_t* src_rgb24, uint8_t* dst_y, int width);
void RGB24ToYJRow_NEON(const uint8_t* src_rgb24, uint8_t* dst_yj, int width);
void RAWToYRow_NEON(const uint8_t* src_raw, uint8_t* dst_y, int width);
@@ -1709,12 +2228,33 @@
void RGB24ToYJRow_Any_AVX2(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
void RAWToYJRow_Any_AVX2(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
void ARGBToYRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
+void ARGBToYRow_Any_NEON_DotProd(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
void ARGBToYJRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
+void ARGBToYJRow_Any_NEON_DotProd(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
void ABGRToYJRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
+void ABGRToYJRow_Any_NEON_DotProd(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
void RGBAToYJRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
+void RGBAToYJRow_Any_NEON_DotProd(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
void BGRAToYRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
+void BGRAToYRow_Any_NEON_DotProd(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
void ABGRToYRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
+void ABGRToYRow_Any_NEON_DotProd(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
void RGBAToYRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
+void RGBAToYRow_Any_NEON_DotProd(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
void RGB24ToYRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
void RGB24ToYJRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
void RAWToYRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
@@ -1874,11 +2414,28 @@
uint8_t* dst_u,
uint8_t* dst_v,
int width);
+void ARGBToUV444Row_Any_NEON_I8MM(const uint8_t* src_ptr,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
+void ARGBToUVJ444Row_Any_NEON(const uint8_t* src_ptr,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
+void ARGBToUVJ444Row_Any_NEON_I8MM(const uint8_t* src_ptr,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
void ARGBToUVRow_Any_NEON(const uint8_t* src_ptr,
int src_stride,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
+void ARGBToUVRow_Any_SVE2(const uint8_t* src_ptr,
+ int src_stride,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
void ARGBToUV444Row_Any_MSA(const uint8_t* src_ptr,
uint8_t* dst_u,
uint8_t* dst_v,
@@ -1911,26 +2468,51 @@
uint8_t* dst_u,
uint8_t* dst_v,
int width);
+void ARGBToUVJRow_Any_SVE2(const uint8_t* src_ptr,
+ int src_stride,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
void ABGRToUVJRow_Any_NEON(const uint8_t* src_ptr,
int src_stride,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
+void ABGRToUVJRow_Any_SVE2(const uint8_t* src_ptr,
+ int src_stride,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
void BGRAToUVRow_Any_NEON(const uint8_t* src_ptr,
int src_stride,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
+void BGRAToUVRow_Any_SVE2(const uint8_t* src_ptr,
+ int src_stride,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
void ABGRToUVRow_Any_NEON(const uint8_t* src_ptr,
int src_stride,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
+void ABGRToUVRow_Any_SVE2(const uint8_t* src_ptr,
+ int src_stride,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
void RGBAToUVRow_Any_NEON(const uint8_t* src_ptr,
int src_stride,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
+void RGBAToUVRow_Any_SVE2(const uint8_t* src_ptr,
+ int src_stride,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
void RGB24ToUVRow_Any_NEON(const uint8_t* src_ptr,
int src_stride,
uint8_t* dst_u,
@@ -2156,11 +2738,43 @@
uint8_t* dst_v,
int width);
+void ARGBToUVJ444Row_SSSE3(const uint8_t* src_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
+void ARGBToUVJ444Row_Any_SSSE3(const uint8_t* src_ptr,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
+
+void ARGBToUV444Row_AVX2(const uint8_t* src_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
+void ARGBToUV444Row_Any_AVX2(const uint8_t* src_ptr,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
+
+void ARGBToUVJ444Row_AVX2(const uint8_t* src_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
+void ARGBToUVJ444Row_Any_AVX2(const uint8_t* src_ptr,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
+
void ARGBToUV444Row_C(const uint8_t* src_argb,
uint8_t* dst_u,
uint8_t* dst_v,
int width);
+void ARGBToUVJ444Row_C(const uint8_t* src_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width);
+
void MirrorRow_AVX2(const uint8_t* src, uint8_t* dst, int width);
void MirrorRow_SSSE3(const uint8_t* src, uint8_t* dst, int width);
void MirrorRow_NEON(const uint8_t* src, uint8_t* dst, int width);
@@ -2429,6 +3043,10 @@
const uint8_t* src_v,
uint8_t* dst_uv,
int width);
+void MergeUVRow_SME(const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_uv,
+ int width);
void MergeUVRow_MSA(const uint8_t* src_u,
const uint8_t* src_v,
uint8_t* dst_uv,
@@ -2504,6 +3122,16 @@
uint8_t* dst_g,
uint8_t* dst_b,
int width);
+void SplitRGBRow_SSE41(const uint8_t* src_rgb,
+ uint8_t* dst_r,
+ uint8_t* dst_g,
+ uint8_t* dst_b,
+ int width);
+void SplitRGBRow_AVX2(const uint8_t* src_rgb,
+ uint8_t* dst_r,
+ uint8_t* dst_g,
+ uint8_t* dst_b,
+ int width);
void SplitRGBRow_NEON(const uint8_t* src_rgb,
uint8_t* dst_r,
uint8_t* dst_g,
@@ -2519,6 +3147,16 @@
uint8_t* dst_g,
uint8_t* dst_b,
int width);
+void SplitRGBRow_Any_SSE41(const uint8_t* src_ptr,
+ uint8_t* dst_r,
+ uint8_t* dst_g,
+ uint8_t* dst_b,
+ int width);
+void SplitRGBRow_Any_AVX2(const uint8_t* src_ptr,
+ uint8_t* dst_r,
+ uint8_t* dst_g,
+ uint8_t* dst_b,
+ int width);
void SplitRGBRow_Any_NEON(const uint8_t* src_ptr,
uint8_t* dst_r,
uint8_t* dst_g,
@@ -2952,6 +3590,11 @@
uint16_t* dst_uv,
int depth,
int width);
+void MergeUVRow_16_SME(const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint16_t* dst_uv,
+ int depth,
+ int width);
void SplitUVRow_16_C(const uint16_t* src_uv,
uint16_t* dst_u,
@@ -2999,6 +3642,10 @@
uint16_t* dst_ptr,
int scale,
int width);
+void MultiplyRow_16_SME(const uint16_t* src_y,
+ uint16_t* dst_y,
+ int scale,
+ int width);
void DivideRow_16_C(const uint16_t* src_y,
uint16_t* dst_y,
@@ -3016,6 +3663,10 @@
uint16_t* dst_y,
int scale,
int width);
+void DivideRow_16_SVE2(const uint16_t* src_y,
+ uint16_t* dst_y,
+ int scale,
+ int width);
void DivideRow_16_Any_NEON(const uint16_t* src_ptr,
uint16_t* dst_ptr,
int scale,
@@ -3054,6 +3705,10 @@
uint8_t* dst_y,
int scale,
int width);
+void Convert16To8Row_AVX512BW(const uint16_t* src_y,
+ uint8_t* dst_y,
+ int scale,
+ int width);
void Convert16To8Row_Any_SSSE3(const uint16_t* src_ptr,
uint8_t* dst_ptr,
int scale,
@@ -3062,6 +3717,10 @@
uint8_t* dst_ptr,
int scale,
int width);
+void Convert16To8Row_Any_AVX512BW(const uint16_t* src_ptr,
+ uint8_t* dst_ptr,
+ int scale,
+ int width);
void Convert16To8Row_NEON(const uint16_t* src_y,
uint8_t* dst_y,
int scale,
@@ -3070,16 +3729,59 @@
uint8_t* dst_ptr,
int scale,
int width);
+void Convert16To8Row_SME(const uint16_t* src_y,
+ uint8_t* dst_y,
+ int scale,
+ int width);
+
+void Convert8To8Row_C(const uint8_t* src_y,
+ uint8_t* dst_y,
+ int scale,
+ int bias,
+ int width);
+void Convert8To8Row_NEON(const uint8_t* src_y,
+ uint8_t* dst_y,
+ int scale,
+ int bias,
+ int width);
+void Convert8To8Row_Any_NEON(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int scale,
+ int bias,
+ int width);
+void Convert8To8Row_SVE2(const uint8_t* src_y,
+ uint8_t* dst_y,
+ int scale,
+ int bias,
+ int width);
+void Convert8To8Row_SME(const uint8_t* src_y,
+ uint8_t* dst_y,
+ int scale,
+ int bias,
+ int width);
+void Convert8To8Row_AVX2(const uint8_t* src_y,
+ uint8_t* dst_y,
+ int scale,
+ int bias,
+ int width);
+void Convert8To8Row_Any_AVX2(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int scale,
+ int bias,
+ int width);
void CopyRow_SSE2(const uint8_t* src, uint8_t* dst, int width);
void CopyRow_AVX(const uint8_t* src, uint8_t* dst, int width);
+void CopyRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width);
void CopyRow_ERMS(const uint8_t* src, uint8_t* dst, int width);
void CopyRow_NEON(const uint8_t* src, uint8_t* dst, int width);
+void CopyRow_SME(const uint8_t* src, uint8_t* dst, int width);
void CopyRow_MIPS(const uint8_t* src, uint8_t* dst, int count);
void CopyRow_RVV(const uint8_t* src, uint8_t* dst, int count);
void CopyRow_C(const uint8_t* src, uint8_t* dst, int count);
void CopyRow_Any_SSE2(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
void CopyRow_Any_AVX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
+void CopyRow_Any_AVX512BW(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
void CopyRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
void CopyRow_16_C(const uint16_t* src, uint16_t* dst, int count);
@@ -3217,6 +3919,7 @@
uint8_t* dst_argb,
int width);
void RAWToARGBRow_SSSE3(const uint8_t* src_raw, uint8_t* dst_argb, int width);
+void RAWToARGBRow_AVX2(const uint8_t* src_raw, uint8_t* dst_argb, int width);
void RAWToRGBARow_SSSE3(const uint8_t* src_raw, uint8_t* dst_rgba, int width);
void RAWToRGB24Row_SSSE3(const uint8_t* src_raw, uint8_t* dst_rgb24, int width);
void RGB565ToARGBRow_SSE2(const uint8_t* src, uint8_t* dst, int width);
@@ -3235,6 +3938,9 @@
void RGB24ToARGBRow_NEON(const uint8_t* src_rgb24,
uint8_t* dst_argb,
int width);
+void RGB24ToARGBRow_SVE2(const uint8_t* src_rgb24,
+ uint8_t* dst_argb,
+ int width);
void RGB24ToARGBRow_MSA(const uint8_t* src_rgb24, uint8_t* dst_argb, int width);
void RGB24ToARGBRow_LSX(const uint8_t* src_rgb24, uint8_t* dst_argb, int width);
void RGB24ToARGBRow_LASX(const uint8_t* src_rgb24,
@@ -3242,13 +3948,16 @@
int width);
void RGB24ToARGBRow_RVV(const uint8_t* src_rgb24, uint8_t* dst_argb, int width);
void RAWToARGBRow_NEON(const uint8_t* src_raw, uint8_t* dst_argb, int width);
+void RAWToARGBRow_SVE2(const uint8_t* src_raw, uint8_t* dst_argb, int width);
void RAWToRGBARow_NEON(const uint8_t* src_raw, uint8_t* dst_rgba, int width);
+void RAWToRGBARow_SVE2(const uint8_t* src_raw, uint8_t* dst_rgba, int width);
void RAWToARGBRow_MSA(const uint8_t* src_raw, uint8_t* dst_argb, int width);
void RAWToARGBRow_LSX(const uint8_t* src_raw, uint8_t* dst_argb, int width);
void RAWToARGBRow_LASX(const uint8_t* src_raw, uint8_t* dst_argb, int width);
void RAWToARGBRow_RVV(const uint8_t* src_raw, uint8_t* dst_argb, int width);
void RAWToRGBARow_RVV(const uint8_t* src_raw, uint8_t* dst_rgba, int width);
void RAWToRGB24Row_NEON(const uint8_t* src_raw, uint8_t* dst_rgb24, int width);
+void RAWToRGB24Row_SVE2(const uint8_t* src_raw, uint8_t* dst_rgb24, int width);
void RAWToRGB24Row_MSA(const uint8_t* src_raw, uint8_t* dst_rgb24, int width);
void RAWToRGB24Row_LSX(const uint8_t* src_raw, uint8_t* dst_rgb24, int width);
void RAWToRGB24Row_RVV(const uint8_t* src_raw, uint8_t* dst_rgb24, int width);
@@ -3267,6 +3976,9 @@
void ARGB1555ToARGBRow_NEON(const uint8_t* src_argb1555,
uint8_t* dst_argb,
int width);
+void ARGB1555ToARGBRow_SVE2(const uint8_t* src_argb1555,
+ uint8_t* dst_argb,
+ int width);
void ARGB1555ToARGBRow_MSA(const uint8_t* src_argb1555,
uint8_t* dst_argb,
int width);
@@ -3310,6 +4022,7 @@
void RAWToARGBRow_Any_SSSE3(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int width);
+void RAWToARGBRow_Any_AVX2(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
void RAWToRGBARow_Any_SSSE3(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int width);
@@ -3435,10 +4148,17 @@
void ARGBToRGB24Row_NEON(const uint8_t* src_argb,
uint8_t* dst_rgb24,
int width);
+void ARGBToRGB24Row_SVE2(const uint8_t* src_argb,
+ uint8_t* dst_rgb24,
+ int width);
void ARGBToRAWRow_NEON(const uint8_t* src_argb, uint8_t* dst_raw, int width);
+void ARGBToRAWRow_SVE2(const uint8_t* src_argb, uint8_t* dst_raw, int width);
void ARGBToRGB565Row_NEON(const uint8_t* src_argb,
uint8_t* dst_rgb565,
int width);
+void ARGBToRGB565Row_SVE2(const uint8_t* src_argb,
+ uint8_t* dst_rgb565,
+ int width);
void ARGBToARGB1555Row_NEON(const uint8_t* src_argb,
uint8_t* dst_argb1555,
int width);
@@ -3449,6 +4169,10 @@
uint8_t* dst_rgb,
uint32_t dither4,
int width);
+void ARGBToRGB565DitherRow_SVE2(const uint8_t* src_argb,
+ uint8_t* dst_rgb,
+ uint32_t dither4,
+ int width);
void ARGBToRGB24Row_MSA(const uint8_t* src_argb, uint8_t* dst_rgb, int width);
void ARGBToRAWRow_MSA(const uint8_t* src_argb, uint8_t* dst_rgb, int width);
void ARGBToRGB565Row_MSA(const uint8_t* src_argb, uint8_t* dst_rgb, int width);
@@ -4507,6 +5231,14 @@
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width);
+void I400ToARGBRow_SVE2(const uint8_t* src_y,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I400ToARGBRow_SME(const uint8_t* src_y,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
void I400ToARGBRow_MSA(const uint8_t* src_y,
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
@@ -4628,6 +5360,10 @@
const uint8_t* uv_buf,
uint8_t* dst_ptr,
int width);
+void ARGBMultiplyRow_SME(const uint8_t* src_argb,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width);
void ARGBMultiplyRow_MSA(const uint8_t* src_argb0,
const uint8_t* src_argb1,
uint8_t* dst_argb,
@@ -4902,6 +5638,42 @@
uint8_t* dst_ptr,
const struct YuvConstants* yuvconstants,
int width);
+void I210ToARGBRow_Any_NEON(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I410ToARGBRow_Any_NEON(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I210ToAR30Row_Any_NEON(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I410ToAR30Row_Any_NEON(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I212ToARGBRow_Any_NEON(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I212ToAR30Row_Any_NEON(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
void I444AlphaToARGBRow_Any_NEON(const uint8_t* y_buf,
const uint8_t* u_buf,
const uint8_t* v_buf,
@@ -4916,6 +5688,20 @@
uint8_t* dst_ptr,
const struct YuvConstants* yuvconstants,
int width);
+void I410AlphaToARGBRow_Any_NEON(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ const uint16_t* a_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void I210AlphaToARGBRow_Any_NEON(const uint16_t* y_buf,
+ const uint16_t* u_buf,
+ const uint16_t* v_buf,
+ const uint16_t* a_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
void I422ToRGBARow_Any_NEON(const uint8_t* y_buf,
const uint8_t* u_buf,
const uint8_t* v_buf,
@@ -4946,6 +5732,12 @@
uint8_t* dst_ptr,
const struct YuvConstants* yuvconstants,
int width);
+void I422ToAR30Row_Any_NEON(const uint8_t* y_buf,
+ const uint8_t* u_buf,
+ const uint8_t* v_buf,
+ uint8_t* dst_ptr,
+ const struct YuvConstants* yuvconstants,
+ int width);
void NV12ToARGBRow_Any_NEON(const uint8_t* y_buf,
const uint8_t* uv_buf,
uint8_t* dst_ptr,
@@ -4983,26 +5775,74 @@
uint8_t* dst_ptr,
const struct YuvConstants* yuvconstants,
int width);
+void ARGBToAR30Row_NEON(const uint8_t* src, uint8_t* dst, int width);
+void ABGRToAR30Row_NEON(const uint8_t* src, uint8_t* dst, int width);
+void ABGRToAR30Row_Any_NEON(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
+void ARGBToAR30Row_Any_NEON(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int width);
void P210ToARGBRow_NEON(const uint16_t* y_buf,
const uint16_t* uv_buf,
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width);
+void P210ToARGBRow_SVE2(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void P210ToARGBRow_SME(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
void P410ToARGBRow_NEON(const uint16_t* y_buf,
const uint16_t* uv_buf,
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width);
+void P410ToARGBRow_SVE2(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void P410ToARGBRow_SME(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width);
void P210ToAR30Row_NEON(const uint16_t* y_buf,
const uint16_t* uv_buf,
uint8_t* dst_ar30,
const struct YuvConstants* yuvconstants,
int width);
+void P210ToAR30Row_SVE2(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void P210ToAR30Row_SME(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width);
void P410ToAR30Row_NEON(const uint16_t* y_buf,
const uint16_t* uv_buf,
uint8_t* dst_ar30,
const struct YuvConstants* yuvconstants,
int width);
+void P410ToAR30Row_SVE2(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width);
+void P410ToAR30Row_SME(const uint16_t* y_buf,
+ const uint16_t* uv_buf,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width);
void P210ToARGBRow_Any_NEON(const uint16_t* y_buf,
const uint16_t* uv_buf,
uint8_t* dst_argb,
@@ -5547,19 +6387,35 @@
int src_stride_ayuv,
uint8_t* dst_uv,
int width);
+void AYUVToUVRow_SVE2(const uint8_t* src_ayuv,
+ int src_stride_ayuv,
+ uint8_t* dst_uv,
+ int width);
void AYUVToVURow_NEON(const uint8_t* src_ayuv,
int src_stride_ayuv,
uint8_t* dst_vu,
int width);
+void AYUVToVURow_SVE2(const uint8_t* src_ayuv,
+ int src_stride_ayuv,
+ uint8_t* dst_vu,
+ int width);
void AYUVToYRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width);
void AYUVToUVRow_Any_NEON(const uint8_t* src_ptr,
int src_stride,
uint8_t* dst_vu,
int width);
+void AYUVToUVRow_Any_SVE2(const uint8_t* src_ptr,
+ int src_stride,
+ uint8_t* dst_vu,
+ int width);
void AYUVToVURow_Any_NEON(const uint8_t* src_ptr,
int src_stride,
uint8_t* dst_vu,
int width);
+void AYUVToVURow_Any_SVE2(const uint8_t* src_ptr,
+ int src_stride,
+ uint8_t* dst_vu,
+ int width);
void I422ToYUY2Row_C(const uint8_t* src_y,
const uint8_t* src_u,
@@ -5755,6 +6611,9 @@
void ARGBGrayRow_C(const uint8_t* src_argb, uint8_t* dst_argb, int width);
void ARGBGrayRow_SSSE3(const uint8_t* src_argb, uint8_t* dst_argb, int width);
void ARGBGrayRow_NEON(const uint8_t* src_argb, uint8_t* dst_argb, int width);
+void ARGBGrayRow_NEON_DotProd(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ int width);
void ARGBGrayRow_MSA(const uint8_t* src_argb, uint8_t* dst_argb, int width);
void ARGBGrayRow_LSX(const uint8_t* src_argb, uint8_t* dst_argb, int width);
void ARGBGrayRow_LASX(const uint8_t* src_argb, uint8_t* dst_argb, int width);
@@ -5762,6 +6621,7 @@
void ARGBSepiaRow_C(uint8_t* dst_argb, int width);
void ARGBSepiaRow_SSSE3(uint8_t* dst_argb, int width);
void ARGBSepiaRow_NEON(uint8_t* dst_argb, int width);
+void ARGBSepiaRow_NEON_DotProd(uint8_t* dst_argb, int width);
void ARGBSepiaRow_MSA(uint8_t* dst_argb, int width);
void ARGBSepiaRow_LSX(uint8_t* dst_argb, int width);
void ARGBSepiaRow_LASX(uint8_t* dst_argb, int width);
@@ -5778,6 +6638,10 @@
uint8_t* dst_argb,
const int8_t* matrix_argb,
int width);
+void ARGBColorMatrixRow_NEON_I8MM(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ const int8_t* matrix_argb,
+ int width);
void ARGBColorMatrixRow_MSA(const uint8_t* src_argb,
uint8_t* dst_argb,
const int8_t* matrix_argb,
@@ -5909,6 +6773,11 @@
ptrdiff_t src_stride,
int dst_width,
int source_y_fraction);
+void InterpolateRow_SME(uint8_t* dst_ptr,
+ const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ int dst_width,
+ int source_y_fraction);
void InterpolateRow_MSA(uint8_t* dst_ptr,
const uint8_t* src_ptr,
ptrdiff_t src_stride,
@@ -5965,6 +6834,11 @@
ptrdiff_t src_stride,
int width,
int source_y_fraction);
+void InterpolateRow_16_SME(uint16_t* dst_ptr,
+ const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ int width,
+ int source_y_fraction);
void InterpolateRow_16To8_C(uint8_t* dst_ptr,
const uint16_t* src_ptr,
@@ -5984,6 +6858,12 @@
int scale,
int width,
int source_y_fraction);
+void InterpolateRow_16To8_SME(uint8_t* dst_ptr,
+ const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ int scale,
+ int width,
+ int source_y_fraction);
void InterpolateRow_16To8_AVX2(uint8_t* dst_ptr,
const uint16_t* src_ptr,
ptrdiff_t src_stride,
@@ -6198,14 +7078,14 @@
uint16_t* dst_ptr,
float param,
int width);
-void HalfFloat1Row_NEON(const uint16_t* src,
+void HalfFloatRow_SVE2(const uint16_t* src,
+ uint16_t* dst,
+ float scale,
+ int width);
+void HalfFloat1Row_SVE2(const uint16_t* src,
uint16_t* dst,
float scale,
int width);
-void HalfFloat1Row_Any_NEON(const uint16_t* src_ptr,
- uint16_t* dst_ptr,
- float param,
- int width);
void HalfFloatRow_MSA(const uint16_t* src,
uint16_t* dst,
float scale,
@@ -6288,6 +7168,7 @@
int width);
void GaussRow_C(const uint32_t* src, uint16_t* dst, int width);
+void GaussRow_NEON(const uint32_t* src, uint16_t* dst, int width);
void GaussCol_C(const uint16_t* src0,
const uint16_t* src1,
const uint16_t* src2,
@@ -6295,6 +7176,13 @@
const uint16_t* src4,
uint32_t* dst,
int width);
+void GaussCol_NEON(const uint16_t* src0,
+ const uint16_t* src1,
+ const uint16_t* src2,
+ const uint16_t* src3,
+ const uint16_t* src4,
+ uint32_t* dst,
+ int width);
void ClampFloatToZero_SSE2(const float* src_x, float* dst_y, int width);
diff --git a/include/libyuv/row_sve.h b/include/libyuv/row_sve.h
new file mode 100644
index 0000000..14a4980
--- /dev/null
+++ b/include/libyuv/row_sve.h
@@ -0,0 +1,1789 @@
+/*
+ * Copyright 2024 The LibYuv Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef INCLUDE_LIBYUV_ROW_SVE_H_
+#define INCLUDE_LIBYUV_ROW_SVE_H_
+
+#include "libyuv/row.h"
+
+#ifdef __cplusplus
+namespace libyuv {
+extern "C" {
+#endif
+
+#if !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__)
+
+#if !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && \
+ defined(__aarch64__)
+#define STREAMING_COMPATIBLE __arm_streaming_compatible
+#else // defined(LIBYUV_DISABLE_SME) || !defined(CLANG_HAS_SME) ||
+ // !defined(__aarch64__)
+#define STREAMING_COMPATIBLE
+#endif // !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) &&
+ // defined(__aarch64__)
+
+#define YUVTORGB_SVE_SETUP \
+ "ld1rb {z28.b}, p0/z, [%[kUVCoeff], #0] \n" \
+ "ld1rb {z29.b}, p0/z, [%[kUVCoeff], #1] \n" \
+ "ld1rb {z30.b}, p0/z, [%[kUVCoeff], #2] \n" \
+ "ld1rb {z31.b}, p0/z, [%[kUVCoeff], #3] \n" \
+ "ld1rh {z24.h}, p0/z, [%[kRGBCoeffBias], #0] \n" \
+ "ld1rh {z25.h}, p0/z, [%[kRGBCoeffBias], #2] \n" \
+ "ld1rh {z26.h}, p0/z, [%[kRGBCoeffBias], #4] \n" \
+ "ld1rh {z27.h}, p0/z, [%[kRGBCoeffBias], #6] \n"
+
+#define READYUV444_SVE \
+ "ld1b {z0.h}, p1/z, [%[src_y]] \n" \
+ "ld1b {z1.h}, p1/z, [%[src_u]] \n" \
+ "ld1b {z2.h}, p1/z, [%[src_v]] \n" \
+ "add %[src_y], %[src_y], %[vl] \n" \
+ "add %[src_u], %[src_u], %[vl] \n" \
+ "add %[src_v], %[src_v], %[vl] \n" \
+ "prfm pldl1keep, [%[src_y], 448] \n" \
+ "prfm pldl1keep, [%[src_u], 448] \n" \
+ "trn1 z0.b, z0.b, z0.b \n" \
+ "prfm pldl1keep, [%[src_v], 448] \n"
+
+#define READYUV400_SVE \
+ "ld1b {z0.h}, p1/z, [%[src_y]] \n" \
+ "inch %[src_y] \n" \
+ "prfm pldl1keep, [%[src_y], 448] \n" \
+ "trn1 z0.b, z0.b, z0.b \n"
+
+#define READYUV422_SVE \
+ "ld1b {z0.h}, p1/z, [%[src_y]] \n" \
+ "ld1b {z1.s}, p1/z, [%[src_u]] \n" \
+ "ld1b {z2.s}, p1/z, [%[src_v]] \n" \
+ "inch %[src_y] \n" \
+ "incw %[src_u] \n" \
+ "incw %[src_v] \n" \
+ "prfm pldl1keep, [%[src_y], 448] \n" \
+ "prfm pldl1keep, [%[src_u], 128] \n" \
+ "prfm pldl1keep, [%[src_v], 128] \n" \
+ "trn1 z0.b, z0.b, z0.b \n" \
+ "trn1 z1.h, z1.h, z1.h \n" \
+ "trn1 z2.h, z2.h, z2.h \n"
+
+// Read twice as much data from YUV, putting the even elements from the Y data
+// in z0.h and odd elements in z1.h. U/V data is not duplicated, stored in
+// z2.h/z3.h.
+#define READYUV422_SVE_2X \
+ "ld1b {z0.b}, p1/z, [%[src_y]] \n" \
+ "ld1b {z2.h}, p1/z, [%[src_u]] \n" \
+ "ld1b {z3.h}, p1/z, [%[src_v]] \n" \
+ "incb %[src_y] \n" \
+ "inch %[src_u] \n" \
+ "inch %[src_v] \n" \
+ "prfm pldl1keep, [%[src_y], 448] \n" \
+ "prfm pldl1keep, [%[src_u], 128] \n" \
+ "prfm pldl1keep, [%[src_v], 128] \n" \
+ "trn2 z1.b, z0.b, z0.b \n" \
+ "trn1 z0.b, z0.b, z0.b \n"
+
+#define READI210_SVE \
+ "ld1h {z3.h}, p1/z, [%[src_y]] \n" \
+ "ld1h {z1.s}, p1/z, [%[src_u]] \n" \
+ "ld1h {z2.s}, p1/z, [%[src_v]] \n" \
+ "incb %[src_y] \n" \
+ "inch %[src_u] \n" \
+ "inch %[src_v] \n" \
+ "lsl z0.h, z3.h, #6 \n" \
+ "trn1 z1.h, z1.h, z1.h \n" \
+ "trn1 z2.h, z2.h, z2.h \n" \
+ "prfm pldl1keep, [%[src_y], 448] \n" \
+ "prfm pldl1keep, [%[src_u], 128] \n" \
+ "prfm pldl1keep, [%[src_v], 128] \n" \
+ "usra z0.h, z3.h, #4 \n" \
+ "uqshrnb z1.b, z1.h, #2 \n" \
+ "uqshrnb z2.b, z2.h, #2 \n"
+
+#define READP210_SVE \
+ "ld1h {z0.h}, p1/z, [%[src_y]] \n" \
+ "ld1h {z1.h}, p2/z, [%[src_uv]] \n" \
+ "incb %[src_y] \n" \
+ "incb %[src_uv] \n" \
+ "prfm pldl1keep, [%[src_y], 448] \n" \
+ "prfm pldl1keep, [%[src_uv], 256] \n" \
+ "tbl z1.b, {z1.b}, z22.b \n"
+
+#define READI410_SVE \
+ "ld1h {z3.h}, p1/z, [%[src_y]] \n" \
+ "lsl z0.h, z3.h, #6 \n" \
+ "usra z0.h, z3.h, #4 \n" \
+ "ld1h {z1.h}, p1/z, [%[src_u]] \n" \
+ "ld1h {z2.h}, p1/z, [%[src_v]] \n" \
+ "incb %[src_y] \n" \
+ "incb %[src_u] \n" \
+ "incb %[src_v] \n" \
+ "prfm pldl1keep, [%[src_y], 448] \n" \
+ "prfm pldl1keep, [%[src_u], 128] \n" \
+ "prfm pldl1keep, [%[src_v], 128] \n" \
+ "uqshrnb z1.b, z1.h, #2 \n" \
+ "uqshrnb z2.b, z2.h, #2 \n"
+
+// We need different predicates for the UV components since we are reading
+// 32-bit (pairs of UV) elements rather than 16-bit Y elements.
+#define READP410_SVE \
+ "ld1h {z0.h}, p1/z, [%[src_y]] \n" \
+ "ld1w {z1.s}, p2/z, [%[src_uv]] \n" \
+ "ld1w {z2.s}, p3/z, [%[src_uv], #1, mul vl] \n" \
+ "incb %[src_y] \n" \
+ "incb %[src_uv], all, mul #2 \n" \
+ "prfm pldl1keep, [%[src_y], 448] \n" \
+ "prfm pldl1keep, [%[src_uv], 256] \n" \
+ "uzp2 z1.b, z1.b, z2.b \n"
+
+#define READI212_SVE \
+ "ld1h {z3.h}, p1/z, [%[src_y]] \n" \
+ "ld1h {z1.s}, p1/z, [%[src_u]] \n" \
+ "ld1h {z2.s}, p1/z, [%[src_v]] \n" \
+ "incb %[src_y] \n" \
+ "inch %[src_u] \n" \
+ "inch %[src_v] \n" \
+ "lsl z0.h, z3.h, #4 \n" \
+ "trn1 z1.h, z1.h, z1.h \n" \
+ "trn1 z2.h, z2.h, z2.h \n" \
+ "prfm pldl1keep, [%[src_y], 448] \n" \
+ "prfm pldl1keep, [%[src_u], 128] \n" \
+ "prfm pldl1keep, [%[src_v], 128] \n" \
+ "usra z0.h, z3.h, #8 \n" \
+ "uqshrnb z1.b, z1.h, #4 \n" \
+ "uqshrnb z2.b, z2.h, #4 \n"
+
+#define I400TORGB_SVE \
+ "umulh z18.h, z24.h, z0.h \n" /* Y */ \
+ "movprfx z16, z18 \n" \
+ "usqadd z16.h, p0/m, z16.h, z4.h \n" /* B */ \
+ "movprfx z17, z18 \n" \
+ "usqadd z17.h, p0/m, z17.h, z6.h \n" /* G */ \
+ "usqadd z18.h, p0/m, z18.h, z5.h \n" /* R */
+
+// We need a different predicate for the UV component to handle the tail.
+// If there is a single element remaining then we want to load one Y element
+// but two UV elements.
+#define READNV_SVE_2X \
+ "ld1b {z0.b}, p1/z, [%[src_y]] \n" /* Y0Y0 */ \
+ "ld1b {z2.b}, p2/z, [%[src_uv]] \n" /* U0V0 or V0U0 */ \
+ "incb %[src_y] \n" \
+ "incb %[src_uv] \n" \
+ "prfm pldl1keep, [%[src_y], 448] \n" \
+ "prfm pldl1keep, [%[src_uv], 256] \n" \
+ "trn2 z1.b, z0.b, z0.b \n" /* YYYY */ \
+ "trn1 z0.b, z0.b, z0.b \n" /* YYYY */
+
+// Like NVTORGB_SVE but U/V components are stored in widened .h elements of
+// z1/z2 rather than even/odd .b lanes of z1.
+#define I4XXTORGB_SVE \
+ "umulh z0.h, z24.h, z0.h \n" /* Y */ \
+ "umullb z6.h, z30.b, z1.b \n" \
+ "umullb z4.h, z28.b, z1.b \n" /* DB */ \
+ "umullb z5.h, z29.b, z2.b \n" /* DR */ \
+ "umlalb z6.h, z31.b, z2.b \n" /* DG */ \
+ "add z17.h, z0.h, z26.h \n" /* G */ \
+ "add z16.h, z0.h, z4.h \n" /* B */ \
+ "add z18.h, z0.h, z5.h \n" /* R */ \
+ "uqsub z17.h, z17.h, z6.h \n" /* G */ \
+ "uqsub z16.h, z16.h, z25.h \n" /* B */ \
+ "uqsub z18.h, z18.h, z27.h \n" /* R */
+
+// Like I4XXTORGB_SVE but U/V components are stored in even/odd .b lanes of z1
+// rather than widened .h elements of z1/z2.
+#define NVTORGB_SVE \
+ "umulh z0.h, z24.h, z0.h \n" /* Y */ \
+ "umullb z6.h, z30.b, z1.b \n" \
+ "umullb z4.h, z28.b, z1.b \n" /* DB */ \
+ "umullt z5.h, z29.b, z1.b \n" /* DR */ \
+ "umlalt z6.h, z31.b, z1.b \n" /* DG */ \
+ "add z17.h, z0.h, z26.h \n" /* G */ \
+ "add z16.h, z0.h, z4.h \n" /* B */ \
+ "add z18.h, z0.h, z5.h \n" /* R */ \
+ "uqsub z17.h, z17.h, z6.h \n" /* G */ \
+ "uqsub z16.h, z16.h, z25.h \n" /* B */ \
+ "uqsub z18.h, z18.h, z27.h \n" /* R */
+
+// The U/V component multiplies do not need to be duplicated in I422, we just
+// need to combine them with Y0/Y1 correctly.
+#define I422TORGB_SVE_2X \
+ "umulh z0.h, z24.h, z0.h \n" /* Y0 */ \
+ "umulh z1.h, z24.h, z1.h \n" /* Y1 */ \
+ "umullb z6.h, z30.b, z2.b \n" \
+ "umullb z4.h, z28.b, z2.b \n" /* DB */ \
+ "umullb z5.h, z29.b, z3.b \n" /* DR */ \
+ "umlalb z6.h, z31.b, z3.b \n" /* DG */ \
+ \
+ "add z17.h, z0.h, z26.h \n" /* G0 */ \
+ "add z21.h, z1.h, z26.h \n" /* G1 */ \
+ "add z16.h, z0.h, z4.h \n" /* B0 */ \
+ "add z20.h, z1.h, z4.h \n" /* B1 */ \
+ "add z18.h, z0.h, z5.h \n" /* R0 */ \
+ "add z22.h, z1.h, z5.h \n" /* R1 */ \
+ "uqsub z17.h, z17.h, z6.h \n" /* G0 */ \
+ "uqsub z21.h, z21.h, z6.h \n" /* G1 */ \
+ "uqsub z16.h, z16.h, z25.h \n" /* B0 */ \
+ "uqsub z20.h, z20.h, z25.h \n" /* B1 */ \
+ "uqsub z18.h, z18.h, z27.h \n" /* R0 */ \
+ "uqsub z22.h, z22.h, z27.h \n" /* R1 */
+
+// clang-format off
+#define NVTORGB_SVE_2X(bt_u, bt_v) \
+ "umulh z0.h, z24.h, z0.h \n" /* Y0 */ \
+ "umulh z1.h, z24.h, z1.h \n" /* Y1 */ \
+ "umull" #bt_u " z6.h, z30.b, z2.b \n" \
+ "umull" #bt_u " z4.h, z28.b, z2.b \n" /* DB */ \
+ "umull" #bt_v " z5.h, z29.b, z2.b \n" /* DR */ \
+ "umlal" #bt_v " z6.h, z31.b, z2.b \n" /* DG */ \
+ \
+ "add z17.h, z0.h, z26.h \n" /* G0 */ \
+ "add z21.h, z1.h, z26.h \n" /* G1 */ \
+ "add z16.h, z0.h, z4.h \n" /* B0 */ \
+ "add z20.h, z1.h, z4.h \n" /* B1 */ \
+ "add z18.h, z0.h, z5.h \n" /* R0 */ \
+ "add z22.h, z1.h, z5.h \n" /* R1 */ \
+ "uqsub z17.h, z17.h, z6.h \n" /* G0 */ \
+ "uqsub z21.h, z21.h, z6.h \n" /* G1 */ \
+ "uqsub z16.h, z16.h, z25.h \n" /* B0 */ \
+ "uqsub z20.h, z20.h, z25.h \n" /* B1 */ \
+ "uqsub z18.h, z18.h, z27.h \n" /* R0 */ \
+ "uqsub z22.h, z22.h, z27.h \n" /* R1 */
+// clang-format on
+
+#define RGBTOARGB8_SVE_TOP_2X \
+ /* Inputs: B: z16.h, G: z17.h, R: z18.h */ \
+ "uqshl z16.h, p0/m, z16.h, #2 \n" /* B0 */ \
+ "uqshl z17.h, p0/m, z17.h, #2 \n" /* G0 */ \
+ "uqshl z18.h, p0/m, z18.h, #2 \n" /* R0 */ \
+ "uqshl z20.h, p0/m, z20.h, #2 \n" /* B1 */ \
+ "uqshl z21.h, p0/m, z21.h, #2 \n" /* G1 */ \
+ "uqshl z22.h, p0/m, z22.h, #2 \n" /* R1 */
+
+// Convert from 2.14 fixed point RGB to 8 bit ARGB, interleaving as BG and RA
+// pairs to allow us to use ST2 for storing rather than ST4.
+#define RGBTOARGB8_SVE \
+ /* Inputs: B: z16.h, G: z17.h, R: z18.h, A: z19.b */ \
+ "uqshrnb z16.b, z16.h, #6 \n" /* B0 */ \
+ "uqshrnb z18.b, z18.h, #6 \n" /* R0 */ \
+ "uqshrnt z16.b, z17.h, #6 \n" /* BG */ \
+ "trn1 z17.b, z18.b, z19.b \n" /* RA */
+
+// Convert from 2.14 fixed point RGBA to 8 bit ARGB, interleaving as BG and RA
+// pairs to allow us to use ST2 for storing rather than ST4.
+#define RGBATOARGB8_SVE \
+ /* Inputs: B: z16.h, G: z17.h, R: z18.h, A: z19.h */ \
+ "uqshrnb z16.b, z16.h, #6 \n" /* B0 */ \
+ "uqshrnt z16.b, z17.h, #6 \n" /* BG */ \
+ "uqshrnb z17.b, z18.h, #6 \n" /* R0 */ \
+ "uqshrnt z17.b, z19.h, #2 \n" /* RA */
+
+// Convert from 2.14 fixed point RGB to 8 bit RGBA, interleaving as AB and GR
+// pairs to allow us to use ST2 for storing rather than ST4.
+#define RGBTORGBA8_SVE \
+ /* Inputs: B: z16.h, G: z17.h, R: z18.h, A: z19.b */ \
+ "uqshrnt z19.b, z16.h, #6 \n" /* AB */ \
+ "uqshrnb z20.b, z17.h, #6 \n" /* G0 */ \
+ "uqshrnt z20.b, z18.h, #6 \n" /* GR */
+
+#define RGBTOARGB8_SVE_2X \
+ /* Inputs: B: z16.h, G: z17.h, R: z18.h, A: z19.b */ \
+ "uqshrnb z16.b, z16.h, #6 \n" /* B0 */ \
+ "uqshrnb z17.b, z17.h, #6 \n" /* G0 */ \
+ "uqshrnb z18.b, z18.h, #6 \n" /* R0 */ \
+ "uqshrnt z16.b, z20.h, #6 \n" /* B1 */ \
+ "uqshrnt z17.b, z21.h, #6 \n" /* G1 */ \
+ "uqshrnt z18.b, z22.h, #6 \n" /* R1 */
+
+// Store AR30 elements. Inputs are 2.14 fixed point RGB. We expect z23 to be
+// populated with 0x3ff0 (0x3fff would also work) to saturate the R input
+// rather than needing a pair of shifts to saturate and then insert into the
+// correct position in the lane.
+#define STOREAR30_SVE \
+ "uqshl z16.h, p0/m, z16.h, #2 \n" /* bbbbbbbbbbxxxxxx */ \
+ "uqshl z17.h, p0/m, z17.h, #2 \n" /* ggggggggggxxxxxx */ \
+ "umin z18.h, p0/m, z18.h, z23.h \n" /* 00rrrrrrrrrrxxxx */ \
+ "orr z18.h, z18.h, #0xc000 \n" /* 11rrrrrrrrrrxxxx */ \
+ "sri z18.h, z17.h, #12 \n" /* 11rrrrrrrrrrgggg */ \
+ "lsl z17.h, z17.h, #4 \n" /* ggggggxxxxxx0000 */ \
+ "sri z17.h, z16.h, #6 \n" /* ggggggbbbbbbbbbb */ \
+ "st2h {z17.h, z18.h}, p1, [%[dst_ar30]] \n" \
+ "incb %[dst_ar30], all, mul #2 \n"
+
+#define YUVTORGB_SVE_REGS \
+ "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z16", "z17", "z18", "z19", \
+ "z20", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", \
+ "z31", "p0", "p1", "p2", "p3"
+
+static inline void I400ToARGBRow_SVE_SC(const uint8_t* src_y,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) STREAMING_COMPATIBLE {
+ uint64_t vl;
+ asm volatile(
+ "cnth %[vl] \n"
+ "ptrue p0.b \n"
+ "dup z19.b, #255 \n" // Alpha
+ YUVTORGB_SVE_SETUP
+ "cmp %w[width], %w[vl] \n"
+ "mov z1.h, #128 \n" // U/V
+ "umullb z6.h, z30.b, z1.b \n"
+ "umullb z4.h, z28.b, z1.b \n" // DB
+ "umullb z5.h, z29.b, z1.b \n" // DR
+ "mla z6.h, p0/m, z31.h, z1.h \n" // DG
+ "sub z4.h, z4.h, z25.h \n"
+ "sub z5.h, z5.h, z27.h \n"
+ "sub z6.h, z26.h, z6.h \n"
+ "b.le 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.h \n"
+ "sub %w[width], %w[width], %w[vl] \n"
+ "1: \n" //
+ READYUV400_SVE I400TORGB_SVE RGBTOARGB8_SVE
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
+ "add %[dst_argb], %[dst_argb], %[vl], lsl #2 \n"
+ "b.gt 1b \n"
+ "add %w[width], %w[width], %w[vl] \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "2: \n"
+ "whilelt p1.h, wzr, %w[width] \n" //
+ READYUV400_SVE I400TORGB_SVE RGBTOARGB8_SVE
+ "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_SVE_REGS);
+}
+
+static inline void I422ToARGBRow_SVE_SC(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) STREAMING_COMPATIBLE {
+ uint64_t vl;
+ asm volatile(
+ "cntb %[vl] \n"
+ "ptrue p0.b \n" //
+ YUVTORGB_SVE_SETUP
+ "dup z19.b, #255 \n" // Alpha
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.b \n"
+ "1: \n" //
+ READYUV422_SVE_2X I422TORGB_SVE_2X RGBTOARGB8_SVE_2X
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st4b {z16.b, z17.b, z18.b, z19.b}, p1, [%[dst_argb]] \n"
+ "incb %[dst_argb], all, mul #4 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "cnth %[vl] \n"
+ "whilelt p1.b, wzr, %w[width] \n" //
+ READYUV422_SVE_2X I422TORGB_SVE_2X RGBTOARGB8_SVE_2X
+ "st4b {z16.b, z17.b, z18.b, z19.b}, p1, [%[dst_argb]] \n"
+
+ "99: \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_SVE_REGS);
+}
+
+static inline void I422ToRGB24Row_SVE_SC(
+ const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) STREAMING_COMPATIBLE {
+ uint64_t vl;
+ asm volatile(
+ "cntb %[vl] \n"
+ "ptrue p0.b \n" //
+ YUVTORGB_SVE_SETUP
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.b \n"
+ "1: \n" //
+ READYUV422_SVE_2X I422TORGB_SVE_2X RGBTOARGB8_SVE_2X
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st3b {z16.b, z17.b, z18.b}, p1, [%[dst_argb]] \n"
+ "incb %[dst_argb], all, mul #3 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "cnth %[vl] \n"
+ "whilelt p1.b, wzr, %w[width] \n" //
+ READYUV422_SVE_2X I422TORGB_SVE_2X RGBTOARGB8_SVE_2X
+ "st3b {z16.b, z17.b, z18.b}, p1, [%[dst_argb]] \n"
+
+ "99: \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_SVE_REGS);
+}
+
+#define RGB8TORGB565_SVE_FROM_TOP_2X \
+ "sri z18.h, z17.h, #5 \n" /* rrrrrgggggg00000 */ \
+ "sri z22.h, z21.h, #5 \n" /* rrrrrgggggg00000 */ \
+ "sri z18.h, z16.h, #11 \n" /* rrrrrggggggbbbbb */ \
+ "sri z22.h, z20.h, #11 \n" /* rrrrrggggggbbbbb */ \
+ "mov z19.d, z22.d \n"
+
+static inline void I422ToRGB565Row_SVE_SC(
+ const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_rgb565,
+ const struct YuvConstants* yuvconstants,
+ int width) STREAMING_COMPATIBLE {
+ uint64_t vl;
+ asm volatile(
+ "cntb %[vl] \n"
+ "ptrue p0.b \n" //
+ YUVTORGB_SVE_SETUP
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.b \n"
+ "1: \n" //
+ READYUV422_SVE_2X I422TORGB_SVE_2X RGBTOARGB8_SVE_TOP_2X
+ "subs %w[width], %w[width], %w[vl] \n" //
+ RGB8TORGB565_SVE_FROM_TOP_2X
+ "st2h {z18.h, z19.h}, p1, [%[dst]] \n"
+ "incb %[dst], all, mul #2 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "cnth %[vl] \n"
+ "whilelt p1.b, wzr, %w[width] \n" //
+ READYUV422_SVE_2X I422TORGB_SVE_2X RGBTOARGB8_SVE_TOP_2X
+ RGB8TORGB565_SVE_FROM_TOP_2X
+ "st2h {z18.h, z19.h}, p1, [%[dst]] \n"
+
+ "99: \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst] "+r"(dst_rgb565), // %[dst]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_SVE_REGS);
+}
+
+#define RGB8TOARGB1555_SVE_FROM_TOP_2X \
+ "dup z0.h, #0x8000 \n" /* 1000000000000000 */ \
+ "dup z1.h, #0x8000 \n" /* 1000000000000000 */ \
+ "sri z0.h, z18.h, #1 \n" /* 1rrrrrxxxxxxxxxx */ \
+ "sri z1.h, z22.h, #1 \n" /* 1rrrrrxxxxxxxxxx */ \
+ "sri z0.h, z17.h, #6 \n" /* 1rrrrrgggggxxxxx */ \
+ "sri z1.h, z21.h, #6 \n" /* 1rrrrrgggggxxxxx */ \
+ "sri z0.h, z16.h, #11 \n" /* 1rrrrrgggggbbbbb */ \
+ "sri z1.h, z20.h, #11 \n" /* 1rrrrrgggggbbbbb */
+
+static inline void I422ToARGB1555Row_SVE_SC(
+ const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb1555,
+ const struct YuvConstants* yuvconstants,
+ int width) STREAMING_COMPATIBLE {
+ uint64_t vl;
+ asm volatile(
+ "cntb %[vl] \n"
+ "ptrue p0.b \n" //
+ YUVTORGB_SVE_SETUP
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.b \n"
+ "1: \n" //
+ READYUV422_SVE_2X I422TORGB_SVE_2X RGBTOARGB8_SVE_TOP_2X
+ "subs %w[width], %w[width], %w[vl] \n" //
+ RGB8TOARGB1555_SVE_FROM_TOP_2X
+ "st2h {z0.h, z1.h}, p1, [%[dst]] \n"
+ "incb %[dst], all, mul #2 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "cnth %[vl] \n"
+ "whilelt p1.b, wzr, %w[width] \n" //
+ READYUV422_SVE_2X I422TORGB_SVE_2X RGBTOARGB8_SVE_TOP_2X
+ RGB8TOARGB1555_SVE_FROM_TOP_2X
+ "st2h {z0.h, z1.h}, p1, [%[dst]] \n"
+
+ "99: \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst] "+r"(dst_argb1555), // %[dst]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_SVE_REGS);
+}
+
+#define RGB8TOARGB4444_SVE_FROM_TOP_2X \
+ "dup z0.h, #0xf000 \n" /* 1111000000000000 */ \
+ "dup z1.h, #0xf000 \n" /* 1111000000000000 */ \
+ "sri z0.h, z18.h, #4 \n" /* 1111rrrrxxxxxxxx */ \
+ "sri z1.h, z22.h, #4 \n" /* 1111rrrrxxxxxxxx */ \
+ "sri z0.h, z17.h, #8 \n" /* 1111rrrrggggxxxx */ \
+ "sri z1.h, z21.h, #8 \n" /* 1111rrrrggggxxxx */ \
+ "sri z0.h, z16.h, #12 \n" /* 1111rrrrggggbbbb */ \
+ "sri z1.h, z20.h, #12 \n" /* 1111rrrrggggbbbb */
+
+static inline void I422ToARGB4444Row_SVE_SC(
+ const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb4444,
+ const struct YuvConstants* yuvconstants,
+ int width) STREAMING_COMPATIBLE {
+ uint64_t vl;
+ asm volatile(
+ "cntb %[vl] \n"
+ "ptrue p0.b \n" //
+ YUVTORGB_SVE_SETUP
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.b \n"
+ "1: \n" //
+ READYUV422_SVE_2X I422TORGB_SVE_2X RGBTOARGB8_SVE_TOP_2X
+ "subs %w[width], %w[width], %w[vl] \n" //
+ RGB8TOARGB4444_SVE_FROM_TOP_2X
+ "st2h {z0.h, z1.h}, p1, [%[dst]] \n"
+ "incb %[dst], all, mul #2 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "cnth %[vl] \n"
+ "whilelt p1.b, wzr, %w[width] \n" //
+ READYUV422_SVE_2X I422TORGB_SVE_2X RGBTOARGB8_SVE_TOP_2X
+ RGB8TOARGB4444_SVE_FROM_TOP_2X
+ "st2h {z0.h, z1.h}, p1, [%[dst]] \n"
+
+ "99: \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst] "+r"(dst_argb4444), // %[dst]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_SVE_REGS);
+}
+
+static inline void I422ToRGBARow_SVE_SC(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) STREAMING_COMPATIBLE {
+ uint64_t vl;
+ asm volatile(
+ "cnth %[vl] \n"
+ "ptrue p0.b \n" //
+ YUVTORGB_SVE_SETUP
+ "dup z19.b, #255 \n" // Alpha
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.le 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.h \n"
+ "1: \n" //
+ READYUV422_SVE I4XXTORGB_SVE RGBTORGBA8_SVE
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st2h {z19.h, z20.h}, p1, [%[dst_argb]] \n"
+ "add %[dst_argb], %[dst_argb], %[vl], lsl #2 \n"
+ "b.gt 1b \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ "whilelt p1.h, wzr, %w[width] \n" //
+ READYUV422_SVE I4XXTORGB_SVE RGBTORGBA8_SVE
+ "st2h {z19.h, z20.h}, p1, [%[dst_argb]] \n"
+
+ "99: \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_SVE_REGS);
+}
+
+static inline void I422AlphaToARGBRow_SVE_SC(
+ const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ const uint8_t* src_a,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) STREAMING_COMPATIBLE {
+ uint64_t vl;
+ asm volatile(
+ "cntb %[vl] \n"
+ "ptrue p0.b \n" //
+ YUVTORGB_SVE_SETUP
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.b \n"
+ "1: \n" //
+ READYUV422_SVE_2X
+ "ld1b {z19.b}, p1/z, [%[src_a]] \n"
+ "add %[src_a], %[src_a], %[vl] \n" // Alpha
+ I422TORGB_SVE_2X RGBTOARGB8_SVE_2X
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st4b {z16.b, z17.b, z18.b, z19.b}, p1, [%[dst_argb]] \n"
+ "incb %[dst_argb], all, mul #4 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "cnth %[vl] \n"
+ "whilelt p1.b, wzr, %w[width] \n" //
+ READYUV422_SVE_2X
+ "ld1b {z19.b}, p1/z, [%[src_a]] \n" // Alpha
+ I422TORGB_SVE_2X RGBTOARGB8_SVE_2X
+ "st4b {z16.b, z17.b, z18.b, z19.b}, p1, [%[dst_argb]] \n"
+
+ "99: \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [src_a] "+r"(src_a), // %[src_a]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_SVE_REGS);
+}
+
+static inline void I444AlphaToARGBRow_SVE_SC(
+ const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ const uint8_t* src_a,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) STREAMING_COMPATIBLE {
+ uint64_t vl;
+ asm volatile(
+ "cnth %[vl] \n"
+ "ptrue p0.b \n" //
+ YUVTORGB_SVE_SETUP
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.h \n"
+ "1: \n" //
+ READYUV444_SVE
+ "ld1b {z19.h}, p1/z, [%[src_a]] \n"
+ "add %[src_a], %[src_a], %[vl] \n" // Alpha
+ I4XXTORGB_SVE RGBTOARGB8_SVE
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
+ "add %[dst_argb], %[dst_argb], %[vl], lsl #2 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p1.h, wzr, %w[width] \n" //
+ READYUV444_SVE
+ "ld1b {z19.h}, p1/z, [%[src_a]] \n" // Alpha
+ I4XXTORGB_SVE RGBTOARGB8_SVE
+ "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
+
+ "99: \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [src_a] "+r"(src_a), // %[src_a]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_SVE_REGS);
+}
+
+static inline void NV12ToARGBRow_SVE_SC(const uint8_t* src_y,
+ const uint8_t* src_uv,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) STREAMING_COMPATIBLE {
+ uint64_t vl;
+ asm("cntb %0" : "=r"(vl));
+ int width_last_y = width & (vl - 1);
+ int width_last_uv = width_last_y + (width_last_y & 1);
+ asm volatile(
+ "ptrue p0.b \n" //
+ YUVTORGB_SVE_SETUP
+ "dup z19.b, #255 \n" // Alpha
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.b \n"
+ "ptrue p2.b \n"
+ "1: \n" //
+ READNV_SVE_2X NVTORGB_SVE_2X(b, t) RGBTOARGB8_SVE_2X
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st4b {z16.b, z17.b, z18.b, z19.b}, p1, [%[dst_argb]] \n"
+ "add %[dst_argb], %[dst_argb], %[vl], lsl #2 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p1.b, wzr, %w[width_last_y] \n"
+ "whilelt p2.b, wzr, %w[width_last_uv] \n" //
+ READNV_SVE_2X NVTORGB_SVE_2X(b, t) RGBTOARGB8_SVE_2X
+ "st4b {z16.b, z17.b, z18.b, z19.b}, p1, [%[dst_argb]] \n"
+
+ "99: \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_uv] "+r"(src_uv), // %[src_uv]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width) // %[width]
+ : [vl] "r"(vl), // %[vl]
+ [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
+ [width_last_y] "r"(width_last_y), // %[width_last_y]
+ [width_last_uv] "r"(width_last_uv) // %[width_last_uv]
+ : "cc", "memory", YUVTORGB_SVE_REGS, "p2");
+}
+
+static inline void NV21ToARGBRow_SVE_SC(const uint8_t* src_y,
+ const uint8_t* src_vu,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) STREAMING_COMPATIBLE {
+ uint64_t vl;
+ asm("cntb %0" : "=r"(vl));
+ int width_last_y = width & (vl - 1);
+ int width_last_uv = width_last_y + (width_last_y & 1);
+ asm volatile(
+ "ptrue p0.b \n" //
+ YUVTORGB_SVE_SETUP
+ "dup z19.b, #255 \n" // Alpha
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.b \n"
+ "ptrue p2.b \n"
+ "1: \n" //
+ READNV_SVE_2X NVTORGB_SVE_2X(t, b) RGBTOARGB8_SVE_2X
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st4b {z16.b, z17.b, z18.b, z19.b}, p1, [%[dst_argb]] \n"
+ "add %[dst_argb], %[dst_argb], %[vl], lsl #2 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p1.b, wzr, %w[width_last_y] \n"
+ "whilelt p2.b, wzr, %w[width_last_uv] \n" //
+ READNV_SVE_2X NVTORGB_SVE_2X(t, b) RGBTOARGB8_SVE_2X
+ "st4b {z16.b, z17.b, z18.b, z19.b}, p1, [%[dst_argb]] \n"
+
+ "99: \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_uv] "+r"(src_vu), // %[src_vu]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width) // %[width]
+ : [vl] "r"(vl), // %[vl]
+ [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
+ [width_last_y] "r"(width_last_y), // %[width_last_y]
+ [width_last_uv] "r"(width_last_uv) // %[width_last_uv]
+ : "cc", "memory", YUVTORGB_SVE_REGS, "p2");
+}
+
+static inline void NV12ToRGB24Row_SVE_SC(
+ const uint8_t* src_y,
+ const uint8_t* src_uv,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width) STREAMING_COMPATIBLE {
+ uint64_t vl;
+ asm("cntb %0" : "=r"(vl));
+ int width_last_y = width & (vl - 1);
+ int width_last_uv = width_last_y + (width_last_y & 1);
+ asm volatile(
+ "ptrue p0.b \n" //
+ YUVTORGB_SVE_SETUP
+ "dup z19.b, #255 \n" // Alpha
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.b \n"
+ "ptrue p2.b \n"
+ "1: \n" //
+ READNV_SVE_2X NVTORGB_SVE_2X(b, t) RGBTOARGB8_SVE_2X
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st3b {z16.b, z17.b, z18.b}, p1, [%[dst_rgb24]] \n"
+ "incb %[dst_rgb24], all, mul #3 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p1.b, wzr, %w[width_last_y] \n"
+ "whilelt p2.b, wzr, %w[width_last_uv] \n" //
+ READNV_SVE_2X NVTORGB_SVE_2X(b, t) RGBTOARGB8_SVE_2X
+ "st3b {z16.b, z17.b, z18.b}, p1, [%[dst_rgb24]] \n"
+
+ "99: \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_uv] "+r"(src_uv), // %[src_uv]
+ [dst_rgb24] "+r"(dst_rgb24), // %[dst_rgb24]
+ [width] "+r"(width) // %[width]
+ : [vl] "r"(vl), // %[vl]
+ [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
+ [width_last_y] "r"(width_last_y), // %[width_last_y]
+ [width_last_uv] "r"(width_last_uv) // %[width_last_uv]
+ : "cc", "memory", YUVTORGB_SVE_REGS, "p2");
+}
+
+static inline void NV21ToRGB24Row_SVE_SC(
+ const uint8_t* src_y,
+ const uint8_t* src_vu,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width) STREAMING_COMPATIBLE {
+ uint64_t vl;
+ asm("cntb %0" : "=r"(vl));
+ int width_last_y = width & (vl - 1);
+ int width_last_uv = width_last_y + (width_last_y & 1);
+ asm volatile(
+ "ptrue p0.b \n" //
+ YUVTORGB_SVE_SETUP
+ "dup z19.b, #255 \n" // Alpha
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.b \n"
+ "ptrue p2.b \n"
+ "1: \n" //
+ READNV_SVE_2X NVTORGB_SVE_2X(t, b) RGBTOARGB8_SVE_2X
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st3b {z16.b, z17.b, z18.b}, p1, [%[dst_rgb24]] \n"
+ "incb %[dst_rgb24], all, mul #3 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p1.b, wzr, %w[width_last_y] \n"
+ "whilelt p2.b, wzr, %w[width_last_uv] \n" //
+ READNV_SVE_2X NVTORGB_SVE_2X(t, b) RGBTOARGB8_SVE_2X
+ "st3b {z16.b, z17.b, z18.b}, p1, [%[dst_rgb24]] \n"
+
+ "99: \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_uv] "+r"(src_vu), // %[src_vu]
+ [dst_rgb24] "+r"(dst_rgb24), // %[dst_rgb24]
+ [width] "+r"(width) // %[width]
+ : [vl] "r"(vl), // %[vl]
+ [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
+ [width_last_y] "r"(width_last_y), // %[width_last_y]
+ [width_last_uv] "r"(width_last_uv) // %[width_last_uv]
+ : "cc", "memory", YUVTORGB_SVE_REGS, "p2");
+}
+
+#define READYUY2_SVE \
+ "ld1w {z0.s}, p2/z, [%[src_yuy2]] \n" /* YUYV */ \
+ "incb %[src_yuy2] \n" \
+ "prfm pldl1keep, [%[src_yuy2], 448] \n" \
+ "tbl z1.b, {z0.b}, z22.b \n" /* UVUV */ \
+ "trn1 z0.b, z0.b, z0.b \n" /* YYYY */
+
+static inline void YUY2ToARGBRow_SVE_SC(const uint8_t* src_yuy2,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) STREAMING_COMPATIBLE {
+ uint32_t nv_uv_start = 0x03010301U;
+ uint32_t nv_uv_step = 0x04040404U;
+ uint64_t vl;
+ asm("cnth %0" : "=r"(vl));
+ int width_last_y = width & (vl - 1);
+ int width_last_uv = width_last_y + (width_last_y & 1);
+ asm volatile(
+ "ptrue p0.b \n"
+ "index z22.s, %w[nv_uv_start], %w[nv_uv_step] \n"
+ "dup z19.b, #255 \n" // Alpha
+ YUVTORGB_SVE_SETUP
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.h \n"
+ "ptrue p2.h \n"
+ "1: \n" //
+ READYUY2_SVE NVTORGB_SVE RGBTOARGB8_SVE
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
+ "add %[dst_argb], %[dst_argb], %[vl], lsl #2 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p1.h, wzr, %w[width_last_y] \n"
+ "whilelt p2.h, wzr, %w[width_last_uv] \n" //
+ READYUY2_SVE NVTORGB_SVE RGBTOARGB8_SVE
+ "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
+
+ "99: \n"
+ : [src_yuy2] "+r"(src_yuy2), // %[src_yuy2]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width) // %[width]
+ : [vl] "r"(vl), // %[vl]
+ [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
+ [nv_uv_start] "r"(nv_uv_start), // %[nv_uv_start]
+ [nv_uv_step] "r"(nv_uv_step), // %[nv_uv_step]
+ [width_last_y] "r"(width_last_y), // %[width_last_y]
+ [width_last_uv] "r"(width_last_uv) // %[width_last_uv]
+ : "cc", "memory", YUVTORGB_SVE_REGS, "p2");
+}
+
+#define READUYVY_SVE \
+ "ld1w {z0.s}, p2/z, [%[src_uyvy]] \n" /* UYVY */ \
+ "incb %[src_uyvy] \n" \
+ "prfm pldl1keep, [%[src_uyvy], 448] \n" \
+ "tbl z1.b, {z0.b}, z22.b \n" /* UVUV */ \
+ "trn2 z0.b, z0.b, z0.b \n" /* YYYY */
+
+static inline void UYVYToARGBRow_SVE_SC(const uint8_t* src_uyvy,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) STREAMING_COMPATIBLE {
+ uint32_t nv_uv_start = 0x02000200U;
+ uint32_t nv_uv_step = 0x04040404U;
+ uint64_t vl;
+ asm("cnth %0" : "=r"(vl));
+ int width_last_y = width & (vl - 1);
+ int width_last_uv = width_last_y + (width_last_y & 1);
+ asm volatile(
+ "ptrue p0.b \n"
+ "index z22.s, %w[nv_uv_start], %w[nv_uv_step] \n"
+ "dup z19.b, #255 \n" // Alpha
+ YUVTORGB_SVE_SETUP
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.h \n"
+ "ptrue p2.h \n"
+ "1: \n" //
+ READUYVY_SVE NVTORGB_SVE RGBTOARGB8_SVE
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
+ "add %[dst_argb], %[dst_argb], %[vl], lsl #2 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "2: \n"
+ "whilelt p1.h, wzr, %w[width_last_y] \n"
+ "whilelt p2.h, wzr, %w[width_last_uv] \n" //
+ READUYVY_SVE NVTORGB_SVE RGBTOARGB8_SVE
+ "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
+
+ "99: \n"
+ : [src_uyvy] "+r"(src_uyvy), // %[src_yuy2]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width) // %[width]
+ : [vl] "r"(vl), // %[vl]
+ [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
+ [nv_uv_start] "r"(nv_uv_start), // %[nv_uv_start]
+ [nv_uv_step] "r"(nv_uv_step), // %[nv_uv_step]
+ [width_last_y] "r"(width_last_y), // %[width_last_y]
+ [width_last_uv] "r"(width_last_uv) // %[width_last_uv]
+ : "cc", "memory", YUVTORGB_SVE_REGS, "p2");
+}
+
+static inline void I210ToARGBRow_SVE_SC(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) STREAMING_COMPATIBLE {
+ uint64_t vl;
+ asm("cnth %0" : "=r"(vl));
+ int width_last_y = width & (vl - 1);
+ asm volatile(
+ "ptrue p0.b \n" //
+ YUVTORGB_SVE_SETUP
+ "dup z19.b, #255 \n" // Alpha
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.h \n"
+ "1: \n" //
+ READI210_SVE I4XXTORGB_SVE RGBTOARGB8_SVE
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
+ "add %[dst_argb], %[dst_argb], %[vl], lsl #2 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p1.h, wzr, %w[width_last_y] \n" //
+ READI210_SVE I4XXTORGB_SVE RGBTOARGB8_SVE
+ "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
+
+ "99: \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width) // %[width]
+ : [vl] "r"(vl), // %[vl]
+ [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
+ [width_last_y] "r"(width_last_y) // %[width_last_y]
+ : "cc", "memory", YUVTORGB_SVE_REGS);
+}
+
+static inline void I210AlphaToARGBRow_SVE_SC(
+ const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ const uint16_t* src_a,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) STREAMING_COMPATIBLE {
+ uint64_t vl;
+ asm("cnth %0" : "=r"(vl));
+ int width_last_y = width & (vl - 1);
+ asm volatile(
+ "ptrue p0.b \n" //
+ YUVTORGB_SVE_SETUP
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.h \n"
+ "1: \n" //
+ READI210_SVE
+ "ld1h {z19.h}, p1/z, [%[src_a]] \n" //
+ I4XXTORGB_SVE
+ "incb %[src_a] \n" //
+ RGBATOARGB8_SVE
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
+ "add %[dst_argb], %[dst_argb], %[vl], lsl #2 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p1.h, wzr, %w[width_last_y] \n" //
+ READI210_SVE
+ "ld1h {z19.h}, p1/z, [%[src_a]] \n" //
+ I4XXTORGB_SVE RGBATOARGB8_SVE
+ "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
+
+ "99: \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [src_a] "+r"(src_a), // %[src_a]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width) // %[width]
+ : [vl] "r"(vl), // %[vl]
+ [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
+ [width_last_y] "r"(width_last_y) // %[width_last_y]
+ : "cc", "memory", YUVTORGB_SVE_REGS);
+}
+
+static inline void I210ToAR30Row_SVE_SC(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) STREAMING_COMPATIBLE {
+ uint64_t vl;
+ asm("cnth %0" : "=r"(vl));
+ int width_last_y = width & (vl - 1);
+ // The limit is used for saturating the 2.14 red channel in STOREAR30_SVE.
+ uint16_t limit = 0x3ff0;
+ asm volatile(
+ "ptrue p0.b \n" //
+ YUVTORGB_SVE_SETUP
+ "dup z23.h, %w[limit] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.h \n"
+ "1: \n" //
+ READI210_SVE I4XXTORGB_SVE STOREAR30_SVE
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p1.h, wzr, %w[width_last_y] \n" //
+ READI210_SVE I4XXTORGB_SVE STOREAR30_SVE
+
+ "99: \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_ar30] "+r"(dst_ar30), // %[dst_ar30]
+ [width] "+r"(width) // %[width]
+ : [vl] "r"(vl), // %[vl]
+ [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
+ [width_last_y] "r"(width_last_y), // %[width_last_y]
+ [limit] "r"(limit) // %[limit]
+ : "cc", "memory", YUVTORGB_SVE_REGS);
+}
+
+// P210 has 10 bits in msb of 16 bit NV12 style layout.
+static inline void P210ToARGBRow_SVE_SC(const uint16_t* src_y,
+ const uint16_t* src_uv,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) STREAMING_COMPATIBLE {
+ uint64_t vl;
+ asm("cnth %0" : "=r"(vl));
+ int width_last_y = width & (vl - 1);
+ int width_last_uv = width_last_y + (width_last_y & 1);
+ uint32_t nv_uv_start = 0x03010301U;
+ uint32_t nv_uv_step = 0x04040404U;
+ asm volatile(
+ "ptrue p0.b \n" //
+ YUVTORGB_SVE_SETUP
+ "index z22.s, %w[nv_uv_start], %w[nv_uv_step] \n"
+ "dup z19.b, #255 \n" // Alpha
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.h \n"
+ "ptrue p2.h \n"
+ "1: \n" //
+ READP210_SVE NVTORGB_SVE RGBTOARGB8_SVE
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
+ "add %[dst_argb], %[dst_argb], %[vl], lsl #2 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p1.h, wzr, %w[width_last_y] \n"
+ "whilelt p2.h, wzr, %w[width_last_uv] \n" //
+ READP210_SVE NVTORGB_SVE RGBTOARGB8_SVE
+ "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
+
+ "99: \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_uv] "+r"(src_uv), // %[src_uv]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width) // %[width]
+ : [vl] "r"(vl), // %[vl]
+ [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
+ [nv_uv_start] "r"(nv_uv_start), // %[nv_uv_start]
+ [nv_uv_step] "r"(nv_uv_step), // %[nv_uv_step]
+ [width_last_y] "r"(width_last_y), // %[width_last_y]
+ [width_last_uv] "r"(width_last_uv) // %[width_last_uv]
+ : "cc", "memory", YUVTORGB_SVE_REGS);
+}
+
+static inline void P210ToAR30Row_SVE_SC(const uint16_t* src_y,
+ const uint16_t* src_uv,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) STREAMING_COMPATIBLE {
+ uint64_t vl;
+ asm("cnth %0" : "=r"(vl));
+ int width_last_y = width & (vl - 1);
+ int width_last_uv = width_last_y + (width_last_y & 1);
+ uint32_t nv_uv_start = 0x03010301U;
+ uint32_t nv_uv_step = 0x04040404U;
+ // The limit is used for saturating the 2.14 red channel in STOREAR30_SVE.
+ uint16_t limit = 0x3ff0;
+ asm volatile(
+ "ptrue p0.b \n" //
+ YUVTORGB_SVE_SETUP
+ "index z22.s, %w[nv_uv_start], %w[nv_uv_step] \n"
+ "dup z23.h, %w[limit] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.h \n"
+ "ptrue p2.h \n"
+ "1: \n" //
+ READP210_SVE NVTORGB_SVE
+ "subs %w[width], %w[width], %w[vl] \n" //
+ STOREAR30_SVE
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p1.h, wzr, %w[width_last_y] \n"
+ "whilelt p2.h, wzr, %w[width_last_uv] \n" //
+ READP210_SVE NVTORGB_SVE STOREAR30_SVE
+
+ "99: \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_uv] "+r"(src_uv), // %[src_uv]
+ [dst_ar30] "+r"(dst_ar30), // %[dst_ar30]
+ [width] "+r"(width) // %[width]
+ : [vl] "r"(vl), // %[vl]
+ [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
+ [nv_uv_start] "r"(nv_uv_start), // %[nv_uv_start]
+ [nv_uv_step] "r"(nv_uv_step), // %[nv_uv_step]
+ [width_last_y] "r"(width_last_y), // %[width_last_y]
+ [width_last_uv] "r"(width_last_uv), // %[width_last_uv]
+ [limit] "r"(limit) // %[limit]
+ : "cc", "memory", YUVTORGB_SVE_REGS);
+}
+
+static inline void I410ToARGBRow_SVE_SC(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) STREAMING_COMPATIBLE {
+ uint64_t vl;
+ asm("cnth %0" : "=r"(vl));
+ int width_last_y = width & (vl - 1);
+ asm volatile(
+ "ptrue p0.b \n" //
+ YUVTORGB_SVE_SETUP
+ "dup z19.b, #255 \n" // Alpha
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.h \n"
+ "1: \n" //
+ READI410_SVE I4XXTORGB_SVE RGBTOARGB8_SVE
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
+ "add %[dst_argb], %[dst_argb], %[vl], lsl #2 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p1.h, wzr, %w[width_last_y] \n" //
+ READI410_SVE I4XXTORGB_SVE RGBTOARGB8_SVE
+ "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
+
+ "99: \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width) // %[width]
+ : [vl] "r"(vl), // %[vl]
+ [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
+ [width_last_y] "r"(width_last_y) // %[width_last_y]
+ : "cc", "memory", YUVTORGB_SVE_REGS);
+}
+
+static inline void I410AlphaToARGBRow_SVE_SC(
+ const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ const uint16_t* src_a,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) STREAMING_COMPATIBLE {
+ uint64_t vl;
+ asm("cnth %0" : "=r"(vl));
+ int width_last_y = width & (vl - 1);
+ asm volatile(
+ "ptrue p0.b \n" //
+ YUVTORGB_SVE_SETUP
+ "cmp %w[width], %w[vl] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.h \n"
+ "1: \n" //
+ READI410_SVE
+ "ld1h {z19.h}, p1/z, [%[src_a]] \n" //
+ I4XXTORGB_SVE
+ "incb %[src_a] \n" //
+ RGBATOARGB8_SVE
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
+ "add %[dst_argb], %[dst_argb], %[vl], lsl #2 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p1.h, wzr, %w[width_last_y] \n" //
+ READI410_SVE
+ "ld1h {z19.h}, p1/z, [%[src_a]] \n" //
+ I4XXTORGB_SVE RGBATOARGB8_SVE
+ "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
+
+ "99: \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [src_a] "+r"(src_a), // %[src_a]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width) // %[width]
+ : [vl] "r"(vl), // %[vl]
+ [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
+ [width_last_y] "r"(width_last_y) // %[width_last_y]
+ : "cc", "memory", YUVTORGB_SVE_REGS);
+}
+
+static inline void I410ToAR30Row_SVE_SC(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) STREAMING_COMPATIBLE {
+ uint64_t vl;
+ asm("cnth %0" : "=r"(vl));
+ int width_last_y = width & (vl - 1);
+ // The limit is used for saturating the 2.14 red channel in STOREAR30_SVE.
+ uint16_t limit = 0x3ff0;
+ asm volatile(
+ "ptrue p0.b \n" //
+ YUVTORGB_SVE_SETUP
+ "dup z23.h, %w[limit] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.h \n"
+ "1: \n" //
+ READI410_SVE I4XXTORGB_SVE STOREAR30_SVE
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p1.h, wzr, %w[width_last_y] \n" //
+ READI410_SVE I4XXTORGB_SVE STOREAR30_SVE
+
+ "99: \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_ar30] "+r"(dst_ar30), // %[dst_argb]
+ [width] "+r"(width) // %[width]
+ : [vl] "r"(vl), // %[vl]
+ [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
+ [width_last_y] "r"(width_last_y), // %[width_last_y]
+ [limit] "r"(limit) // %[limit]
+ : "cc", "memory", YUVTORGB_SVE_REGS);
+}
+
+static inline void P410ToARGBRow_SVE_SC(const uint16_t* src_y,
+ const uint16_t* src_uv,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) STREAMING_COMPATIBLE {
+ uint64_t vl;
+ asm("cnth %0" : "=r"(vl));
+ int width_last_y = width & (vl - 1);
+ asm volatile(
+ "ptrue p0.b \n" //
+ YUVTORGB_SVE_SETUP
+ "dup z19.b, #255 \n" // Alpha
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.h \n"
+ "ptrue p2.s \n"
+ "ptrue p3.s \n"
+ "1: \n" //
+ READP410_SVE NVTORGB_SVE RGBTOARGB8_SVE
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
+ "add %[dst_argb], %[dst_argb], %[vl], lsl #2 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p1.h, wzr, %w[width_last_y] \n"
+ "whilelt p2.s, wzr, %w[width_last_y] \n"
+ "cntw %x[vl] \n"
+ "whilelt p3.s, %w[vl], %w[width_last_y] \n" //
+ READP410_SVE NVTORGB_SVE RGBTOARGB8_SVE
+ "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
+
+ "99: \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_uv] "+r"(src_uv), // %[src_uv]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width) // %[width]
+ : [vl] "r"(vl), // %[vl]
+ [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
+ [width_last_y] "r"(width_last_y) // %[width_last_y]
+ : "cc", "memory", YUVTORGB_SVE_REGS);
+}
+
+static inline void P410ToAR30Row_SVE_SC(const uint16_t* src_y,
+ const uint16_t* src_uv,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) STREAMING_COMPATIBLE {
+ uint64_t vl;
+ asm("cnth %0" : "=r"(vl));
+ int width_last_y = width & (vl - 1);
+ // The limit is used for saturating the 2.14 red channel in STOREAR30_SVE.
+ uint16_t limit = 0x3ff0;
+ asm volatile(
+ "ptrue p0.b \n" //
+ YUVTORGB_SVE_SETUP
+ "dup z23.h, %w[limit] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.h \n"
+ "ptrue p2.s \n"
+ "ptrue p3.s \n"
+ "1: \n" //
+ READP410_SVE NVTORGB_SVE
+ "subs %w[width], %w[width], %w[vl] \n" //
+ STOREAR30_SVE
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p1.h, wzr, %w[width_last_y] \n"
+ "whilelt p2.s, wzr, %w[width_last_y] \n"
+ "cntw %x[vl] \n"
+ "whilelt p3.s, %w[vl], %w[width_last_y] \n" //
+ READP410_SVE NVTORGB_SVE STOREAR30_SVE
+
+ "99: \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_uv] "+r"(src_uv), // %[src_uv]
+ [dst_ar30] "+r"(dst_ar30), // %[dst_ar30]
+ [width] "+r"(width) // %[width]
+ : [vl] "r"(vl), // %[vl]
+ [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
+ [width_last_y] "r"(width_last_y), // %[width_last_y]
+ [limit] "r"(limit) // %[limit]
+ : "cc", "memory", YUVTORGB_SVE_REGS);
+}
+
+static inline void I212ToAR30Row_SVE_SC(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) STREAMING_COMPATIBLE {
+ uint64_t vl;
+ asm("cnth %0" : "=r"(vl));
+ int width_last_y = width & (vl - 1);
+ // The limit is used for saturating the 2.14 red channel in STOREAR30_SVE.
+ uint16_t limit = 0x3ff0;
+ asm volatile(
+ "ptrue p0.b \n" //
+ YUVTORGB_SVE_SETUP
+ "dup z23.h, %w[limit] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.h \n"
+ "1: \n" //
+ READI212_SVE I4XXTORGB_SVE STOREAR30_SVE
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p1.h, wzr, %w[width_last_y] \n" //
+ READI212_SVE I4XXTORGB_SVE STOREAR30_SVE
+
+ "99: \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_ar30] "+r"(dst_ar30), // %[dst_ar30]
+ [width] "+r"(width) // %[width]
+ : [vl] "r"(vl), // %[vl]
+ [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
+ [width_last_y] "r"(width_last_y), // %[width_last_y]
+ [limit] "r"(limit) // %[limit]
+ : "cc", "memory", YUVTORGB_SVE_REGS);
+}
+
+static inline void I212ToARGBRow_SVE_SC(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) STREAMING_COMPATIBLE {
+ uint64_t vl;
+ asm("cnth %0" : "=r"(vl));
+ int width_last_y = width & (vl - 1);
+ asm volatile(
+ "ptrue p0.b \n" //
+ YUVTORGB_SVE_SETUP
+ "dup z19.b, #255 \n" // Alpha
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.h \n"
+ "1: \n" //
+ READI212_SVE I4XXTORGB_SVE RGBTOARGB8_SVE
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
+ "add %[dst_argb], %[dst_argb], %[vl], lsl #2 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p1.h, wzr, %w[width_last_y] \n" //
+ READI212_SVE I4XXTORGB_SVE RGBTOARGB8_SVE
+ "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
+
+ "99: \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width) // %[width]
+ : [vl] "r"(vl), // %[vl]
+ [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
+ [width_last_y] "r"(width_last_y) // %[width_last_y]
+ : "cc", "memory", YUVTORGB_SVE_REGS);
+}
+
+#define CONVERT8TO8_SVE \
+ "ld1b {z0.b}, p0/z, [%[src]] \n" \
+ "ld1b {z1.b}, p1/z, [%[src], #1, mul vl] \n" \
+ "incb %[src], all, mul #2 \n" \
+ "subs %w[width], %w[width], %w[vl], lsl #1 \n" \
+ "umulh z0.b, z0.b, z2.b \n" \
+ "umulh z1.b, z1.b, z2.b \n" \
+ "prfm pldl1keep, [%[src], 448] \n" \
+ "add z0.b, z0.b, z3.b \n" \
+ "add z1.b, z1.b, z3.b \n" \
+ "st1b {z0.b}, p0, [%[dst]] \n" \
+ "st1b {z1.b}, p1, [%[dst], #1, mul vl] \n" \
+ "incb %[dst], all, mul #2 \n"
+
+static inline void Convert8To8Row_SVE_SC(const uint8_t* src_y,
+ uint8_t* dst_y,
+ int scale,
+ int bias,
+ int width) STREAMING_COMPATIBLE {
+ uint64_t vl;
+ asm volatile(
+ "dup z2.b, %w[scale] \n"
+ "dup z3.b, %w[bias] \n"
+ "cntb %[vl] \n"
+ "subs %w[width], %w[width], %w[vl], lsl #1 \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with all-true predicates to avoid predicate
+ // generation overhead.
+ "ptrue p0.b \n"
+ "ptrue p1.b \n"
+ "1: \n" //
+ CONVERT8TO8_SVE
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl], lsl #1 \n"
+ "b.eq 99f \n"
+
+ // Calculate predicates for the final iteration to deal with the tail.
+ "whilelt p0.b, wzr, %w[width] \n"
+ "whilelt p1.b, %w[vl], %w[width] \n" //
+ CONVERT8TO8_SVE
+
+ "99: \n"
+ : [src] "+r"(src_y), // %[src]
+ [dst] "+r"(dst_y), // %[dst]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ : [scale] "r"(scale), // %[scale]
+ [bias] "r"(bias) // %[bias]
+ : "cc", "memory", "z0", "z1", "z2", "z3", "p0", "p1");
+}
+
+#endif // !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__)
+
+#ifdef __cplusplus
+} // extern "C"
+} // namespace libyuv
+#endif
+
+#endif // INCLUDE_LIBYUV_ROW_SVE_H_
diff --git a/include/libyuv/scale.h b/include/libyuv/scale.h
index bfe4a34..1a8cb3d 100644
--- a/include/libyuv/scale.h
+++ b/include/libyuv/scale.h
@@ -287,6 +287,21 @@
int dst_height,
enum FilterMode filtering);
+LIBYUV_API
+int NV24Scale(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_uv,
+ int src_stride_uv,
+ int src_width,
+ int src_height,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_uv,
+ int dst_stride_uv,
+ int dst_width,
+ int dst_height,
+ enum FilterMode filtering);
+
#ifdef __cplusplus
// Legacy API. Deprecated.
LIBYUV_API
diff --git a/include/libyuv/scale_row.h b/include/libyuv/scale_row.h
index 02ed61c..eef68eb 100644
--- a/include/libyuv/scale_row.h
+++ b/include/libyuv/scale_row.h
@@ -12,6 +12,7 @@
#define INCLUDE_LIBYUV_SCALE_ROW_H_
#include "libyuv/basic_types.h"
+#include "libyuv/cpu_support.h"
#include "libyuv/scale.h"
#ifdef __cplusplus
@@ -19,46 +20,11 @@
extern "C" {
#endif
-#if defined(__pnacl__) || defined(__CLR_VER) || \
- (defined(__native_client__) && defined(__x86_64__)) || \
- (defined(__i386__) && !defined(__SSE__) && !defined(__clang__))
-#define LIBYUV_DISABLE_X86
-#endif
-#if defined(__native_client__)
-#define LIBYUV_DISABLE_NEON
-#endif
-// MemorySanitizer does not support assembly code yet. http://crbug.com/344505
-#if defined(__has_feature)
-#if __has_feature(memory_sanitizer) && !defined(LIBYUV_DISABLE_NEON)
-#define LIBYUV_DISABLE_NEON
-#endif
-#if __has_feature(memory_sanitizer) && !defined(LIBYUV_DISABLE_X86)
-#define LIBYUV_DISABLE_X86
-#endif
-#endif
-// GCC >= 4.7.0 required for AVX2.
-#if defined(__GNUC__) && (defined(__x86_64__) || defined(__i386__))
-#if (__GNUC__ > 4) || (__GNUC__ == 4 && (__GNUC_MINOR__ >= 7))
-#define GCC_HAS_AVX2 1
-#endif // GNUC >= 4.7
-#endif // __GNUC__
-
-// clang >= 3.4.0 required for AVX2.
-#if defined(__clang__) && (defined(__x86_64__) || defined(__i386__))
-#if (__clang_major__ > 3) || (__clang_major__ == 3 && (__clang_minor__ >= 4))
-#define CLANG_HAS_AVX2 1
-#endif // clang >= 3.4
-#endif // __clang__
-
-// Visual C 2012 required for AVX2.
-#if defined(_M_IX86) && !defined(__clang__) && defined(_MSC_VER) && \
- _MSC_VER >= 1700
-#define VISUALC_HAS_AVX2 1
-#endif // VisualStudio >= 2012
-
// The following are available on all x86 platforms:
-#if !defined(LIBYUV_DISABLE_X86) && \
- (defined(_M_IX86) || defined(__x86_64__) || defined(__i386__))
+#if !defined(LIBYUV_DISABLE_X86) && \
+ (defined(_M_IX86) || \
+ (defined(__x86_64__) && !defined(LIBYUV_ENABLE_ROWWIN)) || \
+ defined(__i386__))
#define HAS_FIXEDDIV1_X86
#define HAS_FIXEDDIV_X86
#define HAS_SCALEADDROW_SSE2
@@ -77,7 +43,9 @@
// The following are available for gcc/clang x86 platforms:
// TODO(fbarchard): Port to Visual C
-#if !defined(LIBYUV_DISABLE_X86) && (defined(__x86_64__) || defined(__i386__))
+#if !defined(LIBYUV_DISABLE_X86) && \
+ (defined(__x86_64__) || defined(__i386__)) && \
+ !defined(LIBYUV_ENABLE_ROWWIN)
#define HAS_SCALEUVROWDOWN2BOX_SSSE3
#define HAS_SCALEROWUP2_LINEAR_SSE2
#define HAS_SCALEROWUP2_LINEAR_SSSE3
@@ -152,6 +120,22 @@
#define HAS_SCALEUVROWUP2_BILINEAR_16_NEON
#endif
+// The following are available on AArch64 Neon platforms:
+#if !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__)
+#define HAS_SCALEROWDOWN2_16_NEON
+#endif
+
+// The following are available on AArch64 SME platforms:
+#if !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && \
+ defined(__aarch64__)
+#define HAS_SCALEARGBROWDOWN2_SME
+#define HAS_SCALEROWDOWN2_16_SME
+#define HAS_SCALEROWDOWN2_SME
+#define HAS_SCALEUVROWDOWN2BOX_SME
+#define HAS_SCALEUVROWDOWN2LINEAR_SME
+#define HAS_SCALEUVROWDOWN2_SME
+#endif
+
#if !defined(LIBYUV_DISABLE_MSA) && defined(__mips_msa)
#define HAS_SCALEADDROW_MSA
#define HAS_SCALEARGBCOLS_MSA
@@ -178,36 +162,49 @@
#define HAS_SCALEROWDOWN34_LSX
#endif
-#if !defined(LIBYUV_DISABLE_RVV) && defined(__riscv_vector)
-#define HAS_SCALEADDROW_RVV
-// TODO: Test ScaleARGBRowDownEven_RVV and enable it
-// #define HAS_SCALEARGBROWDOWNEVEN_RVV
+// The following are available on RVV with 64 bit elements
+// TODO: Update compiler to support 64 bit
+#if !defined(LIBYUV_DISABLE_RVV) && defined(__riscv_vector) && \
+ defined(__riscv_zve64x)
#define HAS_SCALEUVROWDOWN4_RVV
-#define HAS_SCALEUVROWDOWNEVEN_RVV
-#if __riscv_v_intrinsic == 11000
#define HAS_SCALEARGBROWDOWN2_RVV
+#endif
+
+// The following are available on RVV 1.1
+// TODO: Port to RVV 1.2
+#if !defined(LIBYUV_DISABLE_RVV) && defined(__riscv_vector) && \
+ defined(__riscv_v_intrinsic) && __riscv_v_intrinsic == 11000
+#define HAS_SCALEROWDOWN34_0_BOX_RVV
+#define HAS_SCALEROWDOWN34_1_BOX_RVV
+#define HAS_SCALEROWDOWN38_2_BOX_RVV
+#define HAS_SCALEROWDOWN38_3_BOX_RVV
+#define HAS_SCALEUVROWUP2_BILINEAR_RVV
+#define HAS_SCALEUVROWUP2_LINEAR_RVV
+#define HAS_SCALEROWDOWN34_RVV
+#define HAS_SCALEROWDOWN38_RVV
+#define HAS_SCALEROWUP2_BILINEAR_RVV
+#define HAS_SCALEROWUP2_LINEAR_RVV
+#endif
+
+// The following are available on RVV
+#if !defined(LIBYUV_DISABLE_RVV) && defined(__riscv_vector) && \
+ defined(__riscv_v_intrinsic) && __riscv_v_intrinsic == 11000
+#define HAS_SCALEARGBFILTERCOLS_RVV
#define HAS_SCALEARGBROWDOWN2BOX_RVV
#define HAS_SCALEARGBROWDOWN2LINEAR_RVV
#define HAS_SCALEARGBROWDOWNEVENBOX_RVV
-#define HAS_SCALEROWDOWN2_RVV
#define HAS_SCALEROWDOWN2BOX_RVV
+#define HAS_SCALEADDROW_RVV
+// TODO: Test ScaleARGBRowDownEven_RVV and enable it
+// #define HAS_SCALEARGBROWDOWNEVEN_RVV
+#define HAS_SCALEUVROWDOWNEVEN_RVV
+#define HAS_SCALEROWDOWN2_RVV
#define HAS_SCALEROWDOWN2LINEAR_RVV
-#define HAS_SCALEROWDOWN34_0_BOX_RVV
-#define HAS_SCALEROWDOWN34_1_BOX_RVV
-#define HAS_SCALEROWDOWN34_RVV
-#define HAS_SCALEROWDOWN38_2_BOX_RVV
-#define HAS_SCALEROWDOWN38_3_BOX_RVV
-#define HAS_SCALEROWDOWN38_RVV
#define HAS_SCALEROWDOWN4_RVV
#define HAS_SCALEROWDOWN4BOX_RVV
-#define HAS_SCALEROWUP2_BILINEAR_RVV
-#define HAS_SCALEROWUP2_LINEAR_RVV
#define HAS_SCALEUVROWDOWN2_RVV
#define HAS_SCALEUVROWDOWN2BOX_RVV
#define HAS_SCALEUVROWDOWN2LINEAR_RVV
-#define HAS_SCALEUVROWUP2_BILINEAR_RVV
-#define HAS_SCALEUVROWUP2_LINEAR_RVV
-#endif
#endif
// Scale ARGB vertically with bilinear interpolation.
@@ -958,6 +955,11 @@
int dst_width,
int x,
int dx);
+void ScaleARGBFilterCols_RVV(uint8_t* dst_argb,
+ const uint8_t* src_argb,
+ int dst_width,
+ int x,
+ int dx);
// ARGB Row functions
void ScaleARGBRowDown2_SSE2(const uint8_t* src_argb,
@@ -976,14 +978,26 @@
ptrdiff_t src_stride,
uint8_t* dst,
int dst_width);
+void ScaleARGBRowDown2_SME(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst,
+ int dst_width);
void ScaleARGBRowDown2Linear_NEON(const uint8_t* src_argb,
ptrdiff_t src_stride,
uint8_t* dst_argb,
int dst_width);
+void ScaleARGBRowDown2Linear_SME(const uint8_t* src_argb,
+ ptrdiff_t src_stride,
+ uint8_t* dst_argb,
+ int dst_width);
void ScaleARGBRowDown2Box_NEON(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst,
int dst_width);
+void ScaleARGBRowDown2Box_SME(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst,
+ int dst_width);
void ScaleARGBRowDown2_RVV(const uint8_t* src_argb,
ptrdiff_t src_stride,
uint8_t* dst_argb,
@@ -1180,14 +1194,26 @@
ptrdiff_t src_stride,
uint8_t* dst,
int dst_width);
+void ScaleUVRowDown2_SME(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst,
+ int dst_width);
void ScaleUVRowDown2Linear_NEON(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_uv,
int dst_width);
+void ScaleUVRowDown2Linear_SME(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_uv,
+ int dst_width);
void ScaleUVRowDown2Box_NEON(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst,
int dst_width);
+void ScaleUVRowDown2Box_SME(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst,
+ int dst_width);
void ScaleUVRowDown2_MSA(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_uv,
@@ -1429,21 +1455,55 @@
int dst_width);
// ScaleRowDown2Box also used by planar functions
-// NEON downscalers with interpolation.
-
-// Note - not static due to reuse in convert for 444 to 420.
+// NEON/SME downscalers with interpolation.
void ScaleRowDown2_NEON(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst,
int dst_width);
+void ScaleRowDown2_SME(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst,
+ int dst_width);
+void ScaleRowDown2_16_NEON(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst,
+ int dst_width);
+void ScaleRowDown2_16_SME(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst,
+ int dst_width);
void ScaleRowDown2Linear_NEON(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst,
int dst_width);
+void ScaleRowDown2Linear_SME(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst,
+ int dst_width);
+void ScaleRowDown2Linear_16_NEON(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst,
+ int dst_width);
+void ScaleRowDown2Linear_16_SME(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst,
+ int dst_width);
void ScaleRowDown2Box_NEON(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst,
int dst_width);
+void ScaleRowDown2Box_SME(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst,
+ int dst_width);
+void ScaleRowDown2Box_16_NEON(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst,
+ int dst_width);
+void ScaleRowDown2Box_16_SME(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst,
+ int dst_width);
void ScaleRowDown4_NEON(const uint8_t* src_ptr,
ptrdiff_t src_stride,
diff --git a/include/libyuv/version.h b/include/libyuv/version.h
index a9c5440..2b72bc4 100644
--- a/include/libyuv/version.h
+++ b/include/libyuv/version.h
@@ -11,6 +11,6 @@
#ifndef INCLUDE_LIBYUV_VERSION_H_
#define INCLUDE_LIBYUV_VERSION_H_
-#define LIBYUV_VERSION 1883
+#define LIBYUV_VERSION 1907
#endif // INCLUDE_LIBYUV_VERSION_H_
diff --git a/infra/config/PRESUBMIT.py b/infra/config/PRESUBMIT.py
index f79e08a..0aa465e 100644
--- a/infra/config/PRESUBMIT.py
+++ b/infra/config/PRESUBMIT.py
@@ -6,8 +6,12 @@
def CheckChangeOnUpload(input_api, output_api):
- return input_api.canned_checks.CheckChangedLUCIConfigs(input_api, output_api)
+ return input_api.canned_checks.CheckChangedLUCIConfigs(
+ input_api, output_api
+ )
def CheckChangeOnCommit(input_api, output_api):
- return input_api.canned_checks.CheckChangedLUCIConfigs(input_api, output_api)
+ return input_api.canned_checks.CheckChangedLUCIConfigs(
+ input_api, output_api
+ )
diff --git a/infra/config/commit-queue.cfg b/infra/config/commit-queue.cfg
index 4a8d77f..640b530 100644
--- a/infra/config/commit-queue.cfg
+++ b/infra/config/commit-queue.cfg
@@ -2,7 +2,7 @@
# Do not modify manually.
#
# For the schema of this file, see Config message:
-# https://luci-config.appspot.com/schemas/projects:commit-queue.cfg
+# https://config.luci.app/schemas/projects:commit-queue.cfg
cq_status_host: "chromium-cq-status.appspot.com"
submit_options {
@@ -91,6 +91,7 @@
}
builders {
name: "libyuv/try/linux_msan"
+ experiment_percentage: 100
}
builders {
name: "libyuv/try/linux_rel"
diff --git a/infra/config/cr-buildbucket.cfg b/infra/config/cr-buildbucket.cfg
index 7415851..c2e6177 100644
--- a/infra/config/cr-buildbucket.cfg
+++ b/infra/config/cr-buildbucket.cfg
@@ -2,7 +2,7 @@
# Do not modify manually.
#
# For the schema of this file, see BuildbucketCfg message:
-# https://luci-config.appspot.com/schemas/projects:buildbucket.cfg
+# https://config.luci.app/schemas/projects:buildbucket.cfg
buckets {
name: "ci"
@@ -20,7 +20,7 @@
swarming_tags: "vpython:native-python-wrapper"
dimensions: "cores:8"
dimensions: "cpu:x86-64"
- dimensions: "os:Ubuntu-18.04"
+ dimensions: "os:Ubuntu-22.04"
dimensions: "pool:luci.flex.ci"
exe {
cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
@@ -50,7 +50,7 @@
swarming_tags: "vpython:native-python-wrapper"
dimensions: "cores:8"
dimensions: "cpu:x86-64"
- dimensions: "os:Ubuntu-18.04"
+ dimensions: "os:Ubuntu-22.04"
dimensions: "pool:luci.flex.ci"
exe {
cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
@@ -80,7 +80,7 @@
swarming_tags: "vpython:native-python-wrapper"
dimensions: "cores:8"
dimensions: "cpu:x86-64"
- dimensions: "os:Ubuntu-18.04"
+ dimensions: "os:Ubuntu-22.04"
dimensions: "pool:luci.flex.ci"
exe {
cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
@@ -194,7 +194,7 @@
swarming_tags: "vpython:native-python-wrapper"
dimensions: "cores:8"
dimensions: "cpu:x86-64"
- dimensions: "os:Ubuntu-18.04"
+ dimensions: "os:Ubuntu-22.04"
dimensions: "pool:luci.flex.ci"
exe {
cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
@@ -224,7 +224,7 @@
swarming_tags: "vpython:native-python-wrapper"
dimensions: "cores:8"
dimensions: "cpu:x86-64"
- dimensions: "os:Ubuntu-18.04"
+ dimensions: "os:Ubuntu-22.04"
dimensions: "pool:luci.flex.ci"
exe {
cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
@@ -254,7 +254,7 @@
swarming_tags: "vpython:native-python-wrapper"
dimensions: "cores:8"
dimensions: "cpu:x86-64"
- dimensions: "os:Ubuntu-18.04"
+ dimensions: "os:Ubuntu-22.04"
dimensions: "pool:luci.flex.ci"
exe {
cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
@@ -284,7 +284,7 @@
swarming_tags: "vpython:native-python-wrapper"
dimensions: "cores:8"
dimensions: "cpu:x86-64"
- dimensions: "os:Ubuntu-18.04"
+ dimensions: "os:Ubuntu-22.04"
dimensions: "pool:luci.flex.ci"
exe {
cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
@@ -314,7 +314,7 @@
swarming_tags: "vpython:native-python-wrapper"
dimensions: "cores:8"
dimensions: "cpu:x86-64"
- dimensions: "os:Ubuntu-18.04"
+ dimensions: "os:Ubuntu-22.04"
dimensions: "pool:luci.flex.ci"
exe {
cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
@@ -344,7 +344,7 @@
swarming_tags: "vpython:native-python-wrapper"
dimensions: "cores:8"
dimensions: "cpu:x86-64"
- dimensions: "os:Ubuntu-18.04"
+ dimensions: "os:Ubuntu-22.04"
dimensions: "pool:luci.flex.ci"
exe {
cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
@@ -374,7 +374,7 @@
swarming_tags: "vpython:native-python-wrapper"
dimensions: "cores:8"
dimensions: "cpu:x86-64"
- dimensions: "os:Ubuntu-18.04"
+ dimensions: "os:Ubuntu-22.04"
dimensions: "pool:luci.flex.ci"
exe {
cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
@@ -404,7 +404,7 @@
swarming_tags: "vpython:native-python-wrapper"
dimensions: "cores:8"
dimensions: "cpu:x86-64"
- dimensions: "os:Ubuntu-18.04"
+ dimensions: "os:Ubuntu-22.04"
dimensions: "pool:luci.flex.ci"
exe {
cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
@@ -434,7 +434,7 @@
swarming_tags: "vpython:native-python-wrapper"
dimensions: "cores:8"
dimensions: "cpu:x86-64"
- dimensions: "os:Ubuntu-18.04"
+ dimensions: "os:Ubuntu-22.04"
dimensions: "pool:luci.flex.ci"
exe {
cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
@@ -464,7 +464,7 @@
swarming_tags: "vpython:native-python-wrapper"
dimensions: "cores:8"
dimensions: "cpu:x86-64"
- dimensions: "os:Ubuntu-18.04"
+ dimensions: "os:Ubuntu-22.04"
dimensions: "pool:luci.flex.ci"
exe {
cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
@@ -494,7 +494,7 @@
swarming_tags: "vpython:native-python-wrapper"
dimensions: "cores:8"
dimensions: "cpu:x86-64"
- dimensions: "os:Ubuntu-18.04"
+ dimensions: "os:Ubuntu-22.04"
dimensions: "pool:luci.flex.ci"
exe {
cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
@@ -904,6 +904,9 @@
}
}
}
+ constraints {
+ pools: "luci.flex.ci"
+ }
}
buckets {
name: "cron"
@@ -1049,7 +1052,7 @@
swarming_tags: "vpython:native-python-wrapper"
dimensions: "cores:8"
dimensions: "cpu:x86-64"
- dimensions: "os:Ubuntu-18.04"
+ dimensions: "os:Ubuntu-22.04"
dimensions: "pool:luci.flex.try"
exe {
cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
@@ -1079,7 +1082,7 @@
swarming_tags: "vpython:native-python-wrapper"
dimensions: "cores:8"
dimensions: "cpu:x86-64"
- dimensions: "os:Ubuntu-18.04"
+ dimensions: "os:Ubuntu-22.04"
dimensions: "pool:luci.flex.try"
exe {
cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
@@ -1167,7 +1170,7 @@
swarming_tags: "vpython:native-python-wrapper"
dimensions: "cores:8"
dimensions: "cpu:x86-64"
- dimensions: "os:Ubuntu-18.04"
+ dimensions: "os:Ubuntu-22.04"
dimensions: "pool:luci.flex.try"
exe {
cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
@@ -1197,7 +1200,7 @@
swarming_tags: "vpython:native-python-wrapper"
dimensions: "cores:8"
dimensions: "cpu:x86-64"
- dimensions: "os:Ubuntu-18.04"
+ dimensions: "os:Ubuntu-22.04"
dimensions: "pool:luci.flex.try"
exe {
cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
@@ -1227,7 +1230,7 @@
swarming_tags: "vpython:native-python-wrapper"
dimensions: "cores:8"
dimensions: "cpu:x86-64"
- dimensions: "os:Ubuntu-18.04"
+ dimensions: "os:Ubuntu-22.04"
dimensions: "pool:luci.flex.try"
exe {
cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
@@ -1257,7 +1260,7 @@
swarming_tags: "vpython:native-python-wrapper"
dimensions: "cores:8"
dimensions: "cpu:x86-64"
- dimensions: "os:Ubuntu-18.04"
+ dimensions: "os:Ubuntu-22.04"
dimensions: "pool:luci.flex.try"
exe {
cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
@@ -1287,7 +1290,7 @@
swarming_tags: "vpython:native-python-wrapper"
dimensions: "cores:8"
dimensions: "cpu:x86-64"
- dimensions: "os:Ubuntu-18.04"
+ dimensions: "os:Ubuntu-22.04"
dimensions: "pool:luci.flex.try"
exe {
cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
@@ -1317,7 +1320,7 @@
swarming_tags: "vpython:native-python-wrapper"
dimensions: "cores:8"
dimensions: "cpu:x86-64"
- dimensions: "os:Ubuntu-18.04"
+ dimensions: "os:Ubuntu-22.04"
dimensions: "pool:luci.flex.try"
exe {
cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
@@ -1347,7 +1350,7 @@
swarming_tags: "vpython:native-python-wrapper"
dimensions: "cores:8"
dimensions: "cpu:x86-64"
- dimensions: "os:Ubuntu-18.04"
+ dimensions: "os:Ubuntu-22.04"
dimensions: "pool:luci.flex.try"
exe {
cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
@@ -1377,7 +1380,7 @@
swarming_tags: "vpython:native-python-wrapper"
dimensions: "cores:8"
dimensions: "cpu:x86-64"
- dimensions: "os:Ubuntu-18.04"
+ dimensions: "os:Ubuntu-22.04"
dimensions: "pool:luci.flex.try"
exe {
cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
@@ -1494,7 +1497,7 @@
swarming_tags: "vpython:native-python-wrapper"
dimensions: "cores:8"
dimensions: "cpu:x86-64"
- dimensions: "os:Ubuntu-18.04"
+ dimensions: "os:Ubuntu-22.04"
dimensions: "pool:luci.flex.try"
exe {
cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
@@ -1701,4 +1704,8 @@
}
}
}
+ constraints {
+ pools: "luci.flex.try"
+ service_accounts: "libyuv-try-builder@chops-service-accounts.iam.gserviceaccount.com"
+ }
}
diff --git a/infra/config/luci-logdog.cfg b/infra/config/luci-logdog.cfg
index adc75be..01a3912 100644
--- a/infra/config/luci-logdog.cfg
+++ b/infra/config/luci-logdog.cfg
@@ -2,7 +2,7 @@
# Do not modify manually.
#
# For the schema of this file, see ProjectConfig message:
-# https://luci-config.appspot.com/schemas/projects:luci-logdog.cfg
+# https://config.luci.app/schemas/projects:luci-logdog.cfg
reader_auth_groups: "all"
writer_auth_groups: "luci-logdog-chromium-writers"
diff --git a/infra/config/luci-milo.cfg b/infra/config/luci-milo.cfg
index baf786f..cde1848 100644
--- a/infra/config/luci-milo.cfg
+++ b/infra/config/luci-milo.cfg
@@ -2,7 +2,7 @@
# Do not modify manually.
#
# For the schema of this file, see Project message:
-# https://luci-config.appspot.com/schemas/projects:luci-milo.cfg
+# https://config.luci.app/schemas/projects:luci-milo.cfg
consoles {
id: "main"
diff --git a/infra/config/luci-scheduler.cfg b/infra/config/luci-scheduler.cfg
index 0ec5dd0..33bd6a7 100644
--- a/infra/config/luci-scheduler.cfg
+++ b/infra/config/luci-scheduler.cfg
@@ -2,7 +2,7 @@
# Do not modify manually.
#
# For the schema of this file, see ProjectConfig message:
-# https://luci-config.appspot.com/schemas/projects:luci-scheduler.cfg
+# https://config.luci.app/schemas/projects:luci-scheduler.cfg
job {
id: "Android ARM64 Debug"
diff --git a/infra/config/main.star b/infra/config/main.star
index e83afe4..09a12b6 100755
--- a/infra/config/main.star
+++ b/infra/config/main.star
@@ -21,12 +21,6 @@
# Use LUCI Scheduler BBv2 names and add Scheduler realms configs.
lucicfg.enable_experiment("crbug.com/1182002")
-luci.builder.defaults.experiments.set(
- {
- "luci.recipes.use_python3": 100,
- },
-)
-
lucicfg.config(
lint_checks = ["default"],
config_dir = ".",
@@ -175,6 +169,9 @@
luci.bucket(
name = "ci",
+ constraints = luci.bucket_constraints(
+ pools = ["luci.flex.ci"],
+ ),
)
luci.bucket(
name = "try",
@@ -184,6 +181,12 @@
"service-account-cq",
]),
],
+ constraints = luci.bucket_constraints(
+ pools = ["luci.flex.try"],
+ service_accounts = [
+ "libyuv-try-builder@chops-service-accounts.iam.gserviceaccount.com",
+ ],
+ ),
)
luci.bucket(
name = "cron",
@@ -197,7 +200,7 @@
elif os == "win":
return {"os": "Windows-10", "cores": "8", "cpu": "x86-64"}
elif os == "linux":
- return {"os": "Ubuntu-18.04", "cores": "8", "cpu": "x86-64"}
+ return {"os": "Ubuntu-22.04", "cores": "8", "cpu": "x86-64"}
return {}
def libyuv_ci_builder(name, dimensions, properties, triggered_by):
@@ -327,7 +330,8 @@
try_builder("linux", "linux")
try_builder("linux_asan", "linux")
try_builder("linux_gcc", "linux", experiment_percentage = 100)
-try_builder("linux_msan", "linux")
+# TODO(libyuv:388428508): Make linux_msan not experimental.
+try_builder("linux_msan", "linux", experiment_percentage = 100)
try_builder("linux_rel", "linux")
try_builder("linux_tsan2", "linux")
try_builder("linux_ubsan", "linux")
diff --git a/infra/config/project.cfg b/infra/config/project.cfg
index 3c32711..59b1c24 100644
--- a/infra/config/project.cfg
+++ b/infra/config/project.cfg
@@ -2,12 +2,12 @@
# Do not modify manually.
#
# For the schema of this file, see ProjectCfg message:
-# https://luci-config.appspot.com/schemas/projects:project.cfg
+# https://config.luci.app/schemas/projects:project.cfg
name: "libyuv"
access: "group:all"
lucicfg {
- version: "1.39.14"
+ version: "1.43.16"
package_dir: "."
config_dir: "."
entry_point: "main.star"
diff --git a/infra/config/realms.cfg b/infra/config/realms.cfg
index 16ffaac..e275bdd 100644
--- a/infra/config/realms.cfg
+++ b/infra/config/realms.cfg
@@ -2,7 +2,7 @@
# Do not modify manually.
#
# For the schema of this file, see RealmsCfg message:
-# https://luci-config.appspot.com/schemas/projects:realms.cfg
+# https://config.luci.app/schemas/projects:realms.cfg
realms {
name: "@root"
diff --git a/libyuv.gni b/libyuv.gni
index 343160c..3334df7 100644
--- a/libyuv.gni
+++ b/libyuv.gni
@@ -18,12 +18,17 @@
libyuv_use_neon =
current_cpu == "arm64" ||
(current_cpu == "arm" && (arm_use_neon || arm_optionally_use_neon))
+ libyuv_use_sve = current_cpu == "arm64"
+
+ # Restrict to (is_linux || is_android) to work around undefined symbol linker
+ # errors on Fuchsia, macOS, and compilation errors on Windows.
+ # TODO: bug 359006069 - Remove the restriction after the linker and
+ # compilation errors are fixed.
+ libyuv_use_sme = current_cpu == "arm64" && (is_linux || is_android)
libyuv_use_msa =
(current_cpu == "mips64el" || current_cpu == "mipsel") && mips_use_msa
libyuv_use_mmi =
(current_cpu == "mips64el" || current_cpu == "mipsel") && mips_use_mmi
- libyuv_use_lsx =
- (current_cpu == "loong64") && loongarch64_use_lsx
- libyuv_use_lasx =
- (current_cpu == "loong64") && loongarch64_use_lasx
+ libyuv_use_lsx = current_cpu == "loong64" && loongarch64_use_lsx
+ libyuv_use_lasx = current_cpu == "loong64" && loongarch64_use_lasx
}
diff --git a/libyuv.gypi b/libyuv.gypi
index 48936aa..74fa0fe 100644
--- a/libyuv.gypi
+++ b/libyuv.gypi
@@ -13,11 +13,13 @@
'include/libyuv.h',
'include/libyuv/basic_types.h',
'include/libyuv/compare.h',
+ 'include/libyuv/compare_row.h',
'include/libyuv/convert.h',
'include/libyuv/convert_argb.h',
'include/libyuv/convert_from.h',
'include/libyuv/convert_from_argb.h',
'include/libyuv/cpu_id.h',
+ 'include/libyuv/loongson_intrinsics.h',
'include/libyuv/macros_msa.h',
'include/libyuv/mjpeg_decoder.h',
'include/libyuv/planar_functions.h',
diff --git a/pylintrc b/pylintrc
index b8bea33..57be7df 100644
--- a/pylintrc
+++ b/pylintrc
@@ -13,5 +13,37 @@
[FORMAT]
-# We use two spaces for indents, instead of the usual four spaces or tab.
-indent-string=' '
+# Maximum number of characters on a single line.
+max-line-length=79
+
+# We use four spaces for indents.
+indent-string=' '
+
+[BASIC]
+
+# List of builtins function names that should not be used, separated by a comma
+bad-functions=map,filter,apply,input
+
+# Naming style matching correct module names.
+module-naming-style=snake_case
+
+# Naming style matching correct constant names.
+const-naming-style=UPPER_CASE
+
+# Naming style matching correct class names.
+class-naming-style=PascalCase
+
+# Naming style matching correct function names.
+function-naming-style=snake_case
+
+# Regular expression matching correct method names.
+method-rgx=([a-z_][a-z0-9_]{2,}|setUp|tearDown)$
+
+# Naming style matching correct attribute names.
+attr-naming-style=snake_case
+
+# Naming style matching correct argument names.
+argument-naming-style=snake_case
+
+# Naming style matching correct variable names.
+variable-naming-style=snake_case
diff --git a/riscv_script/riscv-clang.cmake b/riscv_script/riscv-clang.cmake
index e287941..35888ae 100644
--- a/riscv_script/riscv-clang.cmake
+++ b/riscv_script/riscv-clang.cmake
@@ -43,6 +43,7 @@
list(APPEND RISCV_COMPILER_FLAGS "-march=rv64gc")
endif()
endif()
+add_compile_options("-Wuninitialized")
message(STATUS "RISCV_COMPILER_FLAGS: ${RISCV_COMPILER_FLAGS}")
set(CMAKE_C_FLAGS "${RISCV_COMPILER_FLAGS} ${CMAKE_C_FLAGS}")
diff --git a/source/compare.cc b/source/compare.cc
index 50a736b..3ce4cfe 100644
--- a/source/compare.cc
+++ b/source/compare.cc
@@ -44,6 +44,11 @@
HashDjb2_SSE = HashDjb2_AVX2;
}
#endif
+#if defined(HAS_HASHDJB2_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ HashDjb2_SSE = HashDjb2_NEON;
+ }
+#endif
while (count >= (uint64_t)kBlockSize) {
seed = HashDjb2_SSE(src, kBlockSize, seed);
@@ -134,6 +139,11 @@
HammingDistance = HammingDistance_NEON;
}
#endif
+#if defined(HAS_HAMMINGDISTANCE_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd)) {
+ HammingDistance = HammingDistance_NEON_DotProd;
+ }
+#endif
#if defined(HAS_HAMMINGDISTANCE_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
HammingDistance = HammingDistance_SSSE3;
@@ -194,6 +204,11 @@
SumSquareError = SumSquareError_NEON;
}
#endif
+#if defined(HAS_SUMSQUAREERROR_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd)) {
+ SumSquareError = SumSquareError_NEON_DotProd;
+ }
+#endif
#if defined(HAS_SUMSQUAREERROR_SSE2)
if (TestCpuFlag(kCpuHasSSE2)) {
// Note only used for multiples of 16 so count is not checked.
diff --git a/source/compare_gcc.cc b/source/compare_gcc.cc
index 33cbe25..33a725e 100644
--- a/source/compare_gcc.cc
+++ b/source/compare_gcc.cc
@@ -19,13 +19,17 @@
#endif
// This module is for GCC x86 and x64.
-#if !defined(LIBYUV_DISABLE_X86) && (defined(__x86_64__) || defined(__i386__))
+#if !defined(LIBYUV_DISABLE_X86) && \
+ (defined(__x86_64__) || defined(__i386__)) && \
+ !defined(LIBYUV_ENABLE_ROWWIN)
+
+// "memory" clobber prevents the reads from being removed
#if defined(__x86_64__)
uint32_t HammingDistance_SSE42(const uint8_t* src_a,
const uint8_t* src_b,
int count) {
- uint64_t diff = 0u;
+ uint64_t diff;
asm volatile(
"xor %3,%3 \n"
@@ -35,7 +39,7 @@
// Process 32 bytes per loop.
LABELALIGN
- "1: \n"
+ "1: \n"
"mov (%0),%%rcx \n"
"mov 0x8(%0),%%rdx \n"
"xor (%1),%%rcx \n"
@@ -63,9 +67,9 @@
: "+r"(src_a), // %0
"+r"(src_b), // %1
"+r"(count), // %2
- "=r"(diff) // %3
+ "=&r"(diff) // %3
:
- : "memory", "cc", "rcx", "rdx", "rsi", "rdi", "r8", "r9", "r10");
+ : "cc", "memory", "rcx", "rdx", "rsi", "rdi", "r8", "r9", "r10");
return (uint32_t)(diff);
}
@@ -78,7 +82,7 @@
asm volatile(
// Process 16 bytes per loop.
LABELALIGN
- "1: \n"
+ "1: \n"
"mov (%0),%%ecx \n"
"mov 0x4(%0),%%edx \n"
"xor (%1),%%ecx \n"
@@ -104,7 +108,7 @@
"+r"(count), // %2
"+r"(diff) // %3
:
- : "memory", "cc", "ecx", "edx");
+ : "cc", "memory", "ecx", "edx");
return diff;
}
@@ -117,7 +121,7 @@
uint32_t HammingDistance_SSSE3(const uint8_t* src_a,
const uint8_t* src_b,
int count) {
- uint32_t diff = 0u;
+ uint32_t diff;
asm volatile(
"movdqa %4,%%xmm2 \n"
@@ -127,7 +131,7 @@
"sub %0,%1 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"movdqa (%0),%%xmm4 \n"
"movdqa 0x10(%0), %%xmm5 \n"
"pxor (%0,%1), %%xmm4 \n"
@@ -166,7 +170,7 @@
"=r"(diff) // %3
: "m"(kNibbleMask), // %4
"m"(kBitCount) // %5
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ : "cc", "memory", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
"xmm7");
return diff;
@@ -176,7 +180,7 @@
uint32_t HammingDistance_AVX2(const uint8_t* src_a,
const uint8_t* src_b,
int count) {
- uint32_t diff = 0u;
+ uint32_t diff;
asm volatile(
"vbroadcastf128 %4,%%ymm2 \n"
@@ -186,7 +190,7 @@
"sub %0,%1 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"vmovdqa (%0),%%ymm4 \n"
"vmovdqa 0x20(%0), %%ymm5 \n"
"vpxor (%0,%1), %%ymm4, %%ymm4 \n"
@@ -214,15 +218,15 @@
"vpaddd %%ymm1,%%ymm0,%%ymm0 \n"
"vpermq $0xaa,%%ymm0,%%ymm1 \n"
"vpaddd %%ymm1,%%ymm0,%%ymm0 \n"
- "vmovd %%xmm0, %3 \n"
- "vzeroupper \n"
+ "vmovd %%xmm0,%3 \n"
+ "vzeroupper \n"
: "+r"(src_a), // %0
"+r"(src_b), // %1
"+r"(count), // %2
"=r"(diff) // %3
: "m"(kNibbleMask), // %4
"m"(kBitCount) // %5
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6");
+ : "cc", "memory", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6");
return diff;
}
@@ -237,7 +241,7 @@
"pxor %%xmm5,%%xmm5 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm1 \n"
"lea 0x10(%0),%0 \n"
"movdqu (%1),%%xmm2 \n"
@@ -261,13 +265,12 @@
"pshufd $0x1,%%xmm0,%%xmm1 \n"
"paddd %%xmm1,%%xmm0 \n"
"movd %%xmm0,%3 \n"
-
: "+r"(src_a), // %0
"+r"(src_b), // %1
"+r"(count), // %2
- "=g"(sse) // %3
- ::"memory",
- "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm5");
+ "=r"(sse) // %3
+ :
+ : "cc", "memory", "xmm0", "xmm1", "xmm2", "xmm3", "xmm5");
return sse;
}
@@ -305,7 +308,7 @@
"movdqa %4,%%xmm6 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm1 \n"
"lea 0x10(%0),%0 \n"
"pmulld %%xmm6,%%xmm0 \n"
@@ -341,13 +344,13 @@
: "+r"(src), // %0
"+r"(count), // %1
"+rm"(seed), // %2
- "=g"(hash) // %3
+ "=r"(hash) // %3
: "m"(kHash16x33), // %4
"m"(kHashMul0), // %5
"m"(kHashMul1), // %6
"m"(kHashMul2), // %7
"m"(kHashMul3) // %8
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ : "cc", "memory", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
"xmm7");
return hash;
}
diff --git a/source/compare_neon.cc b/source/compare_neon.cc
index afdd601..ee1f7b2 100644
--- a/source/compare_neon.cc
+++ b/source/compare_neon.cc
@@ -31,7 +31,7 @@
asm volatile(
"vmov.u16 q4, #0 \n" // accumulator
- "1: \n"
+ "1: \n"
"vld1.8 {q0, q1}, [%0]! \n"
"vld1.8 {q2, q3}, [%1]! \n"
"veor.32 q0, q0, q2 \n"
@@ -64,7 +64,7 @@
"vmov.u8 q9, #0 \n"
"vmov.u8 q11, #0 \n"
- "1: \n"
+ "1: \n"
"vld1.8 {q0}, [%0]! \n"
"vld1.8 {q1}, [%1]! \n"
"subs %2, %2, #16 \n"
diff --git a/source/compare_neon64.cc b/source/compare_neon64.cc
index 70fb9b9..36c5e57 100644
--- a/source/compare_neon64.cc
+++ b/source/compare_neon64.cc
@@ -29,7 +29,7 @@
asm volatile(
"movi v4.8h, #0 \n"
- "1: \n"
+ "1: \n"
"ld1 {v0.16b, v1.16b}, [%0], #32 \n"
"ld1 {v2.16b, v3.16b}, [%1], #32 \n"
"eor v0.16b, v0.16b, v2.16b \n"
@@ -47,7 +47,7 @@
"fmov %w3, s4 \n"
: "+r"(src_a), "+r"(src_b), "+r"(count), "=r"(diff)
:
- : "cc", "v0", "v1", "v2", "v3", "v4");
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v4");
return diff;
}
@@ -56,12 +56,12 @@
int count) {
uint32_t sse;
asm volatile(
- "eor v16.16b, v16.16b, v16.16b \n"
- "eor v18.16b, v18.16b, v18.16b \n"
- "eor v17.16b, v17.16b, v17.16b \n"
- "eor v19.16b, v19.16b, v19.16b \n"
+ "movi v16.16b, #0 \n"
+ "movi v17.16b, #0 \n"
+ "movi v18.16b, #0 \n"
+ "movi v19.16b, #0 \n"
- "1: \n"
+ "1: \n"
"ld1 {v0.16b}, [%0], #16 \n"
"ld1 {v1.16b}, [%1], #16 \n"
"subs %w2, %w2, #16 \n"
@@ -82,7 +82,136 @@
"fmov %w3, s0 \n"
: "+r"(src_a), "+r"(src_b), "+r"(count), "=r"(sse)
:
- : "cc", "v0", "v1", "v2", "v3", "v16", "v17", "v18", "v19");
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v16", "v17", "v18", "v19");
+ return sse;
+}
+
+static const uvec32 kDjb2Multiplicands[] = {
+ {0x0c3525e1, // 33^15
+ 0xa3476dc1, // 33^14
+ 0x3b4039a1, // 33^13
+ 0x4f5f0981}, // 33^12
+ {0x30f35d61, // 33^11
+ 0x855cb541, // 33^10
+ 0x040a9121, // 33^9
+ 0x747c7101}, // 33^8
+ {0xec41d4e1, // 33^7
+ 0x4cfa3cc1, // 33^6
+ 0x025528a1, // 33^5
+ 0x00121881}, // 33^4
+ {0x00008c61, // 33^3
+ 0x00000441, // 33^2
+ 0x00000021, // 33^1
+ 0x00000001}, // 33^0
+};
+
+static const uvec32 kDjb2WidenIndices[] = {
+ {0xffffff00U, 0xffffff01U, 0xffffff02U, 0xffffff03U},
+ {0xffffff04U, 0xffffff05U, 0xffffff06U, 0xffffff07U},
+ {0xffffff08U, 0xffffff09U, 0xffffff0aU, 0xffffff0bU},
+ {0xffffff0cU, 0xffffff0dU, 0xffffff0eU, 0xffffff0fU},
+};
+
+uint32_t HashDjb2_NEON(const uint8_t* src, int count, uint32_t seed) {
+ uint32_t hash = seed;
+ const uint32_t c16 = 0x92d9e201; // 33^16
+ uint32_t tmp, tmp2;
+ asm("ld1 {v16.4s, v17.4s, v18.4s, v19.4s}, [%[kIdx]] \n"
+ "ld1 {v4.4s, v5.4s, v6.4s, v7.4s}, [%[kMuls]] \n"
+
+ // count is always a multiple of 16.
+ // maintain two accumulators, reduce and then final sum in scalar since
+ // this has better performance on little cores.
+ "1: \n"
+ "ldr q0, [%[src]], #16 \n"
+ "subs %w[count], %w[count], #16 \n"
+ "tbl v3.16b, {v0.16b}, v19.16b \n"
+ "tbl v2.16b, {v0.16b}, v18.16b \n"
+ "tbl v1.16b, {v0.16b}, v17.16b \n"
+ "tbl v0.16b, {v0.16b}, v16.16b \n"
+ "mul v3.4s, v3.4s, v7.4s \n"
+ "mul v2.4s, v2.4s, v6.4s \n"
+ "mla v3.4s, v1.4s, v5.4s \n"
+ "mla v2.4s, v0.4s, v4.4s \n"
+ "addv s1, v3.4s \n"
+ "addv s0, v2.4s \n"
+ "fmov %w[tmp2], s1 \n"
+ "fmov %w[tmp], s0 \n"
+ "add %w[tmp], %w[tmp], %w[tmp2] \n"
+ "madd %w[hash], %w[hash], %w[c16], %w[tmp] \n"
+ "b.gt 1b \n"
+ : [hash] "+r"(hash), // %[hash]
+ [count] "+r"(count), // %[count]
+ [tmp] "=&r"(tmp), // %[tmp]
+ [tmp2] "=&r"(tmp2) // %[tmp2]
+ : [src] "r"(src), // %[src]
+ [kMuls] "r"(kDjb2Multiplicands), // %[kMuls]
+ [kIdx] "r"(kDjb2WidenIndices), // %[kIdx]
+ [c16] "r"(c16) // %[c16]
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16",
+ "v17", "v18", "v19");
+ return hash;
+}
+
+uint32_t HammingDistance_NEON_DotProd(const uint8_t* src_a,
+ const uint8_t* src_b,
+ int count) {
+ uint32_t diff;
+ asm volatile(
+ "movi v4.4s, #0 \n"
+ "movi v5.4s, #0 \n"
+ "movi v6.16b, #1 \n"
+
+ "1: \n"
+ "ldp q0, q1, [%0], #32 \n"
+ "ldp q2, q3, [%1], #32 \n"
+ "eor v0.16b, v0.16b, v2.16b \n"
+ "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead
+ "eor v1.16b, v1.16b, v3.16b \n"
+ "cnt v0.16b, v0.16b \n"
+ "prfm pldl1keep, [%1, 448] \n"
+ "cnt v1.16b, v1.16b \n"
+ "subs %w2, %w2, #32 \n"
+ "udot v4.4s, v0.16b, v6.16b \n"
+ "udot v5.4s, v1.16b, v6.16b \n"
+ "b.gt 1b \n"
+
+ "add v0.4s, v4.4s, v5.4s \n"
+ "addv s0, v0.4s \n"
+ "fmov %w3, s0 \n"
+ : "+r"(src_a), "+r"(src_b), "+r"(count), "=r"(diff)
+ :
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6");
+ return diff;
+}
+
+uint32_t SumSquareError_NEON_DotProd(const uint8_t* src_a,
+ const uint8_t* src_b,
+ int count) {
+ // count is guaranteed to be a multiple of 32.
+ uint32_t sse;
+ asm volatile(
+ "movi v4.4s, #0 \n"
+ "movi v5.4s, #0 \n"
+
+ "1: \n"
+ "ldp q0, q2, [%0], #32 \n"
+ "ldp q1, q3, [%1], #32 \n"
+ "subs %w2, %w2, #32 \n"
+ "uabd v0.16b, v0.16b, v1.16b \n"
+ "uabd v1.16b, v2.16b, v3.16b \n"
+ "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead
+ "udot v4.4s, v0.16b, v0.16b \n"
+ "udot v5.4s, v1.16b, v1.16b \n"
+ "prfm pldl1keep, [%1, 448] \n"
+ "b.gt 1b \n"
+
+ "add v0.4s, v4.4s, v5.4s \n"
+ "addv s0, v0.4s \n"
+ "fmov %w3, s0 \n"
+ : "+r"(src_a), "+r"(src_b), "+r"(count), "=r"(sse)
+ :
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5");
return sse;
}
diff --git a/source/compare_win.cc b/source/compare_win.cc
index 9bb27f1..9d5bb27 100644
--- a/source/compare_win.cc
+++ b/source/compare_win.cc
@@ -23,8 +23,8 @@
#endif
// This module is for 32 bit Visual C x86
-#if !defined(LIBYUV_DISABLE_X86) && defined(_MSC_VER) && \
- !defined(__clang__) && defined(_M_IX86)
+#if !defined(LIBYUV_DISABLE_X86) && defined(_MSC_VER) && defined(_M_IX86) && \
+ (!defined(__clang__) || defined(LIBYUV_ENABLE_ROWWIN))
uint32_t HammingDistance_SSE42(const uint8_t* src_a,
const uint8_t* src_b,
diff --git a/source/convert.cc b/source/convert.cc
index 6ac5bc4..7ebdad2 100644
--- a/source/convert.cc
+++ b/source/convert.cc
@@ -33,7 +33,7 @@
return v >= 0 ? v : -v;
}
-// Any I4xx To I420 format with mirroring.
+// Any I4xx To I420 format
static int I4xxToI420(const uint8_t* src_y,
int src_stride_y,
const uint8_t* src_u,
@@ -50,20 +50,19 @@
int src_y_height,
int src_uv_width,
int src_uv_height) {
- const int dst_y_width = Abs(src_y_width);
+ const int dst_y_width = src_y_width;
const int dst_y_height = Abs(src_y_height);
const int dst_uv_width = SUBSAMPLE(dst_y_width, 1, 1);
const int dst_uv_height = SUBSAMPLE(dst_y_height, 1, 1);
int r;
- if (src_uv_width <= 0 || src_uv_height == 0) {
+ if ((!src_y && dst_y) || !src_u || !src_v || !dst_u || !dst_v ||
+ src_y_width <= 0 || src_y_height == 0 || src_uv_width <= 0 ||
+ src_uv_height == 0) {
return -1;
}
if (dst_y) {
- r = ScalePlane(src_y, src_stride_y, src_y_width, src_y_height, dst_y,
- dst_stride_y, dst_y_width, dst_y_height, kFilterBilinear);
- if (r != 0) {
- return r;
- }
+ CopyPlane(src_y, src_stride_y, dst_y, dst_stride_y, src_y_width,
+ src_y_height);
}
r = ScalePlane(src_u, src_stride_u, src_uv_width, src_uv_height, dst_u,
dst_stride_u, dst_uv_width, dst_uv_height, kFilterBilinear);
@@ -75,9 +74,7 @@
return r;
}
-// Copy I420 with optional flipping.
-// TODO(fbarchard): Use Scale plane which supports mirroring, but ensure
-// is does row coalescing.
+// Copy I420 with optional vertical flipping using negative height.
LIBYUV_API
int I420Copy(const uint8_t* src_y,
int src_stride_y,
@@ -200,8 +197,10 @@
}
// Convert Y plane.
- Convert16To8Plane(src_y, src_stride_y, dst_y, dst_stride_y, scale, width,
- height);
+ if (dst_y) {
+ Convert16To8Plane(src_y, src_stride_y, dst_y, dst_stride_y, scale, width,
+ height);
+ }
// Convert UV planes.
Convert16To8Plane(src_u, src_stride_u, dst_u, dst_stride_u, scale, uv_width,
uv_height);
@@ -510,7 +509,7 @@
dst_v, dst_stride_v, width, height, 12);
}
-// Any Ix10 To I010 format with mirroring.
+// Any Ix10 To I010 format
static int Ix10ToI010(const uint16_t* src_y,
int src_stride_y,
const uint16_t* src_u,
@@ -527,22 +526,19 @@
int height,
int subsample_x,
int subsample_y) {
- const int dst_y_width = Abs(width);
+ const int dst_y_width = width;
const int dst_y_height = Abs(height);
const int src_uv_width = SUBSAMPLE(width, subsample_x, subsample_x);
const int src_uv_height = SUBSAMPLE(height, subsample_y, subsample_y);
const int dst_uv_width = SUBSAMPLE(dst_y_width, 1, 1);
const int dst_uv_height = SUBSAMPLE(dst_y_height, 1, 1);
int r;
- if (width <= 0 || height == 0) {
+ if ((!src_y && dst_y) || !src_u || !src_v || !dst_u || !dst_v || width <= 0 ||
+ height == 0) {
return -1;
}
if (dst_y) {
- r = ScalePlane_12(src_y, src_stride_y, width, height, dst_y, dst_stride_y,
- dst_y_width, dst_y_height, kFilterBilinear);
- if (r != 0) {
- return r;
- }
+ CopyPlane_16(src_y, src_stride_y, dst_y, dst_stride_y, width, height);
}
r = ScalePlane_12(src_u, src_stride_u, src_uv_width, src_uv_height, dst_u,
dst_stride_u, dst_uv_width, dst_uv_height, kFilterBilinear);
@@ -594,7 +590,7 @@
dst_v, dst_stride_v, width, height, 1, 0);
}
-// Any I[420]1[02] to P[420]1[02] format with mirroring.
+// Any I[420]1[02] to P[420]1[02] format
static int IxxxToPxxx(const uint16_t* src_y,
int src_stride_y,
const uint16_t* src_u,
@@ -642,6 +638,165 @@
}
LIBYUV_API
+int I010ToNV12(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_u,
+ int src_stride_u,
+ const uint16_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height) {
+ int y;
+ int halfwidth = (width + 1) >> 1;
+ int halfheight = (height + 1) >> 1;
+ const int scale = 16385; // 16384 for 10 bits
+ void (*Convert16To8Row)(const uint16_t* src_y, uint8_t* dst_y, int scale,
+ int width) = Convert16To8Row_C;
+ void (*MergeUVRow)(const uint8_t* src_u, const uint8_t* src_v,
+ uint8_t* dst_uv, int width) = MergeUVRow_C;
+ if ((!src_y && dst_y) || !src_u || !src_v || !dst_uv || width <= 0 ||
+ height == 0) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ halfheight = (height + 1) >> 1;
+ src_y = src_y + (height - 1) * src_stride_y;
+ src_u = src_u + (halfheight - 1) * src_stride_u;
+ src_v = src_v + (halfheight - 1) * src_stride_v;
+ src_stride_y = -src_stride_y;
+ src_stride_u = -src_stride_u;
+ src_stride_v = -src_stride_v;
+ }
+#if defined(HAS_CONVERT16TO8ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ Convert16To8Row = Convert16To8Row_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ Convert16To8Row = Convert16To8Row_NEON;
+ }
+ }
+#endif
+#if defined(HAS_CONVERT16TO8ROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ Convert16To8Row = Convert16To8Row_SME;
+ }
+#endif
+#if defined(HAS_CONVERT16TO8ROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ Convert16To8Row = Convert16To8Row_Any_SSSE3;
+ if (IS_ALIGNED(width, 16)) {
+ Convert16To8Row = Convert16To8Row_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_CONVERT16TO8ROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ Convert16To8Row = Convert16To8Row_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ Convert16To8Row = Convert16To8Row_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_CONVERT16TO8ROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ Convert16To8Row = Convert16To8Row_Any_AVX512BW;
+ if (IS_ALIGNED(width, 64)) {
+ Convert16To8Row = Convert16To8Row_AVX512BW;
+ }
+ }
+#endif
+
+#if defined(HAS_MERGEUVROW_SSE2)
+ if (TestCpuFlag(kCpuHasSSE2)) {
+ MergeUVRow = MergeUVRow_Any_SSE2;
+ if (IS_ALIGNED(halfwidth, 16)) {
+ MergeUVRow = MergeUVRow_SSE2;
+ }
+ }
+#endif
+#if defined(HAS_MERGEUVROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ MergeUVRow = MergeUVRow_Any_AVX2;
+ if (IS_ALIGNED(halfwidth, 16)) {
+ MergeUVRow = MergeUVRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_MERGEUVROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ MergeUVRow = MergeUVRow_Any_AVX512BW;
+ if (IS_ALIGNED(halfwidth, 32)) {
+ MergeUVRow = MergeUVRow_AVX512BW;
+ }
+ }
+#endif
+#if defined(HAS_MERGEUVROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ MergeUVRow = MergeUVRow_Any_NEON;
+ if (IS_ALIGNED(halfwidth, 16)) {
+ MergeUVRow = MergeUVRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_MERGEUVROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ MergeUVRow = MergeUVRow_SME;
+ }
+#endif
+#if defined(HAS_MERGEUVROW_MSA)
+ if (TestCpuFlag(kCpuHasMSA)) {
+ MergeUVRow = MergeUVRow_Any_MSA;
+ if (IS_ALIGNED(halfwidth, 16)) {
+ MergeUVRow = MergeUVRow_MSA;
+ }
+ }
+#endif
+#if defined(HAS_MERGEUVROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ MergeUVRow = MergeUVRow_Any_LSX;
+ if (IS_ALIGNED(halfwidth, 16)) {
+ MergeUVRow = MergeUVRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_MERGEUVROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ MergeUVRow = MergeUVRow_RVV;
+ }
+#endif
+
+ // Convert Y plane.
+ if (dst_y) {
+ Convert16To8Plane(src_y, src_stride_y, dst_y, dst_stride_y, scale, width,
+ height);
+ }
+
+ {
+ // Allocate a row of uv.
+ align_buffer_64(row_u, ((halfwidth + 31) & ~31) * 2);
+ uint8_t* row_v = row_u + ((halfwidth + 31) & ~31);
+ if (!row_u)
+ return 1;
+
+ for (y = 0; y < halfheight; ++y) {
+ Convert16To8Row(src_u, row_u, scale, halfwidth);
+ Convert16To8Row(src_v, row_v, scale, halfwidth);
+ MergeUVRow(row_u, row_v, dst_uv, halfwidth);
+ src_u += src_stride_u;
+ src_v += src_stride_v;
+ dst_uv += dst_stride_uv;
+ }
+ free_aligned_buffer_64(row_u);
+ }
+ return 0;
+}
+
+LIBYUV_API
int I210ToP210(const uint16_t* src_y,
int src_stride_y,
const uint16_t* src_u,
@@ -989,7 +1144,8 @@
int source_y_fraction) = InterpolateRow_C;
int halfwidth = (width + 1) >> 1;
int halfheight = (height + 1) >> 1;
- if (!src_u || !src_v || !dst_vu || width <= 0 || height == 0) {
+ if ((!src_y && dst_y) || !src_u || !src_v || !dst_vu || width <= 0 ||
+ height == 0) {
return -1;
}
// Negative height means invert the image.
@@ -1035,6 +1191,11 @@
}
}
#endif
+#if defined(HAS_MERGEUVROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ MergeUVRow = MergeUVRow_SME;
+ }
+#endif
#if defined(HAS_MERGEUVROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
MergeUVRow = MergeUVRow_Any_MSA;
@@ -1080,6 +1241,11 @@
}
}
#endif
+#if defined(HAS_INTERPOLATEROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ InterpolateRow = InterpolateRow_SME;
+ }
+#endif
#if defined(HAS_INTERPOLATEROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
InterpolateRow = InterpolateRow_Any_MSA;
@@ -1166,7 +1332,8 @@
int dst_stride_uv,
int width,
int height) {
- if (!src_y || !src_u || !src_v || !dst_uv || width <= 0 || height == 0) {
+ if ((!src_y && dst_y) || !src_u || !src_v || !dst_uv || width <= 0 ||
+ height == 0) {
return -1;
}
// Negative height means invert the image.
@@ -1219,7 +1386,7 @@
int height) {
int halfwidth = (width + 1) >> 1;
int halfheight = (height + 1) >> 1;
- if (!dst_u || !dst_v || width <= 0 || height == 0) {
+ if ((!src_y && dst_y) || !dst_u || !dst_v || width <= 0 || height == 0) {
return -1;
}
// Negative height means invert the image.
@@ -1249,7 +1416,7 @@
int height) {
int halfwidth = (width + 1) >> 1;
int halfheight = (height + 1) >> 1;
- if (!dst_vu || width <= 0 || height == 0) {
+ if ((!src_y && dst_y) || !dst_vu || width <= 0 || height == 0) {
return -1;
}
// Negative height means invert the image.
@@ -1283,7 +1450,8 @@
int height) {
int halfwidth = (width + 1) >> 1;
int halfheight = (height + 1) >> 1;
- if (!src_uv || !dst_u || !dst_v || width <= 0 || height == 0) {
+ if ((!src_y && dst_y) || !src_uv || !dst_u || !dst_v || width <= 0 ||
+ height == 0) {
return -1;
}
// Negative height means invert the image.
@@ -1351,16 +1519,12 @@
int width,
int height) {
int r;
- if (width <= 0 || height == 0) {
+ if ((!src_y && dst_y) || !src_uv || !dst_uv || width <= 0 || height == 0) {
return -1;
}
if (dst_y) {
- r = ScalePlane(src_y, src_stride_y, width, height, dst_y, dst_stride_y,
- Abs(width), Abs(height), kFilterBilinear);
- if (r != 0) {
- return r;
- }
+ CopyPlane(src_y, src_stride_y, dst_y, dst_stride_y, width, height);
}
r = UVScale(src_uv, src_stride_uv, SUBSAMPLE(width, 1, 1),
SUBSAMPLE(height, 1, 1), dst_uv, dst_stride_uv, Abs(width),
@@ -1380,23 +1544,19 @@
int width,
int height) {
int r;
- if (width <= 0 || height == 0) {
+ if ((!src_y && dst_y) || !src_uv || !dst_uv || width <= 0 || height == 0) {
return -1;
}
if (dst_y) {
- r = ScalePlane(src_y, src_stride_y, width, height, dst_y, dst_stride_y,
- Abs(width), Abs(height), kFilterBilinear);
- if (r != 0) {
- return r;
- }
+ CopyPlane(src_y, src_stride_y, dst_y, dst_stride_y, width, height);
}
r = UVScale(src_uv, src_stride_uv, SUBSAMPLE(width, 1, 1), height, dst_uv,
dst_stride_uv, Abs(width), Abs(height), kFilterBilinear);
return r;
}
-// Any P[420]1[02] to I[420]1[02] format with mirroring.
+// Any P[420]1[02] to I[420]1[02] format
static int PxxxToIxxx(const uint16_t* src_y,
int src_stride_y,
const uint16_t* src_uv,
@@ -1414,7 +1574,8 @@
int depth) {
const int uv_width = SUBSAMPLE(width, subsample_x, subsample_x);
const int uv_height = SUBSAMPLE(height, subsample_y, subsample_y);
- if (width <= 0 || height == 0) {
+ if (!src_y || !dst_y || !src_uv || !dst_u || !dst_v || width <= 0 ||
+ height == 0) {
return -1;
}
ConvertToLSBPlane_16(src_y, src_stride_y, dst_y, dst_stride_y, width, height,
@@ -1472,16 +1633,12 @@
int width,
int height) {
int r;
- if (width <= 0 || height == 0) {
+ if ((!src_y && dst_y) || !src_uv || !dst_uv || width <= 0 || height == 0) {
return -1;
}
if (dst_y) {
- r = ScalePlane_16(src_y, src_stride_y, width, height, dst_y, dst_stride_y,
- Abs(width), Abs(height), kFilterBilinear);
- if (r != 0) {
- return r;
- }
+ CopyPlane_16(src_y, src_stride_y, dst_y, dst_stride_y, width, height);
}
r = UVScale_16(src_uv, src_stride_uv, SUBSAMPLE(width, 1, 1),
SUBSAMPLE(height, 1, 1), dst_uv, dst_stride_uv, Abs(width),
@@ -1501,16 +1658,12 @@
int width,
int height) {
int r;
- if (width <= 0 || height == 0) {
+ if ((!src_y && dst_y) || !src_uv || !dst_uv || width <= 0 || height == 0) {
return -1;
}
if (dst_y) {
- r = ScalePlane_16(src_y, src_stride_y, width, height, dst_y, dst_stride_y,
- Abs(width), Abs(height), kFilterBilinear);
- if (r != 0) {
- return r;
- }
+ CopyPlane_16(src_y, src_stride_y, dst_y, dst_stride_y, width, height);
}
r = UVScale_16(src_uv, src_stride_uv, SUBSAMPLE(width, 1, 1), height, dst_uv,
dst_stride_uv, Abs(width), Abs(height), kFilterBilinear);
@@ -1782,6 +1935,14 @@
}
}
#endif
+#if defined(HAS_AYUVTOUVROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ AYUVToUVRow = AYUVToUVRow_Any_SVE2;
+ if (IS_ALIGNED(width, 2)) {
+ AYUVToUVRow = AYUVToUVRow_SVE2;
+ }
+ }
+#endif
for (y = 0; y < height - 1; y += 2) {
AYUVToUVRow(src_ayuv, src_stride_ayuv, dst_uv, width);
@@ -1851,6 +2012,14 @@
}
}
#endif
+#if defined(HAS_AYUVTOVUROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ AYUVToVURow = AYUVToVURow_Any_SVE2;
+ if (IS_ALIGNED(width, 2)) {
+ AYUVToVURow = AYUVToVURow_SVE2;
+ }
+ }
+#endif
for (y = 0; y < height - 1; y += 2) {
AYUVToVURow(src_ayuv, src_stride_ayuv, dst_vu, width);
@@ -1902,6 +2071,14 @@
}
}
#endif
+#if defined(HAS_ARGBTOYROW_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd)) {
+ ARGBToYRow = ARGBToYRow_Any_NEON_DotProd;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYRow = ARGBToYRow_NEON_DotProd;
+ }
+ }
+#endif
#if defined(HAS_ARGBTOUVROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
ARGBToUVRow = ARGBToUVRow_Any_NEON;
@@ -1910,6 +2087,14 @@
}
}
#endif
+#if defined(HAS_ARGBTOUVROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ ARGBToUVRow = ARGBToUVRow_Any_SVE2;
+ if (IS_ALIGNED(width, 2)) {
+ ARGBToUVRow = ARGBToUVRow_SVE2;
+ }
+ }
+#endif
#if defined(HAS_ARGBTOYROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
ARGBToYRow = ARGBToYRow_Any_SSSE3;
@@ -2064,6 +2249,14 @@
}
}
#endif
+#if defined(HAS_ARGBTOYROW_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd)) {
+ ARGBToYRow = ARGBToYRow_Any_NEON_DotProd;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYRow = ARGBToYRow_NEON_DotProd;
+ }
+ }
+#endif
#if defined(HAS_ARGBTOUVROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
ARGBToUVRow = ARGBToUVRow_Any_NEON;
@@ -2072,6 +2265,14 @@
}
}
#endif
+#if defined(HAS_ARGBTOUVROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ ARGBToUVRow = ARGBToUVRow_Any_SVE2;
+ if (IS_ALIGNED(width, 2)) {
+ ARGBToUVRow = ARGBToUVRow_SVE2;
+ }
+ }
+#endif
#if defined(HAS_ARGBTOYROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
ARGBToYRow = ARGBToYRow_Any_SSSE3;
@@ -2227,6 +2428,14 @@
}
}
#endif
+#if defined(HAS_BGRATOYROW_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd)) {
+ BGRAToYRow = BGRAToYRow_Any_NEON_DotProd;
+ if (IS_ALIGNED(width, 16)) {
+ BGRAToYRow = BGRAToYRow_NEON_DotProd;
+ }
+ }
+#endif
#if defined(HAS_BGRATOUVROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
BGRAToUVRow = BGRAToUVRow_Any_NEON;
@@ -2235,6 +2444,14 @@
}
}
#endif
+#if defined(HAS_BGRATOUVROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ BGRAToUVRow = BGRAToUVRow_Any_SVE2;
+ if (IS_ALIGNED(width, 2)) {
+ BGRAToUVRow = BGRAToUVRow_SVE2;
+ }
+ }
+#endif
#if defined(HAS_BGRATOYROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
BGRAToYRow = BGRAToYRow_Any_SSSE3;
@@ -2386,6 +2603,14 @@
}
}
#endif
+#if defined(HAS_ABGRTOYROW_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd)) {
+ ABGRToYRow = ABGRToYRow_Any_NEON_DotProd;
+ if (IS_ALIGNED(width, 16)) {
+ ABGRToYRow = ABGRToYRow_NEON_DotProd;
+ }
+ }
+#endif
#if defined(HAS_ABGRTOUVROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
ABGRToUVRow = ABGRToUVRow_Any_NEON;
@@ -2394,6 +2619,14 @@
}
}
#endif
+#if defined(HAS_ABGRTOUVROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ ABGRToUVRow = ABGRToUVRow_Any_SVE2;
+ if (IS_ALIGNED(width, 2)) {
+ ABGRToUVRow = ABGRToUVRow_SVE2;
+ }
+ }
+#endif
#if defined(HAS_ABGRTOYROW_MSA) && defined(HAS_ABGRTOUVROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
ABGRToYRow = ABGRToYRow_Any_MSA;
@@ -2495,6 +2728,14 @@
}
}
#endif
+#if defined(HAS_RGBATOYROW_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd)) {
+ RGBAToYRow = RGBAToYRow_Any_NEON_DotProd;
+ if (IS_ALIGNED(width, 16)) {
+ RGBAToYRow = RGBAToYRow_NEON_DotProd;
+ }
+ }
+#endif
#if defined(HAS_RGBATOUVROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
RGBAToUVRow = RGBAToUVRow_Any_NEON;
@@ -2503,6 +2744,14 @@
}
}
#endif
+#if defined(HAS_RGBATOUVROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ RGBAToUVRow = RGBAToUVRow_Any_SVE2;
+ if (IS_ALIGNED(width, 2)) {
+ RGBAToUVRow = RGBAToUVRow_SVE2;
+ }
+ }
+#endif
#if defined(HAS_RGBATOYROW_MSA) && defined(HAS_RGBATOUVROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
RGBAToYRow = RGBAToYRow_Any_MSA;
@@ -3016,6 +3265,14 @@
}
}
#endif
+#if defined(HAS_RAWTOARGBROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ RAWToARGBRow = RAWToARGBRow_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ RAWToARGBRow = RAWToARGBRow_AVX2;
+ }
+ }
+#endif
#if defined(HAS_ARGBTOYROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
ARGBToYRow = ARGBToYRow_Any_SSSE3;
@@ -3161,19 +3418,23 @@
}
}
#endif
-#if defined(HAS_RAWTOYJROW_LSX)
+#if defined(HAS_RAWTOYJROW_LSX) && defined(HAS_RAWTOUVJROW_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
+ RAWToUVJRow = RAWToUVJRow_Any_LSX;
RAWToYJRow = RAWToYJRow_Any_LSX;
if (IS_ALIGNED(width, 16)) {
RAWToYJRow = RAWToYJRow_LSX;
+ RAWToUVJRow = RAWToUVJRow_LSX;
}
}
#endif
-#if defined(HAS_RAWTOYJROW_LASX)
+#if defined(HAS_RAWTOYJROW_LASX) && defined(HAS_RAWTOUVJROW_LASX)
if (TestCpuFlag(kCpuHasLASX)) {
+ RAWToUVJRow = RAWToUVJRow_Any_LASX;
RAWToYJRow = RAWToYJRow_Any_LASX;
if (IS_ALIGNED(width, 32)) {
RAWToYJRow = RAWToYJRow_LASX;
+ RAWToUVJRow = RAWToUVJRow_LASX;
}
}
#endif
@@ -3194,6 +3455,14 @@
}
}
#endif
+#if defined(HAS_RAWTOARGBROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ RAWToARGBRow = RAWToARGBRow_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ RAWToARGBRow = RAWToARGBRow_AVX2;
+ }
+ }
+#endif
#if defined(HAS_ARGBTOYJROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
ARGBToYJRow = ARGBToYJRow_Any_SSSE3;
@@ -3272,6 +3541,461 @@
}
#undef HAS_RAWTOYJROW
+// RAW big endian (rgb in memory) to I444
+// 2 step conversion of RAWToARGB then ARGBToY and ARGBToUV444
+LIBYUV_API
+int RAWToI444(const uint8_t* src_raw,
+ int src_stride_raw,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height) {
+ int y;
+ void (*RAWToARGBRow)(const uint8_t* src_rgb, uint8_t* dst_argb, int width) =
+ RAWToARGBRow_C;
+ void (*ARGBToYRow)(const uint8_t* src_raw, uint8_t* dst_y, int width) =
+ ARGBToYRow_C;
+ void (*ARGBToUV444Row)(const uint8_t* src_raw, uint8_t* dst_u, uint8_t* dst_v,
+ int width) = ARGBToUV444Row_C;
+ if (!src_raw || !dst_y || !dst_u || !dst_v || width <= 0 || height == 0) {
+ return -1;
+ }
+ if (height < 0) {
+ height = -height;
+ src_raw = src_raw + (height - 1) * src_stride_raw;
+ src_stride_raw = -src_stride_raw;
+ }
+ // TODO: add row coalesce when main loop handles large width in blocks
+ // TODO: implement UV444 or trim the ifdef below
+#if defined(HAS_ARGBTOUV444ROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ ARGBToUV444Row = ARGBToUV444Row_Any_SSSE3;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToUV444Row = ARGBToUV444Row_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUV444ROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ARGBToUV444Row = ARGBToUV444Row_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ ARGBToUV444Row = ARGBToUV444Row_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUV444ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ARGBToUV444Row = ARGBToUV444Row_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBToUV444Row = ARGBToUV444Row_NEON;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUV444ROW_NEON_I8MM)
+ if (TestCpuFlag(kCpuHasNeonI8MM)) {
+ ARGBToUV444Row = ARGBToUV444Row_Any_NEON_I8MM;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBToUV444Row = ARGBToUV444Row_NEON_I8MM;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUV444ROW_MSA)
+ if (TestCpuFlag(kCpuHasMSA)) {
+ ARGBToUV444Row = ARGBToUV444Row_Any_MSA;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToUV444Row = ARGBToUV444Row_MSA;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUV444ROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGBToUV444Row = ARGBToUV444Row_Any_LSX;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToUV444Row = ARGBToUV444Row_LSX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUV444ROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGBToUV444Row = ARGBToUV444Row_Any_LASX;
+ if (IS_ALIGNED(width, 32)) {
+ ARGBToUV444Row = ARGBToUV444Row_LASX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ ARGBToYRow = ARGBToYRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYRow = ARGBToYRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ARGBToYRow = ARGBToYRow_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ ARGBToYRow = ARGBToYRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ARGBToYRow = ARGBToYRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYRow = ARGBToYRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYROW_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd)) {
+ ARGBToYRow = ARGBToYRow_Any_NEON_DotProd;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYRow = ARGBToYRow_NEON_DotProd;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYROW_MSA)
+ if (TestCpuFlag(kCpuHasMSA)) {
+ ARGBToYRow = ARGBToYRow_Any_MSA;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYRow = ARGBToYRow_MSA;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGBToYRow = ARGBToYRow_Any_LSX;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYRow = ARGBToYRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGBToYRow = ARGBToYRow_Any_LASX;
+ if (IS_ALIGNED(width, 32)) {
+ ARGBToYRow = ARGBToYRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ARGBToYRow = ARGBToYRow_RVV;
+ }
+#endif
+
+#if defined(HAS_RAWTOARGBROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ RAWToARGBRow = RAWToARGBRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 16)) {
+ RAWToARGBRow = RAWToARGBRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_RAWTOARGBROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ RAWToARGBRow = RAWToARGBRow_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ RAWToARGBRow = RAWToARGBRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_RAWTOARGBROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ RAWToARGBRow = RAWToARGBRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ RAWToARGBRow = RAWToARGBRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_RAWTOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ RAWToARGBRow = RAWToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_RAWTOARGBROW_MSA)
+ if (TestCpuFlag(kCpuHasMSA)) {
+ RAWToARGBRow = RAWToARGBRow_Any_MSA;
+ if (IS_ALIGNED(width, 16)) {
+ RAWToARGBRow = RAWToARGBRow_MSA;
+ }
+ }
+#endif
+#if defined(HAS_RAWTOARGBROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ RAWToARGBRow = RAWToARGBRow_Any_LSX;
+ if (IS_ALIGNED(width, 16)) {
+ RAWToARGBRow = RAWToARGBRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_RAWTOARGBROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ RAWToARGBRow = RAWToARGBRow_Any_LASX;
+ if (IS_ALIGNED(width, 32)) {
+ RAWToARGBRow = RAWToARGBRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_RAWTOARGBROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ RAWToARGBRow = RAWToARGBRow_RVV;
+ }
+#endif
+
+ {
+ // Allocate a row of ARGB.
+ const int row_size = width * 4;
+ align_buffer_64(row, row_size);
+ if (!row)
+ return 1;
+
+ for (y = 0; y < height; ++y) {
+ RAWToARGBRow(src_raw, row, width);
+ ARGBToUV444Row(row, dst_u, dst_v, width);
+ ARGBToYRow(row, dst_y, width);
+ src_raw += src_stride_raw;
+ dst_y += dst_stride_y;
+ dst_u += dst_stride_u;
+ dst_v += dst_stride_v;
+ }
+ free_aligned_buffer_64(row);
+ }
+ return 0;
+}
+
+// RAW big endian (rgb in memory) to J444
+// 2 step conversion of RAWToARGB then ARGBToYJ and ARGBToUVJ444
+LIBYUV_API
+int RAWToJ444(const uint8_t* src_raw,
+ int src_stride_raw,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height) {
+ int y;
+ void (*RAWToARGBRow)(const uint8_t* src_rgb, uint8_t* dst_argb, int width) =
+ RAWToARGBRow_C;
+ void (*ARGBToYJRow)(const uint8_t* src_raw, uint8_t* dst_y, int width) =
+ ARGBToYJRow_C;
+ void (*ARGBToUVJ444Row)(const uint8_t* src_raw, uint8_t* dst_u,
+ uint8_t* dst_v, int width) = ARGBToUVJ444Row_C;
+ if (!src_raw || !dst_y || !dst_u || !dst_v || width <= 0 || height == 0) {
+ return -1;
+ }
+ if (height < 0) {
+ height = -height;
+ src_raw = src_raw + (height - 1) * src_stride_raw;
+ src_stride_raw = -src_stride_raw;
+ }
+ // TODO: add row coalesce when main loop handles large width in blocks
+#if defined(HAS_ARGBTOUVJ444ROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ ARGBToUVJ444Row = ARGBToUVJ444Row_Any_SSSE3;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToUVJ444Row = ARGBToUVJ444Row_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVJ444ROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ARGBToUVJ444Row = ARGBToUVJ444Row_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ ARGBToUVJ444Row = ARGBToUVJ444Row_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVJ444ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ARGBToUVJ444Row = ARGBToUVJ444Row_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBToUVJ444Row = ARGBToUVJ444Row_NEON;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVJ444ROW_NEON_I8MM)
+ if (TestCpuFlag(kCpuHasNeonI8MM)) {
+ ARGBToUVJ444Row = ARGBToUVJ444Row_Any_NEON_I8MM;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBToUVJ444Row = ARGBToUVJ444Row_NEON_I8MM;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVJ444ROW_MSA)
+ if (TestCpuFlag(kCpuHasMSA)) {
+ ARGBToUVJ444Row = ARGBToUVJ444Row_Any_MSA;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToUVJ444Row = ARGBToUVJ444Row_MSA;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVJ444ROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGBToUVJ444Row = ARGBToUVJ444Row_Any_LSX;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToUVJ444Row = ARGBToUVJ444Row_LSX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVJ444ROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGBToUVJ444Row = ARGBToUVJ444Row_Any_LASX;
+ if (IS_ALIGNED(width, 32)) {
+ ARGBToUVJ444Row = ARGBToUVJ444Row_LASX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYJROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ ARGBToYJRow = ARGBToYJRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYJRow = ARGBToYJRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYJROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ARGBToYJRow = ARGBToYJRow_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ ARGBToYJRow = ARGBToYJRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYJROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ARGBToYJRow = ARGBToYJRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYJRow = ARGBToYJRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYJROW_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd)) {
+ ARGBToYJRow = ARGBToYJRow_Any_NEON_DotProd;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYJRow = ARGBToYJRow_NEON_DotProd;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYJROW_MSA)
+ if (TestCpuFlag(kCpuHasMSA)) {
+ ARGBToYJRow = ARGBToYJRow_Any_MSA;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYJRow = ARGBToYJRow_MSA;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYJROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGBToYJRow = ARGBToYJRow_Any_LSX;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYJRow = ARGBToYJRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYJROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGBToYJRow = ARGBToYJRow_Any_LASX;
+ if (IS_ALIGNED(width, 32)) {
+ ARGBToYJRow = ARGBToYJRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYJROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ARGBToYJRow = ARGBToYJRow_RVV;
+ }
+#endif
+
+#if defined(HAS_RAWTOARGBROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ RAWToARGBRow = RAWToARGBRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 16)) {
+ RAWToARGBRow = RAWToARGBRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_RAWTOARGBROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ RAWToARGBRow = RAWToARGBRow_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ RAWToARGBRow = RAWToARGBRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_RAWTOARGBROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ RAWToARGBRow = RAWToARGBRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ RAWToARGBRow = RAWToARGBRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_RAWTOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ RAWToARGBRow = RAWToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_RAWTOARGBROW_MSA)
+ if (TestCpuFlag(kCpuHasMSA)) {
+ RAWToARGBRow = RAWToARGBRow_Any_MSA;
+ if (IS_ALIGNED(width, 16)) {
+ RAWToARGBRow = RAWToARGBRow_MSA;
+ }
+ }
+#endif
+#if defined(HAS_RAWTOARGBROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ RAWToARGBRow = RAWToARGBRow_Any_LSX;
+ if (IS_ALIGNED(width, 16)) {
+ RAWToARGBRow = RAWToARGBRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_RAWTOARGBROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ RAWToARGBRow = RAWToARGBRow_Any_LASX;
+ if (IS_ALIGNED(width, 32)) {
+ RAWToARGBRow = RAWToARGBRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_RAWTOARGBROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ RAWToARGBRow = RAWToARGBRow_RVV;
+ }
+#endif
+
+ {
+ // Allocate a row of ARGB.
+ const int row_size = width * 4;
+ align_buffer_64(row, row_size);
+ if (!row)
+ return 1;
+
+ for (y = 0; y < height; ++y) {
+ RAWToARGBRow(src_raw, row, width);
+ ARGBToUVJ444Row(row, dst_u, dst_v, width);
+ ARGBToYJRow(row, dst_y, width);
+ src_raw += src_stride_raw;
+ dst_y += dst_stride_y;
+ dst_u += dst_stride_u;
+ dst_v += dst_stride_v;
+ }
+ free_aligned_buffer_64(row);
+ }
+ return 0;
+}
+
// Convert RGB565 to I420.
LIBYUV_API
int RGB565ToI420(const uint8_t* src_rgb565,
@@ -3314,18 +4038,22 @@
// Neon version does direct RGB565 to YUV.
#if defined(HAS_RGB565TOYROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- RGB565ToUVRow = RGB565ToUVRow_Any_NEON;
RGB565ToYRow = RGB565ToYRow_Any_NEON;
- if (IS_ALIGNED(width, 8)) {
+ if (IS_ALIGNED(width, 16)) {
RGB565ToYRow = RGB565ToYRow_NEON;
- if (IS_ALIGNED(width, 16)) {
- RGB565ToUVRow = RGB565ToUVRow_NEON;
- }
}
}
+#endif
+// Neon version does direct RGB565 to YUV.
+#if defined(HAS_RGB565TOUVROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ RGB565ToUVRow = RGB565ToUVRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ RGB565ToUVRow = RGB565ToUVRow_NEON;
+ }
+ }
+#endif
// MSA version does direct RGB565 to YUV.
-#elif (defined(HAS_RGB565TOYROW_MSA) || defined(HAS_RGB565TOYROW_LSX) || \
- defined(HAS_RGB565TOYROW_LASX))
#if defined(HAS_RGB565TOYROW_MSA) && defined(HAS_RGB565TOUVROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
RGB565ToUVRow = RGB565ToUVRow_Any_MSA;
@@ -3357,7 +4085,6 @@
}
#endif
// Other platforms do intermediate conversion from RGB565 to ARGB.
-#else
#if defined(HAS_RGB565TOARGBROW_SSE2)
if (TestCpuFlag(kCpuHasSSE2)) {
RGB565ToARGBRow = RGB565ToARGBRow_Any_SSE2;
@@ -3382,14 +4109,6 @@
}
}
#endif
-#if defined(HAS_ARGBTOUVROW_SSSE3)
- if (TestCpuFlag(kCpuHasSSSE3)) {
- ARGBToUVRow = ARGBToUVRow_Any_SSSE3;
- if (IS_ALIGNED(width, 16)) {
- ARGBToUVRow = ARGBToUVRow_SSSE3;
- }
- }
-#endif
#if defined(HAS_ARGBTOYROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
ARGBToYRow = ARGBToYRow_Any_AVX2;
@@ -3398,6 +4117,14 @@
}
}
#endif
+#if defined(HAS_ARGBTOUVROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ ARGBToUVRow = ARGBToUVRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToUVRow = ARGBToUVRow_SSSE3;
+ }
+ }
+#endif
#if defined(HAS_ARGBTOUVROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
ARGBToUVRow = ARGBToUVRow_Any_AVX2;
@@ -3406,7 +4133,6 @@
}
}
#endif
-#endif
{
#if !(defined(HAS_RGB565TOYROW_NEON) || defined(HAS_RGB565TOYROW_MSA) || \
defined(HAS_RGB565TOYROW_LSX) || defined(HAS_RGB565TOYROW_LASX))
@@ -3496,18 +4222,22 @@
// Neon version does direct ARGB1555 to YUV.
#if defined(HAS_ARGB1555TOYROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- ARGB1555ToUVRow = ARGB1555ToUVRow_Any_NEON;
ARGB1555ToYRow = ARGB1555ToYRow_Any_NEON;
if (IS_ALIGNED(width, 8)) {
ARGB1555ToYRow = ARGB1555ToYRow_NEON;
- if (IS_ALIGNED(width, 16)) {
- ARGB1555ToUVRow = ARGB1555ToUVRow_NEON;
- }
}
}
+#endif
+#if defined(HAS_ARGB1555TOUVROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ARGB1555ToUVRow = ARGB1555ToUVRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ ARGB1555ToUVRow = ARGB1555ToUVRow_NEON;
+ }
+ }
+#endif
+
// MSA version does direct ARGB1555 to YUV.
-#elif (defined(HAS_ARGB1555TOYROW_MSA) || defined(HAS_ARGB1555TOYROW_LSX) || \
- defined(HAS_ARGB1555TOYROW_LASX))
#if defined(HAS_ARGB1555TOYROW_MSA) && defined(HAS_ARGB1555TOUVROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
ARGB1555ToUVRow = ARGB1555ToUVRow_Any_MSA;
@@ -3538,8 +4268,8 @@
}
}
#endif
+
// Other platforms do intermediate conversion from ARGB1555 to ARGB.
-#else
#if defined(HAS_ARGB1555TOARGBROW_SSE2)
if (TestCpuFlag(kCpuHasSSE2)) {
ARGB1555ToARGBRow = ARGB1555ToARGBRow_Any_SSE2;
@@ -3588,7 +4318,6 @@
}
}
#endif
-#endif
{
#if !(defined(HAS_ARGB1555TOYROW_NEON) || defined(HAS_ARGB1555TOYROW_MSA) || \
defined(HAS_ARGB1555TOYROW_LSX) || defined(HAS_ARGB1555TOYROW_LASX))
@@ -3680,17 +4409,20 @@
// Neon version does direct ARGB4444 to YUV.
#if defined(HAS_ARGB4444TOYROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- ARGB4444ToUVRow = ARGB4444ToUVRow_Any_NEON;
ARGB4444ToYRow = ARGB4444ToYRow_Any_NEON;
if (IS_ALIGNED(width, 8)) {
ARGB4444ToYRow = ARGB4444ToYRow_NEON;
- if (IS_ALIGNED(width, 16)) {
- ARGB4444ToUVRow = ARGB4444ToUVRow_NEON;
- }
}
}
-// Other platforms do intermediate conversion from ARGB4444 to ARGB.
-#else
+#endif
+#if defined(HAS_ARGB4444TOUVROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ARGB4444ToUVRow = ARGB4444ToUVRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ ARGB4444ToUVRow = ARGB4444ToUVRow_NEON;
+ }
+ }
+#endif
#if defined(HAS_ARGB4444TOARGBROW_SSE2)
if (TestCpuFlag(kCpuHasSSE2)) {
ARGB4444ToARGBRow = ARGB4444ToARGBRow_Any_SSE2;
@@ -3803,7 +4535,6 @@
}
}
#endif
-#endif
{
#if !(defined(HAS_ARGB4444TOYROW_NEON))
@@ -4049,6 +4780,138 @@
dst_stride_v, width, height, kRotate0);
}
+// depth is source bits measured from lsb; For msb use 16
+static int Biplanar16bitTo8bit(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height,
+ int subsample_x,
+ int subsample_y,
+ int depth) {
+ int uv_width = SUBSAMPLE(width, subsample_x, subsample_x);
+ int uv_height = SUBSAMPLE(height, subsample_y, subsample_y);
+ int scale = 1 << (24 - depth);
+ if ((!src_y && dst_y) || !src_uv || !dst_uv || width <= 0 || height == 0) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ uv_height = -uv_height;
+ src_y = src_y + (height - 1) * src_stride_y;
+ src_uv = src_uv + (uv_height - 1) * src_stride_uv;
+ src_stride_y = -src_stride_y;
+ src_stride_uv = -src_stride_uv;
+ }
+
+ // Convert Y plane.
+ if (dst_y) {
+ Convert16To8Plane(src_y, src_stride_y, dst_y, dst_stride_y, scale, width,
+ height);
+ }
+ // Convert UV planes.
+ Convert16To8Plane(src_uv, src_stride_uv, dst_uv, dst_stride_uv, scale,
+ uv_width * 2, uv_height);
+ return 0;
+}
+
+// Convert 10 bit P010 to 8 bit NV12.
+// Depth set to 16 because P010 uses 10 msb and this function keeps the upper 8
+// bits of the specified number of bits.
+LIBYUV_API
+int P010ToNV12(const uint16_t* src_y,
+ int src_stride_y,
+ const uint16_t* src_uv,
+ int src_stride_uv,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_uv,
+ int dst_stride_uv,
+ int width,
+ int height) {
+ return Biplanar16bitTo8bit(src_y, src_stride_y, src_uv, src_stride_uv, dst_y,
+ dst_stride_y, dst_uv, dst_stride_uv, width, height,
+ 1, 1, 16);
+}
+
+static int Planar8bitTo8bit(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height,
+ int subsample_x,
+ int subsample_y,
+ int scale_y,
+ int bias_y,
+ int scale_uv,
+ int bias_uv) {
+ int uv_width = SUBSAMPLE(width, subsample_x, subsample_x);
+ int uv_height = SUBSAMPLE(height, subsample_y, subsample_y);
+ if ((!src_y && dst_y) || !src_u || !src_v || !dst_u || !dst_v || width <= 0 ||
+ height == 0) {
+ return -1;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ uv_height = -uv_height;
+ src_y = src_y + (height - 1) * src_stride_y;
+ src_u = src_u + (uv_height - 1) * src_stride_u;
+ src_v = src_v + (uv_height - 1) * src_stride_v;
+ src_stride_y = -src_stride_y;
+ src_stride_u = -src_stride_u;
+ src_stride_v = -src_stride_v;
+ }
+
+ // Convert Y plane.
+ if (dst_y) {
+ Convert8To8Plane(src_y, src_stride_y, dst_y, dst_stride_y, scale_y, bias_y,
+ width, height);
+ }
+ // Convert UV planes.
+ Convert8To8Plane(src_u, src_stride_u, dst_u, dst_stride_u, scale_uv, bias_uv,
+ uv_width, uv_height);
+ Convert8To8Plane(src_v, src_stride_v, dst_v, dst_stride_v, scale_uv, bias_uv,
+ uv_width, uv_height);
+ return 0;
+}
+
+LIBYUV_API
+int J420ToI420(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height) {
+ return Planar8bitTo8bit(src_y, src_stride_y, src_u, src_stride_u, src_v,
+ src_stride_v, dst_y, dst_stride_y, dst_u,
+ dst_stride_u, dst_v, dst_stride_v, width, height, 1,
+ 1, 220, 16, 225, 16);
+}
+
#ifdef __cplusplus
} // extern "C"
} // namespace libyuv
diff --git a/source/convert_argb.cc b/source/convert_argb.cc
index 871fea5..41997fe 100644
--- a/source/convert_argb.cc
+++ b/source/convert_argb.cc
@@ -14,9 +14,6 @@
#include "libyuv/convert_from_argb.h"
#include "libyuv/cpu_id.h"
-#ifdef HAVE_JPEG
-#include "libyuv/mjpeg_decoder.h"
-#endif
#include "libyuv/planar_functions.h" // For CopyPlane and ARGBShuffle.
#include "libyuv/rotate_argb.h"
#include "libyuv/row.h"
@@ -112,6 +109,16 @@
}
}
#endif
+#if defined(HAS_I422TOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I422ToARGBRow = I422ToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_I422TOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I422ToARGBRow = I422ToARGBRow_SME;
+ }
+#endif
#if defined(HAS_I422TOARGBROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
I422ToARGBRow = I422ToARGBRow_Any_MSA;
@@ -366,6 +373,16 @@
}
}
#endif
+#if defined(HAS_I422TOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I422ToARGBRow = I422ToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_I422TOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I422ToARGBRow = I422ToARGBRow_SME;
+ }
+#endif
#if defined(HAS_I422TOARGBROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
I422ToARGBRow = I422ToARGBRow_Any_MSA;
@@ -609,6 +626,16 @@
}
}
#endif
+#if defined(HAS_I444TOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I444ToARGBRow = I444ToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_I444TOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I444ToARGBRow = I444ToARGBRow_SME;
+ }
+#endif
#if defined(HAS_I444TOARGBROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
I444ToARGBRow = I444ToARGBRow_Any_MSA;
@@ -942,6 +969,24 @@
dst_ar30 = dst_ar30 + (height - 1) * dst_stride_ar30;
dst_stride_ar30 = -dst_stride_ar30;
}
+#if defined(HAS_I210TOAR30ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ I210ToAR30Row = I210ToAR30Row_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ I210ToAR30Row = I210ToAR30Row_NEON;
+ }
+ }
+#endif
+#if defined(HAS_I210TOAR30ROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I210ToAR30Row = I210ToAR30Row_SVE2;
+ }
+#endif
+#if defined(HAS_I210TOAR30ROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I210ToAR30Row = I210ToAR30Row_SME;
+ }
+#endif
#if defined(HAS_I210TOAR30ROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
I210ToAR30Row = I210ToAR30Row_Any_SSSE3;
@@ -1118,6 +1163,24 @@
}
}
#endif
+#if defined(HAS_I212TOAR30ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ I212ToAR30Row = I212ToAR30Row_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ I212ToAR30Row = I212ToAR30Row_NEON;
+ }
+ }
+#endif
+#if defined(HAS_I212TOAR30ROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I212ToAR30Row = I212ToAR30Row_SVE2;
+ }
+#endif
+#if defined(HAS_I212TOAR30ROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I212ToAR30Row = I212ToAR30Row_SME;
+ }
+#endif
for (y = 0; y < height; ++y) {
I212ToAR30Row(src_y, src_u, src_v, dst_ar30, yuvconstants, width);
dst_ar30 += dst_stride_ar30;
@@ -1160,6 +1223,24 @@
dst_ar30 = dst_ar30 + (height - 1) * dst_stride_ar30;
dst_stride_ar30 = -dst_stride_ar30;
}
+#if defined(HAS_I210TOAR30ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ I210ToAR30Row = I210ToAR30Row_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ I210ToAR30Row = I210ToAR30Row_NEON;
+ }
+ }
+#endif
+#if defined(HAS_I210TOAR30ROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I210ToAR30Row = I210ToAR30Row_SVE2;
+ }
+#endif
+#if defined(HAS_I210TOAR30ROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I210ToAR30Row = I210ToAR30Row_SME;
+ }
+#endif
#if defined(HAS_I210TOAR30ROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
I210ToAR30Row = I210ToAR30Row_Any_SSSE3;
@@ -1315,6 +1396,24 @@
dst_ar30 = dst_ar30 + (height - 1) * dst_stride_ar30;
dst_stride_ar30 = -dst_stride_ar30;
}
+#if defined(HAS_I410TOAR30ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ I410ToAR30Row = I410ToAR30Row_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ I410ToAR30Row = I410ToAR30Row_NEON;
+ }
+ }
+#endif
+#if defined(HAS_I410TOAR30ROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I410ToAR30Row = I410ToAR30Row_SVE2;
+ }
+#endif
+#if defined(HAS_I410TOAR30ROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I410ToAR30Row = I410ToAR30Row_SME;
+ }
+#endif
#if defined(HAS_I410TOAR30ROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
I410ToAR30Row = I410ToAR30Row_Any_SSSE3;
@@ -1377,6 +1476,24 @@
}
}
#endif
+#if defined(HAS_I210TOARGBROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ I210ToARGBRow = I210ToARGBRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ I210ToARGBRow = I210ToARGBRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_I210TOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I210ToARGBRow = I210ToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_I210TOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I210ToARGBRow = I210ToARGBRow_SME;
+ }
+#endif
#if defined(HAS_I210TOARGBROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
I210ToARGBRow = I210ToARGBRow_Any_AVX2;
@@ -1549,6 +1666,24 @@
}
}
#endif
+#if defined(HAS_I212TOARGBROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ I212ToARGBRow = I212ToARGBRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ I212ToARGBRow = I212ToARGBRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_I212TOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I212ToARGBRow = I212ToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_I212TOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I212ToARGBRow = I212ToARGBRow_SME;
+ }
+#endif
for (y = 0; y < height; ++y) {
I212ToARGBRow(src_y, src_u, src_v, dst_argb, yuvconstants, width);
dst_argb += dst_stride_argb;
@@ -1597,6 +1732,24 @@
}
}
#endif
+#if defined(HAS_I210TOARGBROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ I210ToARGBRow = I210ToARGBRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ I210ToARGBRow = I210ToARGBRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_I210TOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I210ToARGBRow = I210ToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_I210TOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I210ToARGBRow = I210ToARGBRow_SME;
+ }
+#endif
#if defined(HAS_I210TOARGBROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
I210ToARGBRow = I210ToARGBRow_Any_AVX2;
@@ -1758,6 +1911,24 @@
}
}
#endif
+#if defined(HAS_I410TOARGBROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ I410ToARGBRow = I410ToARGBRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ I410ToARGBRow = I410ToARGBRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_I410TOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I410ToARGBRow = I410ToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_I410TOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I410ToARGBRow = I410ToARGBRow_SME;
+ }
+#endif
#if defined(HAS_I410TOARGBROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
I410ToARGBRow = I410ToARGBRow_Any_AVX2;
@@ -1816,6 +1987,24 @@
}
}
#endif
+#if defined(HAS_P210TOARGBROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ P210ToARGBRow = P210ToARGBRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ P210ToARGBRow = P210ToARGBRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_P210TOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ P210ToARGBRow = P210ToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_P210TOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ P210ToARGBRow = P210ToARGBRow_SME;
+ }
+#endif
for (y = 0; y < height; ++y) {
P210ToARGBRow(src_y, src_uv, dst_argb, yuvconstants, width);
dst_argb += dst_stride_argb;
@@ -1867,6 +2056,24 @@
}
}
#endif
+#if defined(HAS_P210TOARGBROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ P210ToARGBRow = P210ToARGBRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ P210ToARGBRow = P210ToARGBRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_P210TOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ P210ToARGBRow = P210ToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_P210TOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ P210ToARGBRow = P210ToARGBRow_SME;
+ }
+#endif
for (y = 0; y < height; ++y) {
P210ToARGBRow(src_y, src_uv, dst_argb, yuvconstants, width);
dst_argb += dst_stride_argb;
@@ -1916,6 +2123,24 @@
}
}
#endif
+#if defined(HAS_P210TOAR30ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ P210ToAR30Row = P210ToAR30Row_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ P210ToAR30Row = P210ToAR30Row_NEON;
+ }
+ }
+#endif
+#if defined(HAS_P210TOAR30ROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ P210ToAR30Row = P210ToAR30Row_SVE2;
+ }
+#endif
+#if defined(HAS_P210TOAR30ROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ P210ToAR30Row = P210ToAR30Row_SME;
+ }
+#endif
for (y = 0; y < height; ++y) {
P210ToAR30Row(src_y, src_uv, dst_ar30, yuvconstants, width);
dst_ar30 += dst_stride_ar30;
@@ -1967,6 +2192,24 @@
}
}
#endif
+#if defined(HAS_P210TOAR30ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ P210ToAR30Row = P210ToAR30Row_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ P210ToAR30Row = P210ToAR30Row_NEON;
+ }
+ }
+#endif
+#if defined(HAS_P210TOAR30ROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ P210ToAR30Row = P210ToAR30Row_SVE2;
+ }
+#endif
+#if defined(HAS_P210TOAR30ROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ P210ToAR30Row = P210ToAR30Row_SME;
+ }
+#endif
for (y = 0; y < height; ++y) {
P210ToAR30Row(src_y, src_uv, dst_ar30, yuvconstants, width);
dst_ar30 += dst_stride_ar30;
@@ -2035,6 +2278,16 @@
}
}
#endif
+#if defined(HAS_I422ALPHATOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I422AlphaToARGBRow = I422AlphaToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_I422ALPHATOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I422AlphaToARGBRow = I422AlphaToARGBRow_SME;
+ }
+#endif
#if defined(HAS_I422ALPHATOARGBROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
I422AlphaToARGBRow = I422AlphaToARGBRow_Any_MSA;
@@ -2101,6 +2354,22 @@
ARGBAttenuateRow = ARGBAttenuateRow_RVV;
}
#endif
+#if defined(HAS_ARGBATTENUATEROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_LSX;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_LASX;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_LASX;
+ }
+ }
+#endif
for (y = 0; y < height; ++y) {
I422AlphaToARGBRow(src_y, src_u, src_v, src_a, dst_argb, yuvconstants,
@@ -2178,6 +2447,16 @@
}
}
#endif
+#if defined(HAS_I422ALPHATOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I422AlphaToARGBRow = I422AlphaToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_I422ALPHATOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I422AlphaToARGBRow = I422AlphaToARGBRow_SME;
+ }
+#endif
#if defined(HAS_I422ALPHATOARGBROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
I422AlphaToARGBRow = I422AlphaToARGBRow_Any_MSA;
@@ -2244,6 +2523,22 @@
ARGBAttenuateRow = ARGBAttenuateRow_RVV;
}
#endif
+#if defined(HAS_ARGBATTENUATEROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_LSX;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_LASX;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_LASX;
+ }
+ }
+#endif
for (y = 0; y < height; ++y) {
I422AlphaToARGBRow(src_y, src_u, src_v, src_a, dst_argb, yuvconstants,
@@ -2319,6 +2614,16 @@
}
}
#endif
+#if defined(HAS_I444ALPHATOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I444AlphaToARGBRow = I444AlphaToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_I444ALPHATOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I444AlphaToARGBRow = I444AlphaToARGBRow_SME;
+ }
+#endif
#if defined(HAS_I444ALPHATOARGBROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
I444AlphaToARGBRow = I444AlphaToARGBRow_Any_MSA;
@@ -2369,6 +2674,22 @@
ARGBAttenuateRow = ARGBAttenuateRow_RVV;
}
#endif
+#if defined(HAS_ARGBATTENUATEROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_LSX;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_LASX;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_LASX;
+ }
+ }
+#endif
for (y = 0; y < height; ++y) {
I444AlphaToARGBRow(src_y, src_u, src_v, src_a, dst_argb, yuvconstants,
@@ -2549,6 +2870,24 @@
dst_argb = dst_argb + (height - 1) * dst_stride_argb;
dst_stride_argb = -dst_stride_argb;
}
+#if defined(HAS_I210ALPHATOARGBROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ I210AlphaToARGBRow = I210AlphaToARGBRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ I210AlphaToARGBRow = I210AlphaToARGBRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_I210ALPHATOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I210AlphaToARGBRow = I210AlphaToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_I210ALPHATOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I210AlphaToARGBRow = I210AlphaToARGBRow_SME;
+ }
+#endif
#if defined(HAS_I210ALPHATOARGBROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
I210AlphaToARGBRow = I210AlphaToARGBRow_Any_SSSE3;
@@ -2602,6 +2941,22 @@
ARGBAttenuateRow = ARGBAttenuateRow_RVV;
}
#endif
+#if defined(HAS_ARGBATTENUATEROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_LSX;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_LASX;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_LASX;
+ }
+ }
+#endif
for (y = 0; y < height; ++y) {
I210AlphaToARGBRow(src_y, src_u, src_v, src_a, dst_argb, yuvconstants,
@@ -2655,6 +3010,24 @@
dst_argb = dst_argb + (height - 1) * dst_stride_argb;
dst_stride_argb = -dst_stride_argb;
}
+#if defined(HAS_I210ALPHATOARGBROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ I210AlphaToARGBRow = I210AlphaToARGBRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ I210AlphaToARGBRow = I210AlphaToARGBRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_I210ALPHATOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I210AlphaToARGBRow = I210AlphaToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_I210ALPHATOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I210AlphaToARGBRow = I210AlphaToARGBRow_SME;
+ }
+#endif
#if defined(HAS_I210ALPHATOARGBROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
I210AlphaToARGBRow = I210AlphaToARGBRow_Any_SSSE3;
@@ -2708,6 +3081,22 @@
ARGBAttenuateRow = ARGBAttenuateRow_RVV;
}
#endif
+#if defined(HAS_ARGBATTENUATEROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_LSX;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_LASX;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_LASX;
+ }
+ }
+#endif
for (y = 0; y < height; ++y) {
I210AlphaToARGBRow(src_y, src_u, src_v, src_a, dst_argb, yuvconstants,
@@ -2759,6 +3148,24 @@
dst_argb = dst_argb + (height - 1) * dst_stride_argb;
dst_stride_argb = -dst_stride_argb;
}
+#if defined(HAS_I410ALPHATOARGBROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ I410AlphaToARGBRow = I410AlphaToARGBRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ I410AlphaToARGBRow = I410AlphaToARGBRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_I410ALPHATOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I410AlphaToARGBRow = I410AlphaToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_I410ALPHATOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I410AlphaToARGBRow = I410AlphaToARGBRow_SME;
+ }
+#endif
#if defined(HAS_I410ALPHATOARGBROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
I410AlphaToARGBRow = I410AlphaToARGBRow_Any_SSSE3;
@@ -2812,6 +3219,22 @@
ARGBAttenuateRow = ARGBAttenuateRow_RVV;
}
#endif
+#if defined(HAS_ARGBATTENUATEROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_LSX;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_LASX;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_LASX;
+ }
+ }
+#endif
for (y = 0; y < height; ++y) {
I410AlphaToARGBRow(src_y, src_u, src_v, src_a, dst_argb, yuvconstants,
@@ -2881,6 +3304,16 @@
}
}
#endif
+#if defined(HAS_I400TOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I400ToARGBRow = I400ToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_I400TOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I400ToARGBRow = I400ToARGBRow_SME;
+ }
+#endif
#if defined(HAS_I400TOARGBROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
I400ToARGBRow = I400ToARGBRow_Any_MSA;
@@ -3323,6 +3756,11 @@
}
}
#endif
+#if defined(HAS_RGB24TOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ RGB24ToARGBRow = RGB24ToARGBRow_SVE2;
+ }
+#endif
#if defined(HAS_RGB24TOARGBROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
RGB24ToARGBRow = RGB24ToARGBRow_Any_MSA;
@@ -3395,6 +3833,14 @@
}
}
#endif
+#if defined(HAS_RAWTOARGBROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ RAWToARGBRow = RAWToARGBRow_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ RAWToARGBRow = RAWToARGBRow_AVX2;
+ }
+ }
+#endif
#if defined(HAS_RAWTOARGBROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
RAWToARGBRow = RAWToARGBRow_Any_NEON;
@@ -3403,6 +3849,11 @@
}
}
#endif
+#if defined(HAS_RAWTOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ RAWToARGBRow = RAWToARGBRow_SVE2;
+ }
+#endif
#if defined(HAS_RAWTOARGBROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
RAWToARGBRow = RAWToARGBRow_Any_MSA;
@@ -3483,6 +3934,11 @@
}
}
#endif
+#if defined(HAS_RAWTORGBAROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ RAWToRGBARow = RAWToRGBARow_SVE2;
+ }
+#endif
#if defined(HAS_RAWTORGBAROW_RVV)
if (TestCpuFlag(kCpuHasRVV)) {
RAWToRGBARow = RAWToRGBARow_RVV;
@@ -3542,7 +3998,7 @@
#if defined(HAS_RGB565TOARGBROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
RGB565ToARGBRow = RGB565ToARGBRow_Any_NEON;
- if (IS_ALIGNED(width, 8)) {
+ if (IS_ALIGNED(width, 16)) {
RGB565ToARGBRow = RGB565ToARGBRow_NEON;
}
}
@@ -3625,11 +4081,16 @@
#if defined(HAS_ARGB1555TOARGBROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
ARGB1555ToARGBRow = ARGB1555ToARGBRow_Any_NEON;
- if (IS_ALIGNED(width, 8)) {
+ if (IS_ALIGNED(width, 16)) {
ARGB1555ToARGBRow = ARGB1555ToARGBRow_NEON;
}
}
#endif
+#if defined(HAS_ARGB1555TOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ ARGB1555ToARGBRow = ARGB1555ToARGBRow_SVE2;
+ }
+#endif
#if defined(HAS_ARGB1555TOARGBROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
ARGB1555ToARGBRow = ARGB1555ToARGBRow_Any_MSA;
@@ -4019,6 +4480,16 @@
}
}
#endif
+#if defined(HAS_NV12TOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ NV12ToARGBRow = NV12ToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_NV12TOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ NV12ToARGBRow = NV12ToARGBRow_SME;
+ }
+#endif
#if defined(HAS_NV12TOARGBROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
NV12ToARGBRow = NV12ToARGBRow_Any_MSA;
@@ -4109,6 +4580,16 @@
}
}
#endif
+#if defined(HAS_NV21TOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ NV21ToARGBRow = NV21ToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_NV21TOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ NV21ToARGBRow = NV21ToARGBRow_SME;
+ }
+#endif
#if defined(HAS_NV21TOARGBROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
NV21ToARGBRow = NV21ToARGBRow_Any_MSA;
@@ -4242,6 +4723,16 @@
}
}
#endif
+#if defined(HAS_NV12TORGB24ROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ NV12ToRGB24Row = NV12ToRGB24Row_SVE2;
+ }
+#endif
+#if defined(HAS_NV12TORGB24ROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ NV12ToRGB24Row = NV12ToRGB24Row_SME;
+ }
+#endif
#if defined(HAS_NV12TORGB24ROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
NV12ToRGB24Row = NV12ToRGB24Row_Any_SSSE3;
@@ -4308,6 +4799,16 @@
}
}
#endif
+#if defined(HAS_NV21TORGB24ROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ NV21ToRGB24Row = NV21ToRGB24Row_SVE2;
+ }
+#endif
+#if defined(HAS_NV21TORGB24ROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ NV21ToRGB24Row = NV21ToRGB24Row_SME;
+ }
+#endif
#if defined(HAS_NV21TORGB24ROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
NV21ToRGB24Row = NV21ToRGB24Row_Any_SSSE3;
@@ -4455,14 +4956,15 @@
return 0;
}
-// Convert YUY2 to ARGB.
+// Convert YUY2 to ARGB with matrix.
LIBYUV_API
-int YUY2ToARGB(const uint8_t* src_yuy2,
- int src_stride_yuy2,
- uint8_t* dst_argb,
- int dst_stride_argb,
- int width,
- int height) {
+int YUY2ToARGBMatrix(const uint8_t* src_yuy2,
+ int src_stride_yuy2,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height) {
int y;
void (*YUY2ToARGBRow)(const uint8_t* src_yuy2, uint8_t* dst_argb,
const struct YuvConstants* yuvconstants, int width) =
@@ -4506,6 +5008,16 @@
}
}
#endif
+#if defined(HAS_YUY2TOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ YUY2ToARGBRow = YUY2ToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_YUY2TOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ YUY2ToARGBRow = YUY2ToARGBRow_SME;
+ }
+#endif
#if defined(HAS_YUY2TOARGBROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
YUY2ToARGBRow = YUY2ToARGBRow_Any_MSA;
@@ -4523,21 +5035,34 @@
}
#endif
for (y = 0; y < height; ++y) {
- YUY2ToARGBRow(src_yuy2, dst_argb, &kYuvI601Constants, width);
+ YUY2ToARGBRow(src_yuy2, dst_argb, yuvconstants, width);
src_yuy2 += src_stride_yuy2;
dst_argb += dst_stride_argb;
}
return 0;
}
-// Convert UYVY to ARGB.
+// Convert YUY2 to ARGB.
LIBYUV_API
-int UYVYToARGB(const uint8_t* src_uyvy,
- int src_stride_uyvy,
+int YUY2ToARGB(const uint8_t* src_yuy2,
+ int src_stride_yuy2,
uint8_t* dst_argb,
int dst_stride_argb,
int width,
int height) {
+ return YUY2ToARGBMatrix(src_yuy2, src_stride_yuy2, dst_argb, dst_stride_argb,
+ &kYuvI601Constants, width, height);
+}
+
+// Convert UYVY to ARGB with matrix.
+LIBYUV_API
+int UYVYToARGBMatrix(const uint8_t* src_uyvy,
+ int src_stride_uyvy,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ const struct YuvConstants* yuvconstants,
+ int width,
+ int height) {
int y;
void (*UYVYToARGBRow)(const uint8_t* src_uyvy, uint8_t* dst_argb,
const struct YuvConstants* yuvconstants, int width) =
@@ -4581,6 +5106,16 @@
}
}
#endif
+#if defined(HAS_UYVYTOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ UYVYToARGBRow = UYVYToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_UYVYTOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ UYVYToARGBRow = UYVYToARGBRow_SME;
+ }
+#endif
#if defined(HAS_UYVYTOARGBROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
UYVYToARGBRow = UYVYToARGBRow_Any_MSA;
@@ -4598,12 +5133,25 @@
}
#endif
for (y = 0; y < height; ++y) {
- UYVYToARGBRow(src_uyvy, dst_argb, &kYuvI601Constants, width);
+ UYVYToARGBRow(src_uyvy, dst_argb, yuvconstants, width);
src_uyvy += src_stride_uyvy;
dst_argb += dst_stride_argb;
}
return 0;
}
+
+// Convert UYVY to ARGB.
+LIBYUV_API
+int UYVYToARGB(const uint8_t* src_uyvy,
+ int src_stride_uyvy,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ int width,
+ int height) {
+ return UYVYToARGBMatrix(src_uyvy, src_stride_uyvy, dst_argb, dst_stride_argb,
+ &kYuvI601Constants, width, height);
+}
+
static void WeavePixels(const uint8_t* src_u,
const uint8_t* src_v,
int src_pixel_stride_uv,
@@ -4775,6 +5323,16 @@
}
}
#endif
+#if defined(HAS_I422TORGBAROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I422ToRGBARow = I422ToRGBARow_SVE2;
+ }
+#endif
+#if defined(HAS_I422TORGBAROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I422ToRGBARow = I422ToRGBARow_SME;
+ }
+#endif
#if defined(HAS_I422TORGBAROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
I422ToRGBARow = I422ToRGBARow_Any_MSA;
@@ -5003,6 +5561,16 @@
}
}
#endif
+#if defined(HAS_I422TORGBAROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I422ToRGBARow = I422ToRGBARow_SVE2;
+ }
+#endif
+#if defined(HAS_I422TORGBAROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I422ToRGBARow = I422ToRGBARow_SME;
+ }
+#endif
#if defined(HAS_I422TORGBAROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
I422ToRGBARow = I422ToRGBARow_Any_MSA;
@@ -5133,6 +5701,16 @@
}
}
#endif
+#if defined(HAS_I422TORGB24ROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I422ToRGB24Row = I422ToRGB24Row_SVE2;
+ }
+#endif
+#if defined(HAS_I422TORGB24ROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I422ToRGB24Row = I422ToRGB24Row_SME;
+ }
+#endif
#if defined(HAS_I422TORGB24ROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
I422ToRGB24Row = I422ToRGB24Row_Any_MSA;
@@ -5335,6 +5913,16 @@
}
}
#endif
+#if defined(HAS_I422TORGB24ROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I422ToRGB24Row = I422ToRGB24Row_SVE2;
+ }
+#endif
+#if defined(HAS_I422TORGB24ROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I422ToRGB24Row = I422ToRGB24Row_SME;
+ }
+#endif
#if defined(HAS_I422TORGB24ROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
I422ToRGB24Row = I422ToRGB24Row_Any_MSA;
@@ -5462,6 +6050,16 @@
}
}
#endif
+#if defined(HAS_I422TOARGB1555ROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I422ToARGB1555Row = I422ToARGB1555Row_SVE2;
+ }
+#endif
+#if defined(HAS_I422TOARGB1555ROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I422ToARGB1555Row = I422ToARGB1555Row_SME;
+ }
+#endif
#if defined(HAS_I422TOARGB1555ROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
I422ToARGB1555Row = I422ToARGB1555Row_Any_MSA;
@@ -5551,6 +6149,16 @@
}
}
#endif
+#if defined(HAS_I422TOARGB4444ROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I422ToARGB4444Row = I422ToARGB4444Row_SVE2;
+ }
+#endif
+#if defined(HAS_I422TOARGB4444ROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I422ToARGB4444Row = I422ToARGB4444Row_SME;
+ }
+#endif
#if defined(HAS_I422TOARGB4444ROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
I422ToARGB4444Row = I422ToARGB4444Row_Any_MSA;
@@ -5641,6 +6249,16 @@
}
}
#endif
+#if defined(HAS_I422TORGB565ROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I422ToRGB565Row = I422ToRGB565Row_SVE2;
+ }
+#endif
+#if defined(HAS_I422TORGB565ROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I422ToRGB565Row = I422ToRGB565Row_SME;
+ }
+#endif
#if defined(HAS_I422TORGB565ROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
I422ToRGB565Row = I422ToRGB565Row_Any_MSA;
@@ -5781,6 +6399,16 @@
}
}
#endif
+#if defined(HAS_I422TORGB565ROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I422ToRGB565Row = I422ToRGB565Row_SVE2;
+ }
+#endif
+#if defined(HAS_I422TORGB565ROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I422ToRGB565Row = I422ToRGB565Row_SME;
+ }
+#endif
#if defined(HAS_I422TORGB565ROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
I422ToRGB565Row = I422ToRGB565Row_Any_MSA;
@@ -5904,6 +6532,16 @@
}
}
#endif
+#if defined(HAS_I422TOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I422ToARGBRow = I422ToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_I422TOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I422ToARGBRow = I422ToARGBRow_SME;
+ }
+#endif
#if defined(HAS_I422TOARGBROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
I422ToARGBRow = I422ToARGBRow_Any_MSA;
@@ -5957,6 +6595,11 @@
}
}
#endif
+#if defined(HAS_ARGBTORGB565DITHERROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_SVE2;
+ }
+#endif
#if defined(HAS_ARGBTORGB565DITHERROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_Any_MSA;
@@ -6049,6 +6692,14 @@
}
}
#endif
+#if defined(HAS_I422TOAR30ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ I422ToAR30Row = I422ToAR30Row_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ I422ToAR30Row = I422ToAR30Row_NEON;
+ }
+ }
+#endif
for (y = 0; y < height; ++y) {
I422ToAR30Row(src_y, src_u, src_v, dst_ar30, yuvconstants, width);
@@ -6185,6 +6836,16 @@
}
}
#endif
+#if defined(HAS_I444TOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I444ToARGBRow = I444ToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_I444TOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I444ToARGBRow = I444ToARGBRow_SME;
+ }
+#endif
#if defined(HAS_I444TOARGBROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
I444ToARGBRow = I444ToARGBRow_Any_MSA;
@@ -6332,6 +6993,16 @@
}
}
#endif
+#if defined(HAS_I444TOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I444ToARGBRow = I444ToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_I444TOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I444ToARGBRow = I444ToARGBRow_SME;
+ }
+#endif
#if defined(HAS_I444TOARGBROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
I444ToARGBRow = I444ToARGBRow_Any_MSA;
@@ -6582,6 +7253,24 @@
dst_ar30 = dst_ar30 + (height - 1) * dst_stride_ar30;
dst_stride_ar30 = -dst_stride_ar30;
}
+#if defined(HAS_I410TOAR30ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ I410ToAR30Row = I410ToAR30Row_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ I410ToAR30Row = I410ToAR30Row_NEON;
+ }
+ }
+#endif
+#if defined(HAS_I410TOAR30ROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I410ToAR30Row = I410ToAR30Row_SVE2;
+ }
+#endif
+#if defined(HAS_I410TOAR30ROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I410ToAR30Row = I410ToAR30Row_SME;
+ }
+#endif
#if defined(HAS_I410TOAR30ROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
I410ToAR30Row = I410ToAR30Row_Any_SSSE3;
@@ -6688,6 +7377,24 @@
dst_ar30 = dst_ar30 + (height - 1) * dst_stride_ar30;
dst_stride_ar30 = -dst_stride_ar30;
}
+#if defined(HAS_I410TOAR30ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ I410ToAR30Row = I410ToAR30Row_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ I410ToAR30Row = I410ToAR30Row_NEON;
+ }
+ }
+#endif
+#if defined(HAS_I410TOAR30ROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I410ToAR30Row = I410ToAR30Row_SVE2;
+ }
+#endif
+#if defined(HAS_I410TOAR30ROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I410ToAR30Row = I410ToAR30Row_SME;
+ }
+#endif
#if defined(HAS_I410TOAR30ROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
I410ToAR30Row = I410ToAR30Row_Any_SSSE3;
@@ -6781,6 +7488,24 @@
}
}
#endif
+#if defined(HAS_I410TOARGBROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ I410ToARGBRow = I410ToARGBRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ I410ToARGBRow = I410ToARGBRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_I410TOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I410ToARGBRow = I410ToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_I410TOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I410ToARGBRow = I410ToARGBRow_SME;
+ }
+#endif
#if defined(HAS_I410TOARGBROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
I410ToARGBRow = I410ToARGBRow_Any_AVX2;
@@ -6886,6 +7611,24 @@
}
}
#endif
+#if defined(HAS_I410TOARGBROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ I410ToARGBRow = I410ToARGBRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ I410ToARGBRow = I410ToARGBRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_I410TOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I410ToARGBRow = I410ToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_I410TOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I410ToARGBRow = I410ToARGBRow_SME;
+ }
+#endif
#if defined(HAS_I410TOARGBROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
I410ToARGBRow = I410ToARGBRow_Any_AVX2;
@@ -6996,6 +7739,16 @@
}
}
#endif
+#if defined(HAS_I444ALPHATOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I444AlphaToARGBRow = I444AlphaToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_I444ALPHATOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I444AlphaToARGBRow = I444AlphaToARGBRow_SME;
+ }
+#endif
#if defined(HAS_I444ALPHATOARGBROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
I444AlphaToARGBRow = I444AlphaToARGBRow_Any_MSA;
@@ -7054,6 +7807,22 @@
ARGBAttenuateRow = ARGBAttenuateRow_RVV;
}
#endif
+#if defined(HAS_ARGBATTENUATEROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_LSX;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_LASX;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_LASX;
+ }
+ }
+#endif
#if defined(HAS_SCALEROWUP2_BILINEAR_SSE2)
if (TestCpuFlag(kCpuHasSSE2)) {
@@ -7206,6 +7975,16 @@
}
}
#endif
+#if defined(HAS_I444ALPHATOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I444AlphaToARGBRow = I444AlphaToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_I444ALPHATOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I444AlphaToARGBRow = I444AlphaToARGBRow_SME;
+ }
+#endif
#if defined(HAS_I444ALPHATOARGBROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
I444AlphaToARGBRow = I444AlphaToARGBRow_Any_MSA;
@@ -7264,6 +8043,22 @@
ARGBAttenuateRow = ARGBAttenuateRow_RVV;
}
#endif
+#if defined(HAS_ARGBATTENUATEROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_LSX;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_LASX;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_LASX;
+ }
+ }
+#endif
#if defined(HAS_SCALEROWUP2_LINEAR_SSE2)
if (TestCpuFlag(kCpuHasSSE2)) {
@@ -7357,6 +8152,24 @@
dst_argb = dst_argb + (height - 1) * dst_stride_argb;
dst_stride_argb = -dst_stride_argb;
}
+#if defined(HAS_I410ALPHATOARGBROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ I410AlphaToARGBRow = I410AlphaToARGBRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ I410AlphaToARGBRow = I410AlphaToARGBRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_I410ALPHATOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I410AlphaToARGBRow = I410AlphaToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_I410ALPHATOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I410AlphaToARGBRow = I410AlphaToARGBRow_SME;
+ }
+#endif
#if defined(HAS_I410ALPHATOARGBROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
I410AlphaToARGBRow = I410AlphaToARGBRow_Any_SSSE3;
@@ -7410,6 +8223,22 @@
ARGBAttenuateRow = ARGBAttenuateRow_RVV;
}
#endif
+#if defined(HAS_ARGBATTENUATEROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_LSX;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_LASX;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_LASX;
+ }
+ }
+#endif
#if defined(HAS_SCALEROWUP2_BILINEAR_12_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
@@ -7525,6 +8354,24 @@
dst_argb = dst_argb + (height - 1) * dst_stride_argb;
dst_stride_argb = -dst_stride_argb;
}
+#if defined(HAS_I410ALPHATOARGBROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ I410AlphaToARGBRow = I410AlphaToARGBRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ I410AlphaToARGBRow = I410AlphaToARGBRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_I410ALPHATOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I410AlphaToARGBRow = I410AlphaToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_I410ALPHATOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I410AlphaToARGBRow = I410AlphaToARGBRow_SME;
+ }
+#endif
#if defined(HAS_I410ALPHATOARGBROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
I410AlphaToARGBRow = I410AlphaToARGBRow_Any_SSSE3;
@@ -7578,6 +8425,22 @@
ARGBAttenuateRow = ARGBAttenuateRow_RVV;
}
#endif
+#if defined(HAS_ARGBATTENUATEROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_LSX;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBATTENUATEROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_Any_LASX;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBAttenuateRow = ARGBAttenuateRow_LASX;
+ }
+ }
+#endif
#if defined(HAS_SCALEROWUP2_LINEAR_12_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
@@ -7663,6 +8526,24 @@
}
}
#endif
+#if defined(HAS_P410TOARGBROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ P410ToARGBRow = P410ToARGBRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ P410ToARGBRow = P410ToARGBRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_P410TOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ P410ToARGBRow = P410ToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_P410TOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ P410ToARGBRow = P410ToARGBRow_SME;
+ }
+#endif
#ifdef HAS_SCALEUVROWUP2_BILINEAR_16_SSE41
if (TestCpuFlag(kCpuHasSSE41)) {
@@ -7756,6 +8637,24 @@
}
}
#endif
+#if defined(HAS_P410TOARGBROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ P410ToARGBRow = P410ToARGBRow_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ P410ToARGBRow = P410ToARGBRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_P410TOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ P410ToARGBRow = P410ToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_P410TOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ P410ToARGBRow = P410ToARGBRow_SME;
+ }
+#endif
#ifdef HAS_SCALEUVROWUP2_LINEAR_16_SSE41
if (TestCpuFlag(kCpuHasSSE41)) {
@@ -7835,6 +8734,24 @@
}
}
#endif
+#if defined(HAS_P410TOAR30ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ P410ToAR30Row = P410ToAR30Row_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ P410ToAR30Row = P410ToAR30Row_NEON;
+ }
+ }
+#endif
+#if defined(HAS_P410TOAR30ROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ P410ToAR30Row = P410ToAR30Row_SVE2;
+ }
+#endif
+#if defined(HAS_P410TOAR30ROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ P410ToAR30Row = P410ToAR30Row_SME;
+ }
+#endif
#ifdef HAS_SCALEUVROWUP2_BILINEAR_16_SSE41
if (TestCpuFlag(kCpuHasSSE41)) {
@@ -7928,6 +8845,24 @@
}
}
#endif
+#if defined(HAS_P410TOAR30ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ P410ToAR30Row = P410ToAR30Row_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ P410ToAR30Row = P410ToAR30Row_NEON;
+ }
+ }
+#endif
+#if defined(HAS_P410TOAR30ROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ P410ToAR30Row = P410ToAR30Row_SVE2;
+ }
+#endif
+#if defined(HAS_P410TOAR30ROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ P410ToAR30Row = P410ToAR30Row_SME;
+ }
+#endif
#ifdef HAS_SCALEUVROWUP2_LINEAR_16_SSE41
if (TestCpuFlag(kCpuHasSSE41)) {
diff --git a/source/convert_from.cc b/source/convert_from.cc
index e69da9e..f118702 100644
--- a/source/convert_from.cc
+++ b/source/convert_from.cc
@@ -30,8 +30,6 @@
}
// I420 To any I4xx YUV format with mirroring.
-// TODO(fbarchard): Consider kFilterNone for Y, or CopyPlane
-
static int I420ToI4xx(const uint8_t* src_y,
int src_stride_y,
const uint8_t* src_u,
@@ -48,21 +46,17 @@
int src_y_height,
int dst_uv_width,
int dst_uv_height) {
- const int dst_y_width = Abs(src_y_width);
- const int dst_y_height = Abs(src_y_height);
const int src_uv_width = SUBSAMPLE(src_y_width, 1, 1);
const int src_uv_height = SUBSAMPLE(src_y_height, 1, 1);
int r;
- if (src_y_width == 0 || src_y_height == 0 || dst_uv_width <= 0 ||
+ if ((!src_y && dst_y) || !src_u || !src_v || !dst_u || !dst_v ||
+ src_y_width <= 0 || src_y_height == 0 || dst_uv_width <= 0 ||
dst_uv_height <= 0) {
return -1;
}
if (dst_y) {
- r = ScalePlane(src_y, src_stride_y, src_y_width, src_y_height, dst_y,
- dst_stride_y, dst_y_width, dst_y_height, kFilterBilinear);
- if (r != 0) {
- return r;
- }
+ CopyPlane(src_y, src_stride_y, dst_y, dst_stride_y, src_y_width,
+ src_y_height);
}
r = ScalePlane(src_u, src_stride_u, src_uv_width, src_uv_height, dst_u,
dst_stride_u, dst_uv_width, dst_uv_height, kFilterBilinear);
@@ -231,25 +225,22 @@
int width,
int height) {
int r;
- if (width == 0 || height == 0) {
+ if ((!src_y && dst_y) || !src_u || !src_v || !dst_u || !dst_v || width <= 0 ||
+ height == 0) {
return -1;
}
if (dst_y) {
- r = ScalePlane_12(src_y, src_stride_y, width, height, dst_y, dst_stride_y,
- Abs(width), Abs(height), kFilterBilinear);
- if (r != 0) {
- return r;
- }
+ CopyPlane_16(src_y, src_stride_y, dst_y, dst_stride_y, width, height);
}
r = ScalePlane_12(src_u, src_stride_u, SUBSAMPLE(width, 1, 1),
- SUBSAMPLE(height, 1, 1), dst_u, dst_stride_u, Abs(width),
+ SUBSAMPLE(height, 1, 1), dst_u, dst_stride_u, width,
Abs(height), kFilterBilinear);
if (r != 0) {
return r;
}
r = ScalePlane_12(src_v, src_stride_v, SUBSAMPLE(width, 1, 1),
- SUBSAMPLE(height, 1, 1), dst_v, dst_stride_v, Abs(width),
+ SUBSAMPLE(height, 1, 1), dst_v, dst_stride_v, width,
Abs(height), kFilterBilinear);
return r;
}
@@ -271,24 +262,21 @@
int width,
int height) {
int r;
- if (width == 0 || height == 0) {
+ if ((!src_y && dst_y) || !src_u || !src_v || !dst_u || !dst_v || width <= 0 ||
+ height == 0) {
return -1;
}
if (dst_y) {
- r = ScalePlane_12(src_y, src_stride_y, width, height, dst_y, dst_stride_y,
- Abs(width), Abs(height), kFilterBilinear);
- if (r != 0) {
- return r;
- }
+ CopyPlane_16(src_y, src_stride_y, dst_y, dst_stride_y, width, height);
}
r = ScalePlane_12(src_u, src_stride_u, SUBSAMPLE(width, 1, 1), height, dst_u,
- dst_stride_u, Abs(width), Abs(height), kFilterBilinear);
+ dst_stride_u, width, Abs(height), kFilterBilinear);
if (r != 0) {
return r;
}
r = ScalePlane_12(src_v, src_stride_v, SUBSAMPLE(width, 1, 1), height, dst_v,
- dst_stride_v, Abs(width), Abs(height), kFilterBilinear);
+ dst_stride_v, width, Abs(height), kFilterBilinear);
return r;
}
@@ -310,24 +298,21 @@
int width,
int height) {
int r;
- if (width == 0 || height == 0) {
+ if ((!src_y && dst_y) || !src_u || !src_v || !dst_u || !dst_v || width <= 0 ||
+ height == 0) {
return -1;
}
if (dst_y) {
- r = ScalePlane(src_y, src_stride_y, width, height, dst_y, dst_stride_y,
- Abs(width), Abs(height), kFilterBilinear);
- if (r != 0) {
- return r;
- }
+ CopyPlane(src_y, src_stride_y, dst_y, dst_stride_y, width, height);
}
r = ScalePlane(src_u, src_stride_u, SUBSAMPLE(width, 1, 1), height, dst_u,
- dst_stride_u, Abs(width), Abs(height), kFilterBilinear);
+ dst_stride_u, width, Abs(height), kFilterBilinear);
if (r != 0) {
return r;
}
r = ScalePlane(src_v, src_stride_v, SUBSAMPLE(width, 1, 1), height, dst_v,
- dst_stride_v, Abs(width), Abs(height), kFilterBilinear);
+ dst_stride_v, width, Abs(height), kFilterBilinear);
return r;
}
@@ -342,12 +327,6 @@
if (!src_y || !dst_y || width <= 0 || height == 0) {
return -1;
}
- // Negative height means invert the image.
- if (height < 0) {
- height = -height;
- src_y = src_y + (height - 1) * src_stride_y;
- src_stride_y = -src_stride_y;
- }
CopyPlane(src_y, src_stride_y, dst_y, dst_stride_y, width, height);
return 0;
}
@@ -699,7 +678,8 @@
int height) {
int halfwidth = (width + 1) / 2;
int halfheight = (height + 1) / 2;
- if (!src_y || !src_u || !src_v || !dst_uv || width <= 0 || height == 0) {
+ if ((!src_y && dst_y) || !src_u || !src_v || !dst_uv || width <= 0 ||
+ height == 0) {
return -1;
}
// Negative height means invert the image.
diff --git a/source/convert_from_argb.cc b/source/convert_from_argb.cc
index b45de8c..8d2e8d0 100644
--- a/source/convert_from_argb.cc
+++ b/source/convert_from_argb.cc
@@ -60,6 +60,14 @@
}
}
#endif
+#if defined(HAS_ARGBTOUV444ROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ARGBToUV444Row = ARGBToUV444Row_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ ARGBToUV444Row = ARGBToUV444Row_AVX2;
+ }
+ }
+#endif
#if defined(HAS_ARGBTOUV444ROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
ARGBToUV444Row = ARGBToUV444Row_Any_NEON;
@@ -68,6 +76,14 @@
}
}
#endif
+#if defined(HAS_ARGBTOUV444ROW_NEON_I8MM)
+ if (TestCpuFlag(kCpuHasNeonI8MM)) {
+ ARGBToUV444Row = ARGBToUV444Row_Any_NEON_I8MM;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBToUV444Row = ARGBToUV444Row_NEON_I8MM;
+ }
+ }
+#endif
#if defined(HAS_ARGBTOUV444ROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
ARGBToUV444Row = ARGBToUV444Row_Any_MSA;
@@ -116,6 +132,14 @@
}
}
#endif
+#if defined(HAS_ARGBTOYROW_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd)) {
+ ARGBToYRow = ARGBToYRow_Any_NEON_DotProd;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYRow = ARGBToYRow_NEON_DotProd;
+ }
+ }
+#endif
#if defined(HAS_ARGBTOYROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
ARGBToYRow = ARGBToYRow_Any_MSA;
@@ -231,6 +255,14 @@
}
}
#endif
+#if defined(HAS_ARGBTOYROW_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd)) {
+ ARGBToYRow = ARGBToYRow_Any_NEON_DotProd;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYRow = ARGBToYRow_NEON_DotProd;
+ }
+ }
+#endif
#if defined(HAS_ARGBTOUVROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
ARGBToUVRow = ARGBToUVRow_Any_NEON;
@@ -239,6 +271,14 @@
}
}
#endif
+#if defined(HAS_ARGBTOUVROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ ARGBToUVRow = ARGBToUVRow_Any_SVE2;
+ if (IS_ALIGNED(width, 2)) {
+ ARGBToUVRow = ARGBToUVRow_SVE2;
+ }
+ }
+#endif
#if defined(HAS_ARGBTOYROW_MSA) && defined(HAS_ARGBTOUVROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
ARGBToYRow = ARGBToYRow_Any_MSA;
@@ -331,6 +371,14 @@
}
}
#endif
+#if defined(HAS_ARGBTOYROW_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd)) {
+ ARGBToYRow = ARGBToYRow_Any_NEON_DotProd;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYRow = ARGBToYRow_NEON_DotProd;
+ }
+ }
+#endif
#if defined(HAS_ARGBTOUVROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
ARGBToUVRow = ARGBToUVRow_Any_NEON;
@@ -339,6 +387,14 @@
}
}
#endif
+#if defined(HAS_ARGBTOUVROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ ARGBToUVRow = ARGBToUVRow_Any_SVE2;
+ if (IS_ALIGNED(width, 2)) {
+ ARGBToUVRow = ARGBToUVRow_SVE2;
+ }
+ }
+#endif
#if defined(HAS_ARGBTOYROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
ARGBToYRow = ARGBToYRow_Any_SSSE3;
@@ -438,6 +494,11 @@
}
}
#endif
+#if defined(HAS_MERGEUVROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ MergeUVRow_ = MergeUVRow_SME;
+ }
+#endif
#if defined(HAS_MERGEUVROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
MergeUVRow_ = MergeUVRow_Any_MSA;
@@ -553,6 +614,14 @@
}
}
#endif
+#if defined(HAS_ARGBTOYROW_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd)) {
+ ARGBToYRow = ARGBToYRow_Any_NEON_DotProd;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYRow = ARGBToYRow_NEON_DotProd;
+ }
+ }
+#endif
#if defined(HAS_ARGBTOUVROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
ARGBToUVRow = ARGBToUVRow_Any_NEON;
@@ -561,6 +630,14 @@
}
}
#endif
+#if defined(HAS_ARGBTOUVROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ ARGBToUVRow = ARGBToUVRow_Any_SVE2;
+ if (IS_ALIGNED(width, 2)) {
+ ARGBToUVRow = ARGBToUVRow_SVE2;
+ }
+ }
+#endif
#if defined(HAS_ARGBTOYROW_MSA) && defined(HAS_ARGBTOUVROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
ARGBToYRow = ARGBToYRow_Any_MSA;
@@ -638,6 +715,11 @@
}
}
#endif
+#if defined(HAS_MERGEUVROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ MergeUVRow_ = MergeUVRow_SME;
+ }
+#endif
#if defined(HAS_MERGEUVROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
MergeUVRow_ = MergeUVRow_Any_MSA;
@@ -752,6 +834,14 @@
}
}
#endif
+#if defined(HAS_ABGRTOYROW_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd)) {
+ ABGRToYRow = ABGRToYRow_Any_NEON_DotProd;
+ if (IS_ALIGNED(width, 16)) {
+ ABGRToYRow = ABGRToYRow_NEON_DotProd;
+ }
+ }
+#endif
#if defined(HAS_ABGRTOUVROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
ABGRToUVRow = ABGRToUVRow_Any_NEON;
@@ -760,6 +850,14 @@
}
}
#endif
+#if defined(HAS_ABGRTOUVROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ ABGRToUVRow = ABGRToUVRow_Any_SVE2;
+ if (IS_ALIGNED(width, 2)) {
+ ABGRToUVRow = ABGRToUVRow_SVE2;
+ }
+ }
+#endif
#if defined(HAS_ABGRTOYROW_MSA) && defined(HAS_ABGRTOUVROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
ABGRToYRow = ABGRToYRow_Any_MSA;
@@ -825,6 +923,11 @@
}
}
#endif
+#if defined(HAS_MERGEUVROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ MergeUVRow_ = MergeUVRow_SME;
+ }
+#endif
#if defined(HAS_MERGEUVROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
MergeUVRow_ = MergeUVRow_Any_MSA;
@@ -940,6 +1043,14 @@
}
}
#endif
+#if defined(HAS_ABGRTOYROW_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd)) {
+ ABGRToYRow = ABGRToYRow_Any_NEON_DotProd;
+ if (IS_ALIGNED(width, 16)) {
+ ABGRToYRow = ABGRToYRow_NEON_DotProd;
+ }
+ }
+#endif
#if defined(HAS_ABGRTOUVROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
ABGRToUVRow = ABGRToUVRow_Any_NEON;
@@ -948,6 +1059,14 @@
}
}
#endif
+#if defined(HAS_ABGRTOUVROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ ABGRToUVRow = ABGRToUVRow_Any_SVE2;
+ if (IS_ALIGNED(width, 2)) {
+ ABGRToUVRow = ABGRToUVRow_SVE2;
+ }
+ }
+#endif
#if defined(HAS_ABGRTOYROW_MSA) && defined(HAS_ABGRTOUVROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
ABGRToYRow = ABGRToYRow_Any_MSA;
@@ -1013,6 +1132,11 @@
}
}
#endif
+#if defined(HAS_MERGEUVROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ MergeUVRow_ = MergeUVRow_SME;
+ }
+#endif
#if defined(HAS_MERGEUVROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
MergeUVRow_ = MergeUVRow_Any_MSA;
@@ -1133,6 +1257,14 @@
}
}
#endif
+#if defined(HAS_ARGBTOYROW_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd)) {
+ ARGBToYRow = ARGBToYRow_Any_NEON_DotProd;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYRow = ARGBToYRow_NEON_DotProd;
+ }
+ }
+#endif
#if defined(HAS_ARGBTOUVROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
ARGBToUVRow = ARGBToUVRow_Any_NEON;
@@ -1141,6 +1273,14 @@
}
}
#endif
+#if defined(HAS_ARGBTOUVROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ ARGBToUVRow = ARGBToUVRow_Any_SVE2;
+ if (IS_ALIGNED(width, 2)) {
+ ARGBToUVRow = ARGBToUVRow_SVE2;
+ }
+ }
+#endif
#if defined(HAS_ARGBTOYROW_MSA) && defined(HAS_ARGBTOUVROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
ARGBToYRow = ARGBToYRow_Any_MSA;
@@ -1329,6 +1469,14 @@
}
}
#endif
+#if defined(HAS_ARGBTOYROW_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd)) {
+ ARGBToYRow = ARGBToYRow_Any_NEON_DotProd;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYRow = ARGBToYRow_NEON_DotProd;
+ }
+ }
+#endif
#if defined(HAS_ARGBTOUVROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
ARGBToUVRow = ARGBToUVRow_Any_NEON;
@@ -1337,6 +1485,14 @@
}
}
#endif
+#if defined(HAS_ARGBTOUVROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ ARGBToUVRow = ARGBToUVRow_Any_SVE2;
+ if (IS_ALIGNED(width, 2)) {
+ ARGBToUVRow = ARGBToUVRow_SVE2;
+ }
+ }
+#endif
#if defined(HAS_ARGBTOYROW_MSA) && defined(HAS_ARGBTOUVROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
ARGBToYRow = ARGBToYRow_Any_MSA;
@@ -1501,6 +1657,14 @@
}
}
#endif
+#if defined(HAS_ARGBTOYROW_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd)) {
+ ARGBToYRow = ARGBToYRow_Any_NEON_DotProd;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYRow = ARGBToYRow_NEON_DotProd;
+ }
+ }
+#endif
#if defined(HAS_ARGBTOYROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
ARGBToYRow = ARGBToYRow_Any_MSA;
@@ -1654,6 +1818,11 @@
}
}
#endif
+#if defined(HAS_ARGBTORGB24ROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ ARGBToRGB24Row = ARGBToRGB24Row_SVE2;
+ }
+#endif
#if defined(HAS_ARGBTORGB24ROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
ARGBToRGB24Row = ARGBToRGB24Row_Any_MSA;
@@ -1741,6 +1910,11 @@
}
}
#endif
+#if defined(HAS_ARGBTORAWROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ ARGBToRAWRow = ARGBToRAWRow_SVE2;
+ }
+#endif
#if defined(HAS_ARGBTORAWROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
ARGBToRAWRow = ARGBToRAWRow_Any_MSA;
@@ -1832,6 +2006,11 @@
}
}
#endif
+#if defined(HAS_ARGBTORGB565DITHERROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_SVE2;
+ }
+#endif
#if defined(HAS_ARGBTORGB565DITHERROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_Any_MSA;
@@ -1917,6 +2096,11 @@
}
}
#endif
+#if defined(HAS_ARGBTORGB565ROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ ARGBToRGB565Row = ARGBToRGB565Row_SVE2;
+ }
+#endif
#if defined(HAS_ARGBTORGB565ROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
ARGBToRGB565Row = ARGBToRGB565Row_Any_MSA;
@@ -2140,6 +2324,14 @@
height = 1;
src_stride_abgr = dst_stride_ar30 = 0;
}
+#if defined(HAS_ABGRTOAR30ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ABGRToAR30Row = ABGRToAR30Row_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ ABGRToAR30Row = ABGRToAR30Row_NEON;
+ }
+ }
+#endif
#if defined(HAS_ABGRTOAR30ROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
ABGRToAR30Row = ABGRToAR30Row_Any_SSSE3;
@@ -2189,6 +2381,14 @@
height = 1;
src_stride_argb = dst_stride_ar30 = 0;
}
+#if defined(HAS_ARGBTOAR30ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ARGBToAR30Row = ARGBToAR30Row_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBToAR30Row = ARGBToAR30Row_NEON;
+ }
+ }
+#endif
#if defined(HAS_ARGBTOAR30ROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
ARGBToAR30Row = ARGBToAR30Row_Any_SSSE3;
@@ -2213,6 +2413,167 @@
return 0;
}
+// ARGB little endian (bgra in memory) to J444
+LIBYUV_API
+int ARGBToJ444(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_u,
+ int dst_stride_u,
+ uint8_t* dst_v,
+ int dst_stride_v,
+ int width,
+ int height) {
+ int y;
+ void (*ARGBToYJRow)(const uint8_t* src_argb, uint8_t* dst_y, int width) =
+ ARGBToYJRow_C;
+ void (*ARGBToUVJ444Row)(const uint8_t* src_argb, uint8_t* dst_u,
+ uint8_t* dst_v, int width) = ARGBToUVJ444Row_C;
+ if (!src_argb || !dst_y || !dst_u || !dst_v || width <= 0 || height == 0) {
+ return -1;
+ }
+ if (height < 0) {
+ height = -height;
+ src_argb = src_argb + (height - 1) * src_stride_argb;
+ src_stride_argb = -src_stride_argb;
+ }
+ // Coalesce rows.
+ if (src_stride_argb == width * 4 && dst_stride_y == width &&
+ dst_stride_u == width && dst_stride_v == width) {
+ width *= height;
+ height = 1;
+ src_stride_argb = dst_stride_y = dst_stride_u = dst_stride_v = 0;
+ }
+#if defined(HAS_ARGBTOUVJ444ROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ ARGBToUVJ444Row = ARGBToUVJ444Row_Any_SSSE3;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToUVJ444Row = ARGBToUVJ444Row_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVJ444ROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ARGBToUVJ444Row = ARGBToUVJ444Row_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ ARGBToUVJ444Row = ARGBToUVJ444Row_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVJ444ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ARGBToUVJ444Row = ARGBToUVJ444Row_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBToUVJ444Row = ARGBToUVJ444Row_NEON;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVJ444ROW_NEON_I8MM)
+ if (TestCpuFlag(kCpuHasNeonI8MM)) {
+ ARGBToUVJ444Row = ARGBToUVJ444Row_Any_NEON_I8MM;
+ if (IS_ALIGNED(width, 8)) {
+ ARGBToUVJ444Row = ARGBToUVJ444Row_NEON_I8MM;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVJ444ROW_MSA)
+ if (TestCpuFlag(kCpuHasMSA)) {
+ ARGBToUVJ444Row = ARGBToUVJ444Row_Any_MSA;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToUVJ444Row = ARGBToUVJ444Row_MSA;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVJ444ROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGBToUVJ444Row = ARGBToUVJ444Row_Any_LSX;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToUVJ444Row = ARGBToUVJ444Row_LSX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOUVJ444ROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGBToUVJ444Row = ARGBToUVJ444Row_Any_LASX;
+ if (IS_ALIGNED(width, 32)) {
+ ARGBToUVJ444Row = ARGBToUVJ444Row_LASX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYJROW_SSSE3)
+ if (TestCpuFlag(kCpuHasSSSE3)) {
+ ARGBToYJRow = ARGBToYJRow_Any_SSSE3;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYJRow = ARGBToYJRow_SSSE3;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYJROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ ARGBToYJRow = ARGBToYJRow_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ ARGBToYJRow = ARGBToYJRow_AVX2;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYJROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ ARGBToYJRow = ARGBToYJRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYJRow = ARGBToYJRow_NEON;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYJROW_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd)) {
+ ARGBToYJRow = ARGBToYJRow_Any_NEON_DotProd;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYJRow = ARGBToYJRow_NEON_DotProd;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYJROW_MSA)
+ if (TestCpuFlag(kCpuHasMSA)) {
+ ARGBToYJRow = ARGBToYJRow_Any_MSA;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYJRow = ARGBToYJRow_MSA;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYJROW_LSX)
+ if (TestCpuFlag(kCpuHasLSX)) {
+ ARGBToYJRow = ARGBToYJRow_Any_LSX;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYJRow = ARGBToYJRow_LSX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYJROW_LASX)
+ if (TestCpuFlag(kCpuHasLASX)) {
+ ARGBToYJRow = ARGBToYJRow_Any_LASX;
+ if (IS_ALIGNED(width, 32)) {
+ ARGBToYJRow = ARGBToYJRow_LASX;
+ }
+ }
+#endif
+#if defined(HAS_ARGBTOYJROW_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ARGBToYJRow = ARGBToYJRow_RVV;
+ }
+#endif
+
+ for (y = 0; y < height; ++y) {
+ ARGBToUVJ444Row(src_argb, dst_u, dst_v, width);
+ ARGBToYJRow(src_argb, dst_y, width);
+ src_argb += src_stride_argb;
+ dst_y += dst_stride_y;
+ dst_u += dst_stride_u;
+ dst_v += dst_stride_v;
+ }
+ return 0;
+}
+
// Convert ARGB to J420. (JPeg full range I420).
LIBYUV_API
int ARGBToJ420(const uint8_t* src_argb,
@@ -2248,6 +2609,14 @@
}
}
#endif
+#if defined(HAS_ARGBTOYJROW_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd)) {
+ ARGBToYJRow = ARGBToYJRow_Any_NEON_DotProd;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYJRow = ARGBToYJRow_NEON_DotProd;
+ }
+ }
+#endif
#if defined(HAS_ARGBTOUVJROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
ARGBToUVJRow = ARGBToUVJRow_Any_NEON;
@@ -2256,6 +2625,14 @@
}
}
#endif
+#if defined(HAS_ARGBTOUVJROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ ARGBToUVJRow = ARGBToUVJRow_Any_SVE2;
+ if (IS_ALIGNED(width, 2)) {
+ ARGBToUVJRow = ARGBToUVJRow_SVE2;
+ }
+ }
+#endif
#if defined(HAS_ARGBTOYJROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
ARGBToYJRow = ARGBToYJRow_Any_SSSE3;
@@ -2416,6 +2793,14 @@
}
}
#endif
+#if defined(HAS_ARGBTOYJROW_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd)) {
+ ARGBToYJRow = ARGBToYJRow_Any_NEON_DotProd;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYJRow = ARGBToYJRow_NEON_DotProd;
+ }
+ }
+#endif
#if defined(HAS_ARGBTOUVJROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
ARGBToUVJRow = ARGBToUVJRow_Any_NEON;
@@ -2424,6 +2809,14 @@
}
}
#endif
+#if defined(HAS_ARGBTOUVJROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ ARGBToUVJRow = ARGBToUVJRow_Any_SVE2;
+ if (IS_ALIGNED(width, 2)) {
+ ARGBToUVJRow = ARGBToUVJRow_SVE2;
+ }
+ }
+#endif
#if defined(HAS_ARGBTOYJROW_MSA) && defined(HAS_ARGBTOUVJROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
ARGBToYJRow = ARGBToYJRow_Any_MSA;
@@ -2522,6 +2915,14 @@
}
}
#endif
+#if defined(HAS_ARGBTOYJROW_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd)) {
+ ARGBToYJRow = ARGBToYJRow_Any_NEON_DotProd;
+ if (IS_ALIGNED(width, 16)) {
+ ARGBToYJRow = ARGBToYJRow_NEON_DotProd;
+ }
+ }
+#endif
#if defined(HAS_ARGBTOYJROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
ARGBToYJRow = ARGBToYJRow_Any_MSA;
@@ -2593,6 +2994,14 @@
}
}
#endif
+#if defined(HAS_RGBATOYJROW_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd)) {
+ RGBAToYJRow = RGBAToYJRow_Any_NEON_DotProd;
+ if (IS_ALIGNED(width, 16)) {
+ RGBAToYJRow = RGBAToYJRow_NEON_DotProd;
+ }
+ }
+#endif
#if defined(HAS_RGBATOYJROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
RGBAToYJRow = RGBAToYJRow_Any_MSA;
@@ -2698,6 +3107,14 @@
}
}
#endif
+#if defined(HAS_ABGRTOYJROW_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd)) {
+ ABGRToYJRow = ABGRToYJRow_Any_NEON_DotProd;
+ if (IS_ALIGNED(width, 16)) {
+ ABGRToYJRow = ABGRToYJRow_NEON_DotProd;
+ }
+ }
+#endif
#if defined(HAS_ABGRTOUVJROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
ABGRToUVJRow = ABGRToUVJRow_Any_NEON;
@@ -2706,6 +3123,14 @@
}
}
#endif
+#if defined(HAS_ABGRTOUVJROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ ABGRToUVJRow = ABGRToUVJRow_Any_SVE2;
+ if (IS_ALIGNED(width, 2)) {
+ ABGRToUVJRow = ABGRToUVJRow_SVE2;
+ }
+ }
+#endif
#if defined(HAS_ABGRTOYJROW_MSA) && defined(HAS_ABGRTOUVJROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
ABGRToYJRow = ABGRToYJRow_Any_MSA;
@@ -2828,6 +3253,14 @@
}
}
#endif
+#if defined(HAS_ABGRTOYJROW_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd)) {
+ ABGRToYJRow = ABGRToYJRow_Any_NEON_DotProd;
+ if (IS_ALIGNED(width, 16)) {
+ ABGRToYJRow = ABGRToYJRow_NEON_DotProd;
+ }
+ }
+#endif
#if defined(HAS_ABGRTOUVJROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
ABGRToUVJRow = ABGRToUVJRow_Any_NEON;
@@ -2836,6 +3269,14 @@
}
}
#endif
+#if defined(HAS_ABGRTOUVJROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ ABGRToUVJRow = ABGRToUVJRow_Any_SVE2;
+ if (IS_ALIGNED(width, 2)) {
+ ABGRToUVJRow = ABGRToUVJRow_SVE2;
+ }
+ }
+#endif
#if defined(HAS_ABGRTOYJROW_MSA) && defined(HAS_ABGRTOUVJROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
ABGRToYJRow = ABGRToYJRow_Any_MSA;
@@ -2930,6 +3371,14 @@
}
}
#endif
+#if defined(HAS_ABGRTOYJROW_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd)) {
+ ABGRToYJRow = ABGRToYJRow_Any_NEON_DotProd;
+ if (IS_ALIGNED(width, 16)) {
+ ABGRToYJRow = ABGRToYJRow_NEON_DotProd;
+ }
+ }
+#endif
#if defined(HAS_ABGRTOYJROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
ABGRToYJRow = ABGRToYJRow_Any_MSA;
@@ -3197,6 +3646,14 @@
}
}
#endif
+#if defined(HAS_RAWTOARGBROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ RAWToARGBRow = RAWToARGBRow_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ RAWToARGBRow = RAWToARGBRow_AVX2;
+ }
+ }
+#endif
#if defined(HAS_ARGBTOYJROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
ARGBToYJRow = ARGBToYJRow_Any_SSSE3;
@@ -3262,6 +3719,11 @@
}
}
#endif
+#if defined(HAS_MERGEUVROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ MergeUVRow_ = MergeUVRow_SME;
+ }
+#endif
#if defined(HAS_MERGEUVROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
MergeUVRow_ = MergeUVRow_Any_MSA;
diff --git a/source/convert_to_argb.cc b/source/convert_to_argb.cc
index 84df16c..72d21b0 100644
--- a/source/convert_to_argb.cc
+++ b/source/convert_to_argb.cc
@@ -10,6 +10,10 @@
#include "libyuv/convert_argb.h"
+#include <limits.h>
+#include <stdint.h>
+#include <stdlib.h>
+
#include "libyuv/cpu_id.h"
#ifdef HAVE_JPEG
#include "libyuv/mjpeg_decoder.h"
@@ -66,7 +70,8 @@
uint8_t* rotate_buffer = NULL;
int abs_crop_height = (crop_height < 0) ? -crop_height : crop_height;
- if (dst_argb == NULL || sample == NULL || src_width <= 0 || crop_width <= 0 ||
+ if (dst_argb == NULL || sample == NULL || src_width <= 0 ||
+ src_width > INT_MAX / 4 || crop_width <= 0 || crop_width > INT_MAX / 4 ||
src_height == 0 || crop_height == 0) {
return -1;
}
@@ -75,8 +80,12 @@
}
if (need_buf) {
- int argb_size = crop_width * 4 * abs_crop_height;
- rotate_buffer = (uint8_t*)malloc(argb_size); /* NOLINT */
+ const uint64_t rotate_buffer_size =
+ (uint64_t)crop_width * 4 * abs_crop_height;
+ if (rotate_buffer_size > SIZE_MAX) {
+ return -1; // Invalid size.
+ }
+ rotate_buffer = (uint8_t*)malloc((size_t)rotate_buffer_size);
if (!rotate_buffer) {
return 1; // Out of memory runtime error.
}
diff --git a/source/convert_to_i420.cc b/source/convert_to_i420.cc
index 5869ecd..aab071e 100644
--- a/source/convert_to_i420.cc
+++ b/source/convert_to_i420.cc
@@ -8,10 +8,13 @@
* be found in the AUTHORS file in the root of the source tree.
*/
-#include <stdlib.h>
-
#include "libyuv/convert.h"
+#include <limits.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+
#include "libyuv/video_common.h"
#ifdef __cplusplus
@@ -46,7 +49,6 @@
const uint8_t* src;
const uint8_t* src_uv;
const int abs_src_height = (src_height < 0) ? -src_height : src_height;
- // TODO(nisse): Why allow crop_height < 0?
const int abs_crop_height = (crop_height < 0) ? -crop_height : crop_height;
int r = 0;
LIBYUV_BOOL need_buf =
@@ -64,7 +66,8 @@
(src_height < 0) ? -abs_crop_height : abs_crop_height;
if (!dst_y || !dst_u || !dst_v || !sample || src_width <= 0 ||
- crop_width <= 0 || src_height == 0 || crop_height == 0) {
+ src_width > INT_MAX / 4 || crop_width <= 0 || src_height == 0 ||
+ crop_height == 0) {
return -1;
}
@@ -76,7 +79,12 @@
if (need_buf) {
int y_size = crop_width * abs_crop_height;
int uv_size = ((crop_width + 1) / 2) * ((abs_crop_height + 1) / 2);
- rotate_buffer = (uint8_t*)malloc(y_size + uv_size * 2); /* NOLINT */
+ const uint64_t rotate_buffer_size =
+ (uint64_t)y_size + (uint64_t)uv_size * 2;
+ if (rotate_buffer_size > SIZE_MAX) {
+ return -1; // Invalid size.
+ }
+ rotate_buffer = (uint8_t*)malloc((size_t)rotate_buffer_size);
if (!rotate_buffer) {
return 1; // Out of memory runtime error.
}
diff --git a/source/cpu_id.cc b/source/cpu_id.cc
index eedce16..e4acbec 100644
--- a/source/cpu_id.cc
+++ b/source/cpu_id.cc
@@ -23,6 +23,22 @@
#include <stdio.h> // For fopen()
#include <string.h>
+#if defined(__linux__) && defined(__aarch64__)
+#include <sys/auxv.h> // For getauxval()
+#endif
+
+#if defined(_WIN32) && defined(__aarch64__)
+#undef WIN32_LEAN_AND_MEAN
+#define WIN32_LEAN_AND_MEAN
+#undef WIN32_EXTRA_LEAN
+#define WIN32_EXTRA_LEAN
+#include <windows.h> // For IsProcessorFeaturePresent()
+#endif
+
+#if defined(__APPLE__) && defined(__aarch64__)
+#include <sys/sysctl.h> // For sysctlbyname()
+#endif
+
#ifdef __cplusplus
namespace libyuv {
extern "C" {
@@ -132,6 +148,13 @@
#pragma optimize("g", on)
#endif
+static int cpuinfo_search(const char* cpuinfo_line,
+ const char* needle,
+ int needle_len) {
+ const char* p = strstr(cpuinfo_line, needle);
+ return p && (p[needle_len] == ' ' || p[needle_len] == '\n');
+}
+
// Based on libvpx arm_cpudetect.c
// For Arm, but public to allow testing on any CPU
LIBYUV_API SAFEBUFFERS int ArmCpuCaps(const char* cpuinfo_name) {
@@ -143,25 +166,118 @@
return kCpuHasNEON;
}
memset(cpuinfo_line, 0, sizeof(cpuinfo_line));
+ int features = 0;
while (fgets(cpuinfo_line, sizeof(cpuinfo_line), f)) {
if (memcmp(cpuinfo_line, "Features", 8) == 0) {
- char* p = strstr(cpuinfo_line, " neon");
- if (p && (p[5] == ' ' || p[5] == '\n')) {
- fclose(f);
- return kCpuHasNEON;
- }
- // aarch64 uses asimd for Neon.
- p = strstr(cpuinfo_line, " asimd");
- if (p) {
- fclose(f);
- return kCpuHasNEON;
+ if (cpuinfo_search(cpuinfo_line, " neon", 5)) {
+ features |= kCpuHasNEON;
}
}
}
fclose(f);
- return 0;
+ return features;
}
+#ifdef __aarch64__
+#ifdef __linux__
+// Define hwcap values ourselves: building with an old auxv header where these
+// hwcap values are not defined should not prevent features from being enabled.
+#define YUV_AARCH64_HWCAP_ASIMDDP (1 << 20)
+#define YUV_AARCH64_HWCAP_SVE (1 << 22)
+#define YUV_AARCH64_HWCAP2_SVE2 (1 << 1)
+#define YUV_AARCH64_HWCAP2_I8MM (1 << 13)
+#define YUV_AARCH64_HWCAP2_SME (1 << 23)
+
+// For AArch64, but public to allow testing on any CPU.
+LIBYUV_API SAFEBUFFERS int AArch64CpuCaps(unsigned long hwcap,
+ unsigned long hwcap2) {
+ // Neon is mandatory on AArch64, so enable regardless of hwcaps.
+ int features = kCpuHasNEON;
+
+ // Don't try to enable later extensions unless earlier extensions are also
+ // reported available. Some of these constraints aren't strictly required by
+ // the architecture, but are satisfied by all micro-architectures of
+ // interest. This also avoids an issue on some emulators where true
+ // architectural constraints are not satisfied, e.g. SVE2 may be reported as
+ // available while SVE is not.
+ if (hwcap & YUV_AARCH64_HWCAP_ASIMDDP) {
+ features |= kCpuHasNeonDotProd;
+ if (hwcap2 & YUV_AARCH64_HWCAP2_I8MM) {
+ features |= kCpuHasNeonI8MM;
+ if (hwcap & YUV_AARCH64_HWCAP_SVE) {
+ features |= kCpuHasSVE;
+ if (hwcap2 & YUV_AARCH64_HWCAP2_SVE2) {
+ features |= kCpuHasSVE2;
+ if (hwcap2 & YUV_AARCH64_HWCAP2_SME) {
+ features |= kCpuHasSME;
+ }
+ }
+ }
+ }
+ }
+ return features;
+}
+
+#elif defined(_WIN32)
+// For AArch64, but public to allow testing on any CPU.
+LIBYUV_API SAFEBUFFERS int AArch64CpuCaps() {
+ // Neon is mandatory on AArch64, so enable unconditionally.
+ int features = kCpuHasNEON;
+
+ // For more information on IsProcessorFeaturePresent(), see:
+ // https://learn.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-isprocessorfeaturepresent#parameters
+#ifdef PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE
+ if (IsProcessorFeaturePresent(PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE)) {
+ features |= kCpuHasNeonDotProd;
+ }
+#endif
+ // No Neon I8MM or SVE feature detection available here at time of writing.
+ return features;
+}
+
+#elif defined(__APPLE__)
+static bool have_feature(const char* feature) {
+ // For more information on sysctlbyname(), see:
+ // https://developer.apple.com/documentation/kernel/1387446-sysctlbyname/determining_instruction_set_characteristics
+ int64_t feature_present = 0;
+ size_t size = sizeof(feature_present);
+ if (sysctlbyname(feature, &feature_present, &size, NULL, 0) != 0) {
+ return false;
+ }
+ return feature_present;
+}
+
+// For AArch64, but public to allow testing on any CPU.
+LIBYUV_API SAFEBUFFERS int AArch64CpuCaps() {
+ // Neon is mandatory on AArch64, so enable unconditionally.
+ int features = kCpuHasNEON;
+
+ if (have_feature("hw.optional.arm.FEAT_DotProd")) {
+ features |= kCpuHasNeonDotProd;
+ if (have_feature("hw.optional.arm.FEAT_I8MM")) {
+ features |= kCpuHasNeonI8MM;
+ if (have_feature("hw.optional.arm.FEAT_SME2")) {
+ features |= kCpuHasSME;
+ }
+ }
+ }
+ // No SVE feature detection available here at time of writing.
+ return features;
+}
+
+#else // !defined(__linux__) && !defined(_WIN32) && !defined(__APPLE__)
+// For AArch64, but public to allow testing on any CPU.
+LIBYUV_API SAFEBUFFERS int AArch64CpuCaps() {
+ // Neon is mandatory on AArch64, so enable unconditionally.
+ int features = kCpuHasNEON;
+
+ // TODO(libyuv:980) support feature detection on other platforms.
+
+ return features;
+}
+#endif
+#endif // defined(__aarch64__)
+
LIBYUV_API SAFEBUFFERS int RiscvCpuCaps(const char* cpuinfo_name) {
char cpuinfo_line[512];
int flag = 0;
@@ -293,17 +409,24 @@
int cpu_info1[4] = {0, 0, 0, 0};
int cpu_info7[4] = {0, 0, 0, 0};
int cpu_einfo7[4] = {0, 0, 0, 0};
+ int cpu_info24[4] = {0, 0, 0, 0};
+ int cpu_amdinfo21[4] = {0, 0, 0, 0};
CpuId(0, 0, cpu_info0);
CpuId(1, 0, cpu_info1);
if (cpu_info0[0] >= 7) {
CpuId(7, 0, cpu_info7);
CpuId(7, 1, cpu_einfo7);
+ CpuId(0x80000021, 0, cpu_amdinfo21);
+ }
+ if (cpu_info0[0] >= 0x24) {
+ CpuId(0x24, 0, cpu_info24);
}
cpu_info = kCpuHasX86 | ((cpu_info1[3] & 0x04000000) ? kCpuHasSSE2 : 0) |
((cpu_info1[2] & 0x00000200) ? kCpuHasSSSE3 : 0) |
((cpu_info1[2] & 0x00080000) ? kCpuHasSSE41 : 0) |
((cpu_info1[2] & 0x00100000) ? kCpuHasSSE42 : 0) |
- ((cpu_info7[1] & 0x00000200) ? kCpuHasERMS : 0);
+ ((cpu_info7[1] & 0x00000200) ? kCpuHasERMS : 0) |
+ ((cpu_info7[3] & 0x00000010) ? kCpuHasFSMR : 0);
// AVX requires OS saves YMM registers.
if (((cpu_info1[2] & 0x1c000000) == 0x1c000000) && // AVX and OSXSave
@@ -314,15 +437,21 @@
((cpu_einfo7[0] & 0x00000010) ? kCpuHasAVXVNNI : 0) |
((cpu_einfo7[3] & 0x00000010) ? kCpuHasAVXVNNIINT8 : 0);
+ cpu_info |= ((cpu_amdinfo21[0] & 0x00008000) ? kCpuHasERMS : 0);
+
// Detect AVX512bw
if ((GetXCR0() & 0xe0) == 0xe0) {
- cpu_info |= (cpu_info7[1] & 0x40000000) ? kCpuHasAVX512BW : 0;
- cpu_info |= (cpu_info7[1] & 0x80000000) ? kCpuHasAVX512VL : 0;
- cpu_info |= (cpu_info7[2] & 0x00000002) ? kCpuHasAVX512VBMI : 0;
- cpu_info |= (cpu_info7[2] & 0x00000040) ? kCpuHasAVX512VBMI2 : 0;
- cpu_info |= (cpu_info7[2] & 0x00000800) ? kCpuHasAVX512VNNI : 0;
- cpu_info |= (cpu_info7[2] & 0x00001000) ? kCpuHasAVX512VBITALG : 0;
- cpu_info |= (cpu_einfo7[3] & 0x00080000) ? kCpuHasAVX10 : 0;
+ cpu_info |= ((cpu_info7[1] & 0x40000000) ? kCpuHasAVX512BW : 0) |
+ ((cpu_info7[1] & 0x80000000) ? kCpuHasAVX512VL : 0) |
+ ((cpu_info7[2] & 0x00000002) ? kCpuHasAVX512VBMI : 0) |
+ ((cpu_info7[2] & 0x00000040) ? kCpuHasAVX512VBMI2 : 0) |
+ ((cpu_info7[2] & 0x00000800) ? kCpuHasAVX512VNNI : 0) |
+ ((cpu_info7[2] & 0x00001000) ? kCpuHasAVX512VBITALG : 0) |
+ ((cpu_einfo7[3] & 0x00080000) ? kCpuHasAVX10 : 0) |
+ ((cpu_info7[3] & 0x02000000) ? kCpuHasAMXINT8 : 0);
+ if (cpu_info0[0] >= 0x24 && (cpu_einfo7[3] & 0x00080000)) {
+ cpu_info |= ((cpu_info24[1] & 0xFF) >= 2) ? kCpuHasAVX10_2 : 0;
+ }
}
}
#endif
@@ -334,21 +463,31 @@
cpu_info = LoongarchCpuCaps();
cpu_info |= kCpuHasLOONGARCH;
#endif
-#if defined(__arm__) || defined(__aarch64__)
-// gcc -mfpu=neon defines __ARM_NEON__
-// __ARM_NEON__ generates code that requires Neon. NaCL also requires Neon.
-// For Linux, /proc/cpuinfo can be tested but without that assume Neon.
-#if defined(__ARM_NEON__) || defined(__native_client__) || !defined(__linux__)
- cpu_info = kCpuHasNEON;
-// For aarch64(arm64), /proc/cpuinfo's feature is not complete, e.g. no neon
-// flag in it.
-// So for aarch64, neon enabling is hard coded here.
-#endif
#if defined(__aarch64__)
+#if defined(__linux__)
+ // getauxval is supported since Android SDK version 18, minimum at time of
+ // writing is 21, so should be safe to always use this. If getauxval is
+ // somehow disabled then getauxval returns 0, which will leave Neon enabled
+ // since Neon is mandatory on AArch64.
+ unsigned long hwcap = getauxval(AT_HWCAP);
+ unsigned long hwcap2 = getauxval(AT_HWCAP2);
+ cpu_info = AArch64CpuCaps(hwcap, hwcap2);
+#else
+ cpu_info = AArch64CpuCaps();
+#endif
+ cpu_info |= kCpuHasARM;
+#endif // __aarch64__
+#if defined(__arm__)
+ // gcc -mfpu=neon defines __ARM_NEON__
+ // __ARM_NEON__ generates code that requires Neon. NaCL also requires Neon.
+ // For Linux, /proc/cpuinfo can be tested but without that assume Neon.
+ // Linux arm parse text file for neon detect.
+#if defined(__linux__)
+ cpu_info = ArmCpuCaps("/proc/cpuinfo");
+#elif defined(__ARM_NEON__)
cpu_info = kCpuHasNEON;
#else
- // Linux arm parse text file for neon detect.
- cpu_info = ArmCpuCaps("/proc/cpuinfo");
+ cpu_info = 0;
#endif
cpu_info |= kCpuHasARM;
#endif // __arm__
diff --git a/source/mjpeg_decoder.cc b/source/mjpeg_decoder.cc
index 0141da8..b93a849 100644
--- a/source/mjpeg_decoder.cc
+++ b/source/mjpeg_decoder.cc
@@ -13,8 +13,7 @@
#ifdef HAVE_JPEG
#include <assert.h>
-#if !defined(__pnacl__) && !defined(__CLR_VER) && \
- !defined(COVERAGE_ENABLED) && !defined(TARGET_IPHONE_SIMULATOR)
+#if !defined(__pnacl__) && !defined(__CLR_VER) && !defined(COVERAGE_ENABLED)
// Must be included before jpeglib.
#include <setjmp.h>
#define HAVE_SETJMP
diff --git a/source/planar_functions.cc b/source/planar_functions.cc
index 1c94e26..c2d4b67 100644
--- a/source/planar_functions.cc
+++ b/source/planar_functions.cc
@@ -14,9 +14,6 @@
#include <string.h> // for memset()
#include "libyuv/cpu_id.h"
-#ifdef HAVE_JPEG
-#include "libyuv/mjpeg_decoder.h"
-#endif
#include "libyuv/row.h"
#include "libyuv/scale_row.h" // for ScaleRowDown2
@@ -65,6 +62,11 @@
CopyRow = IS_ALIGNED(width, 64) ? CopyRow_AVX : CopyRow_Any_AVX;
}
#endif
+#if defined(HAS_COPYROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ CopyRow = IS_ALIGNED(width, 128) ? CopyRow_AVX512BW : CopyRow_Any_AVX512BW;
+ }
+#endif
#if defined(HAS_COPYROW_ERMS)
if (TestCpuFlag(kCpuHasERMS)) {
CopyRow = CopyRow_ERMS;
@@ -75,6 +77,11 @@
CopyRow = IS_ALIGNED(width, 32) ? CopyRow_NEON : CopyRow_Any_NEON;
}
#endif
+#if defined(HAS_COPYROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ CopyRow = CopyRow_SME;
+ }
+#endif
#if defined(HAS_COPYROW_RVV)
if (TestCpuFlag(kCpuHasRVV)) {
CopyRow = CopyRow_RVV;
@@ -136,6 +143,11 @@
}
}
#endif
+#if defined(HAS_CONVERT16TO8ROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ Convert16To8Row = Convert16To8Row_SME;
+ }
+#endif
#if defined(HAS_CONVERT16TO8ROW_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
Convert16To8Row = Convert16To8Row_Any_SSSE3;
@@ -152,6 +164,14 @@
}
}
#endif
+#if defined(HAS_CONVERT16TO8ROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ Convert16To8Row = Convert16To8Row_Any_AVX512BW;
+ if (IS_ALIGNED(width, 64)) {
+ Convert16To8Row = Convert16To8Row_AVX512BW;
+ }
+ }
+#endif
// Convert plane
for (y = 0; y < height; ++y) {
@@ -214,6 +234,70 @@
}
}
+// Convert a plane of 8 bit data to 8 bit
+LIBYUV_API
+void Convert8To8Plane(const uint8_t* src_y,
+ int src_stride_y,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ int scale, // 220 for Y, 225 to UV
+ int bias, // 16
+ int width,
+ int height) {
+ int y;
+ void (*Convert8To8Row)(const uint8_t* src_y, uint8_t* dst_y, int scale,
+ int bias, int width) = Convert8To8Row_C;
+
+ if (width <= 0 || height == 0) {
+ return;
+ }
+ // Negative height means invert the image.
+ if (height < 0) {
+ height = -height;
+ dst_y = dst_y + (height - 1) * dst_stride_y;
+ dst_stride_y = -dst_stride_y;
+ }
+ // Coalesce rows.
+ if (src_stride_y == width && dst_stride_y == width) {
+ width *= height;
+ height = 1;
+ src_stride_y = dst_stride_y = 0;
+ }
+#if defined(HAS_CONVERT8TO8ROW_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ Convert8To8Row = Convert8To8Row_Any_NEON;
+ if (IS_ALIGNED(width, 32)) {
+ Convert8To8Row = Convert8To8Row_NEON;
+ }
+ }
+#endif
+#if defined(HAS_CONVERT8TO8ROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ Convert8To8Row = Convert8To8Row_SVE2;
+ }
+#endif
+#if defined(HAS_CONVERT8TO8ROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ Convert8To8Row = Convert8To8Row_SME;
+ }
+#endif
+#if defined(HAS_CONVERT8TO8ROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ Convert8To8Row = Convert8To8Row_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ Convert8To8Row = Convert8To8Row_AVX2;
+ }
+ }
+#endif
+
+ // Convert plane
+ for (y = 0; y < height; ++y) {
+ Convert8To8Row(src_y, dst_y, scale, bias, width);
+ src_y += src_stride_y;
+ dst_y += dst_stride_y;
+ }
+}
+
// Copy I422.
LIBYUV_API
int I422Copy(const uint8_t* src_y,
@@ -625,6 +709,11 @@
}
}
#endif
+#if defined(HAS_MERGEUVROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ MergeUVRow = MergeUVRow_SME;
+ }
+#endif
#if defined(HAS_MERGEUVROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
MergeUVRow = MergeUVRow_Any_MSA;
@@ -764,6 +853,11 @@
}
}
#endif
+#if defined(HAS_MERGEUVROW_16_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ MergeUVRow_16 = MergeUVRow_16_SME;
+ }
+#endif
for (y = 0; y < height; ++y) {
// Merge a row of U and V into a row of UV.
@@ -819,6 +913,11 @@
}
}
#endif
+#if defined(HAS_MULTIPLYROW_16_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ MultiplyRow_16 = MultiplyRow_16_SME;
+ }
+#endif
for (y = 0; y < height; ++y) {
MultiplyRow_16(src_y, dst_y, scale, width);
@@ -872,6 +971,11 @@
}
}
#endif
+#if defined(HAS_DIVIDEROW_16_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ DivideRow = DivideRow_16_SVE2;
+ }
+#endif
for (y = 0; y < height; ++y) {
DivideRow(src_y, dst_y, scale, width);
@@ -1275,6 +1379,22 @@
}
}
#endif
+#if defined(HAS_SPLITRGBROW_SSE41)
+ if (TestCpuFlag(kCpuHasSSE41)) {
+ SplitRGBRow = SplitRGBRow_Any_SSE41;
+ if (IS_ALIGNED(width, 16)) {
+ SplitRGBRow = SplitRGBRow_SSE41;
+ }
+ }
+#endif
+#if defined(HAS_SPLITRGBROW_AVX2)
+ if (TestCpuFlag(kCpuHasAVX2)) {
+ SplitRGBRow = SplitRGBRow_Any_AVX2;
+ if (IS_ALIGNED(width, 32)) {
+ SplitRGBRow = SplitRGBRow_AVX2;
+ }
+ }
+#endif
#if defined(HAS_SPLITRGBROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
SplitRGBRow = SplitRGBRow_Any_NEON;
@@ -2597,7 +2717,7 @@
int halfwidth = (width + 1) >> 1;
int halfheight = (height + 1) >> 1;
- if (!src_y || !src_u || !src_v || !dst_u || !dst_v || width <= 0 ||
+ if ((!src_y && dst_y) || !src_u || !src_v || !dst_u || !dst_v || width <= 0 ||
height == 0) {
return -1;
}
@@ -2637,7 +2757,7 @@
int halfwidth = (width + 1) >> 1;
int halfheight = (height + 1) >> 1;
- if (!src_y || !src_uv || !dst_uv || width <= 0 || height == 0) {
+ if ((!src_y && dst_y) || !src_uv || !dst_uv || width <= 0 || height == 0) {
return -1;
}
@@ -3103,6 +3223,11 @@
}
}
#endif
+#if defined(HAS_ARGBMULTIPLYROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ ARGBMultiplyRow = ARGBMultiplyRow_SME;
+ }
+#endif
#if defined(HAS_ARGBMULTIPLYROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
ARGBMultiplyRow = ARGBMultiplyRow_Any_MSA;
@@ -3361,6 +3486,11 @@
}
}
#endif
+#if defined(HAS_RAWTORGB24ROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ RAWToRGB24Row = RAWToRGB24Row_SVE2;
+ }
+#endif
#if defined(HAS_RAWTORGB24ROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
RAWToRGB24Row = RAWToRGB24Row_Any_MSA;
@@ -3741,6 +3871,11 @@
ARGBGrayRow = ARGBGrayRow_NEON;
}
#endif
+#if defined(HAS_ARGBGRAYROW_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd) && IS_ALIGNED(width, 8)) {
+ ARGBGrayRow = ARGBGrayRow_NEON_DotProd;
+ }
+#endif
#if defined(HAS_ARGBGRAYROW_MSA)
if (TestCpuFlag(kCpuHasMSA) && IS_ALIGNED(width, 8)) {
ARGBGrayRow = ARGBGrayRow_MSA;
@@ -3796,6 +3931,11 @@
ARGBGrayRow = ARGBGrayRow_NEON;
}
#endif
+#if defined(HAS_ARGBGRAYROW_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd) && IS_ALIGNED(width, 8)) {
+ ARGBGrayRow = ARGBGrayRow_NEON_DotProd;
+ }
+#endif
#if defined(HAS_ARGBGRAYROW_MSA)
if (TestCpuFlag(kCpuHasMSA) && IS_ALIGNED(width, 8)) {
ARGBGrayRow = ARGBGrayRow_MSA;
@@ -3849,6 +3989,11 @@
ARGBSepiaRow = ARGBSepiaRow_NEON;
}
#endif
+#if defined(HAS_ARGBSEPIAROW_NEON_DOTPROD)
+ if (TestCpuFlag(kCpuHasNeonDotProd) && IS_ALIGNED(width, 8)) {
+ ARGBSepiaRow = ARGBSepiaRow_NEON_DotProd;
+ }
+#endif
#if defined(HAS_ARGBSEPIAROW_MSA)
if (TestCpuFlag(kCpuHasMSA) && IS_ALIGNED(width, 8)) {
ARGBSepiaRow = ARGBSepiaRow_MSA;
@@ -3910,6 +4055,11 @@
ARGBColorMatrixRow = ARGBColorMatrixRow_NEON;
}
#endif
+#if defined(HAS_ARGBCOLORMATRIXROW_NEON_I8MM)
+ if (TestCpuFlag(kCpuHasNeonI8MM) && IS_ALIGNED(width, 8)) {
+ ARGBColorMatrixRow = ARGBColorMatrixRow_NEON_I8MM;
+ }
+#endif
#if defined(HAS_ARGBCOLORMATRIXROW_MSA)
if (TestCpuFlag(kCpuHasMSA) && IS_ALIGNED(width, 8)) {
ARGBColorMatrixRow = ARGBColorMatrixRow_MSA;
@@ -4357,6 +4507,11 @@
}
}
#endif
+#if defined(HAS_INTERPOLATEROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ InterpolateRow = InterpolateRow_SME;
+ }
+#endif
#if defined(HAS_INTERPOLATEROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
InterpolateRow = InterpolateRow_Any_MSA;
@@ -4442,6 +4597,11 @@
}
}
#endif
+#if defined(HAS_INTERPOLATEROW_16_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ InterpolateRow_16 = InterpolateRow_16_SME;
+ }
+#endif
#if defined(HAS_INTERPOLATEROW_16_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
InterpolateRow_16 = InterpolateRow_16_Any_MSA;
@@ -5152,14 +5312,26 @@
}
#endif
#if defined(HAS_HALFFLOATROW_NEON)
- if (TestCpuFlag(kCpuHasNEON)) {
- HalfFloatRow =
- (scale == 1.0f) ? HalfFloat1Row_Any_NEON : HalfFloatRow_Any_NEON;
- if (IS_ALIGNED(width, 8)) {
- HalfFloatRow = (scale == 1.0f) ? HalfFloat1Row_NEON : HalfFloatRow_NEON;
+ if (TestCpuFlag(kCpuHasNEON)
+#if defined(__arm__)
+ // When scale is 1/65535 the scale * 2^-112 used to convert is a denormal.
+ // But when Neon vmul is asked to multiply a normal float by that
+ // denormal scale, even though the result would have been normal, it
+ // flushes to zero. The scalar version of vmul supports denormals.
+ && scale >= 1.0f / 4096.0f
+#endif
+ ) {
+ HalfFloatRow = HalfFloatRow_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ HalfFloatRow = HalfFloatRow_NEON;
}
}
#endif
+#if defined(HAS_HALFFLOATROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ HalfFloatRow = scale == 1.0f ? HalfFloat1Row_SVE2 : HalfFloatRow_SVE2;
+ }
+#endif
#if defined(HAS_HALFFLOATROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
HalfFloatRow = HalfFloatRow_Any_MSA;
@@ -5634,6 +5806,11 @@
}
}
#endif
+#if defined(HAS_INTERPOLATEROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ InterpolateRow = InterpolateRow_SME;
+ }
+#endif
#if defined(HAS_INTERPOLATEROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
InterpolateRow = InterpolateRow_Any_MSA;
diff --git a/source/rotate.cc b/source/rotate.cc
index 3f8332c..1ef2e38 100644
--- a/source/rotate.cc
+++ b/source/rotate.cc
@@ -31,7 +31,12 @@
int width,
int height) {
int i = height;
-#if defined(HAS_TRANSPOSEWX16_MSA) || defined(HAS_TRANSPOSEWX16_LSX)
+#if defined(HAS_TRANSPOSEWXH_SME)
+ void (*TransposeWxH)(const uint8_t* src, int src_stride, uint8_t* dst,
+ int dst_stride, int width, int height) = NULL;
+#endif
+#if defined(HAS_TRANSPOSEWX16_MSA) || defined(HAS_TRANSPOSEWX16_LSX) || \
+ defined(HAS_TRANSPOSEWX16_NEON)
void (*TransposeWx16)(const uint8_t* src, int src_stride, uint8_t* dst,
int dst_stride, int width) = TransposeWx16_C;
#else
@@ -47,6 +52,19 @@
}
}
#endif
+#if defined(HAS_TRANSPOSEWX16_NEON)
+ if (TestCpuFlag(kCpuHasNEON)) {
+ TransposeWx16 = TransposeWx16_Any_NEON;
+ if (IS_ALIGNED(width, 16)) {
+ TransposeWx16 = TransposeWx16_NEON;
+ }
+ }
+#endif
+#if defined(HAS_TRANSPOSEWXH_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ TransposeWxH = TransposeWxH_SME;
+ }
+#endif
#if defined(HAS_TRANSPOSEWX8_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
TransposeWx8 = TransposeWx8_Any_SSSE3;
@@ -80,7 +98,14 @@
}
#endif
-#if defined(HAS_TRANSPOSEWX16_MSA) || defined(HAS_TRANSPOSEWX16_LSX)
+#if defined(HAS_TRANSPOSEWXH_SME)
+ if (TransposeWxH) {
+ TransposeWxH(src, src_stride, dst, dst_stride, width, height);
+ return;
+ }
+#endif
+#if defined(HAS_TRANSPOSEWX16_MSA) || defined(HAS_TRANSPOSEWX16_LSX) || \
+ defined(HAS_TRANSPOSEWX16_NEON)
// Work across the source in 16x16 tiles
while (i >= 16) {
TransposeWx16(src, src_stride, dst, dst_stride, width);
@@ -209,6 +234,11 @@
CopyRow = IS_ALIGNED(width, 64) ? CopyRow_AVX : CopyRow_Any_AVX;
}
#endif
+#if defined(HAS_COPYROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ CopyRow = IS_ALIGNED(width, 128) ? CopyRow_AVX512BW : CopyRow_Any_AVX512BW;
+ }
+#endif
#if defined(HAS_COPYROW_ERMS)
if (TestCpuFlag(kCpuHasERMS)) {
CopyRow = CopyRow_ERMS;
@@ -219,6 +249,11 @@
CopyRow = IS_ALIGNED(width, 32) ? CopyRow_NEON : CopyRow_Any_NEON;
}
#endif
+#if defined(HAS_COPYROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ CopyRow = CopyRow_SME;
+ }
+#endif
#if defined(HAS_COPYROW_RVV)
if (TestCpuFlag(kCpuHasRVV)) {
CopyRow = CopyRow_RVV;
@@ -248,6 +283,11 @@
int width,
int height) {
int i = height;
+#if defined(HAS_TRANSPOSEUVWXH_SME)
+ void (*TransposeUVWxH)(const uint8_t* src, int src_stride, uint8_t* dst_a,
+ int dst_stride_a, uint8_t* dst_b, int dst_stride_b,
+ int width, int height) = TransposeUVWxH_C;
+#endif
#if defined(HAS_TRANSPOSEUVWX16_MSA)
void (*TransposeUVWx16)(const uint8_t* src, int src_stride, uint8_t* dst_a,
int dst_stride_a, uint8_t* dst_b, int dst_stride_b,
@@ -279,7 +319,15 @@
#else
#if defined(HAS_TRANSPOSEUVWX8_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- TransposeUVWx8 = TransposeUVWx8_NEON;
+ TransposeUVWx8 = TransposeUVWx8_Any_NEON;
+ if (IS_ALIGNED(width, 8)) {
+ TransposeUVWx8 = TransposeUVWx8_NEON;
+ }
+ }
+#endif
+#if defined(HAS_TRANSPOSEUVWXH_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ TransposeUVWxH = TransposeUVWxH_SME;
}
#endif
#if defined(HAS_TRANSPOSEUVWX8_SSE2)
@@ -292,6 +340,13 @@
#endif
#endif /* defined(HAS_TRANSPOSEUVWX16_MSA) */
+#if defined(HAS_TRANSPOSEUVWXH_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ TransposeUVWxH(src, src_stride, dst_a, dst_stride_a, dst_b, dst_stride_b,
+ width, i);
+ return;
+ }
+#endif
#if defined(HAS_TRANSPOSEUVWX16_MSA)
// Work through the source in 8x8 tiles.
while (i >= 16) {
diff --git a/source/rotate_any.cc b/source/rotate_any.cc
index 88ca787..ac94253 100644
--- a/source/rotate_any.cc
+++ b/source/rotate_any.cc
@@ -18,31 +18,34 @@
extern "C" {
#endif
-#define TANY(NAMEANY, TPOS_SIMD, MASK) \
- void NAMEANY(const uint8_t* src, int src_stride, uint8_t* dst, \
- int dst_stride, int width) { \
- int r = width & MASK; \
- int n = width - r; \
- if (n > 0) { \
- TPOS_SIMD(src, src_stride, dst, dst_stride, n); \
- } \
- TransposeWx8_C(src + n, src_stride, dst + n * dst_stride, dst_stride, r); \
+#define TANY(NAMEANY, TPOS_SIMD, TPOS_C, MASK) \
+ void NAMEANY(const uint8_t* src, int src_stride, uint8_t* dst, \
+ int dst_stride, int width) { \
+ int r = width & MASK; \
+ int n = width - r; \
+ if (n > 0) { \
+ TPOS_SIMD(src, src_stride, dst, dst_stride, n); \
+ } \
+ TPOS_C(src + n, src_stride, dst + n * dst_stride, dst_stride, r); \
}
#ifdef HAS_TRANSPOSEWX8_NEON
-TANY(TransposeWx8_Any_NEON, TransposeWx8_NEON, 7)
+TANY(TransposeWx8_Any_NEON, TransposeWx8_NEON, TransposeWx8_C, 7)
+#endif
+#ifdef HAS_TRANSPOSEWX16_NEON
+TANY(TransposeWx16_Any_NEON, TransposeWx16_NEON, TransposeWx16_C, 15)
#endif
#ifdef HAS_TRANSPOSEWX8_SSSE3
-TANY(TransposeWx8_Any_SSSE3, TransposeWx8_SSSE3, 7)
+TANY(TransposeWx8_Any_SSSE3, TransposeWx8_SSSE3, TransposeWx8_C, 7)
#endif
#ifdef HAS_TRANSPOSEWX8_FAST_SSSE3
-TANY(TransposeWx8_Fast_Any_SSSE3, TransposeWx8_Fast_SSSE3, 15)
+TANY(TransposeWx8_Fast_Any_SSSE3, TransposeWx8_Fast_SSSE3, TransposeWx8_C, 15)
#endif
#ifdef HAS_TRANSPOSEWX16_MSA
-TANY(TransposeWx16_Any_MSA, TransposeWx16_MSA, 15)
+TANY(TransposeWx16_Any_MSA, TransposeWx16_MSA, TransposeWx16_C, 15)
#endif
#ifdef HAS_TRANSPOSEWX16_LSX
-TANY(TransposeWx16_Any_LSX, TransposeWx16_LSX, 15)
+TANY(TransposeWx16_Any_LSX, TransposeWx16_LSX, TransposeWx16_C, 15)
#endif
#undef TANY
diff --git a/source/rotate_argb.cc b/source/rotate_argb.cc
index d55fac4..7fda09d 100644
--- a/source/rotate_argb.cc
+++ b/source/rotate_argb.cc
@@ -189,6 +189,12 @@
CopyRow = IS_ALIGNED(width * 4, 64) ? CopyRow_AVX : CopyRow_Any_AVX;
}
#endif
+#if defined(HAS_COPYROW_AVX512BW)
+ if (TestCpuFlag(kCpuHasAVX512BW)) {
+ CopyRow =
+ IS_ALIGNED(width * 4, 128) ? CopyRow_AVX512BW : CopyRow_Any_AVX512BW;
+ }
+#endif
#if defined(HAS_COPYROW_ERMS)
if (TestCpuFlag(kCpuHasERMS)) {
CopyRow = CopyRow_ERMS;
@@ -199,6 +205,11 @@
CopyRow = IS_ALIGNED(width * 4, 32) ? CopyRow_NEON : CopyRow_Any_NEON;
}
#endif
+#if defined(HAS_COPYROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ CopyRow = CopyRow_SME;
+ }
+#endif
#if defined(HAS_COPYROW_RVV)
if (TestCpuFlag(kCpuHasRVV)) {
CopyRow = CopyRow_RVV;
diff --git a/source/rotate_common.cc b/source/rotate_common.cc
index e72608e..e0341fe 100644
--- a/source/rotate_common.cc
+++ b/source/rotate_common.cc
@@ -36,6 +36,16 @@
}
}
+void TransposeWx16_C(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width) {
+ TransposeWx8_C(src, src_stride, dst, dst_stride, width);
+ TransposeWx8_C((src + 8 * src_stride), src_stride, (dst + 8), dst_stride,
+ width);
+}
+
void TransposeUVWx8_C(const uint8_t* src,
int src_stride,
uint8_t* dst_a,
diff --git a/source/rotate_gcc.cc b/source/rotate_gcc.cc
index fd5eee0..ae7436b 100644
--- a/source/rotate_gcc.cc
+++ b/source/rotate_gcc.cc
@@ -17,7 +17,9 @@
#endif
// This module is for GCC x86 and x64.
-#if !defined(LIBYUV_DISABLE_X86) && (defined(__x86_64__) || defined(__i386__))
+#if !defined(LIBYUV_DISABLE_X86) && \
+ (defined(__x86_64__) || defined(__i386__)) && \
+ !defined(LIBYUV_ENABLE_ROWWIN)
// Transpose 8x8. 32 or 64 bit, but not NaCL for 64 bit.
#if defined(HAS_TRANSPOSEWX8_SSSE3)
@@ -30,7 +32,7 @@
// Read in the data from the source pointer.
// First round of bit swap.
LABELALIGN
- "1: \n"
+ "1: \n"
"movq (%0),%%xmm0 \n"
"movq (%0,%3),%%xmm1 \n"
"lea (%0,%3,2),%0 \n"
@@ -120,7 +122,7 @@
// Read in the data from the source pointer.
// First round of bit swap.
LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n"
"movdqu (%0,%3),%%xmm1 \n"
"lea (%0,%3,2),%0 \n"
@@ -265,7 +267,7 @@
// Read in the data from the source pointer.
// First round of bit swap.
LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n"
"movdqu (%0,%4),%%xmm1 \n"
"lea (%0,%4,2),%0 \n"
@@ -393,7 +395,7 @@
int width) {
asm volatile(
// Main loop transpose 4x4. Read a column, write a row.
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n" // a b c d
"movdqu (%0,%3),%%xmm1 \n" // e f g h
"lea (%0,%3,2),%0 \n" // src += stride * 2
@@ -449,7 +451,7 @@
int width) {
asm volatile(
// Main loop transpose 2 blocks of 4x4. Read a column, write a row.
- "1: \n"
+ "1: \n"
"vmovdqu (%0),%%xmm0 \n" // a b c d
"vmovdqu (%0,%3),%%xmm1 \n" // e f g h
"lea (%0,%3,2),%0 \n" // src += stride * 2
@@ -484,7 +486,7 @@
"sub %4,%1 \n"
"sub $0x8,%2 \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: "+r"(src), // %0
"+r"(dst), // %1
"+rm"(width) // %2
diff --git a/source/rotate_lsx.cc b/source/rotate_lsx.cc
index 94a2b91..b292803 100644
--- a/source/rotate_lsx.cc
+++ b/source/rotate_lsx.cc
@@ -61,16 +61,6 @@
_dst += _stride2; \
}
-void TransposeWx16_C(const uint8_t* src,
- int src_stride,
- uint8_t* dst,
- int dst_stride,
- int width) {
- TransposeWx8_C(src, src_stride, dst, dst_stride, width);
- TransposeWx8_C((src + 8 * src_stride), src_stride, (dst + 8), dst_stride,
- width);
-}
-
void TransposeUVWx16_C(const uint8_t* src,
int src_stride,
uint8_t* dst_a,
diff --git a/source/rotate_msa.cc b/source/rotate_msa.cc
index 99bdca6..d4e62b1 100644
--- a/source/rotate_msa.cc
+++ b/source/rotate_msa.cc
@@ -51,16 +51,6 @@
out3 = (v16u8)__msa_ilvl_d((v2i64)in3, (v2i64)in2); \
}
-void TransposeWx16_C(const uint8_t* src,
- int src_stride,
- uint8_t* dst,
- int dst_stride,
- int width) {
- TransposeWx8_C(src, src_stride, dst, dst_stride, width);
- TransposeWx8_C((src + 8 * src_stride), src_stride, (dst + 8), dst_stride,
- width);
-}
-
void TransposeUVWx16_C(const uint8_t* src,
int src_stride,
uint8_t* dst_a,
diff --git a/source/rotate_neon.cc b/source/rotate_neon.cc
index 569a731..27bd225 100644
--- a/source/rotate_neon.cc
+++ b/source/rotate_neon.cc
@@ -21,38 +21,35 @@
#if !defined(LIBYUV_DISABLE_NEON) && defined(__ARM_NEON__) && \
!defined(__aarch64__)
-static const uvec8 kVTbl4x4Transpose = {0, 4, 8, 12, 1, 5, 9, 13,
- 2, 6, 10, 14, 3, 7, 11, 15};
-
void TransposeWx8_NEON(const uint8_t* src,
int src_stride,
uint8_t* dst,
int dst_stride,
int width) {
- const uint8_t* src_temp;
+ const uint8_t* temp;
asm volatile(
// loops are on blocks of 8. loop will stop when
// counter gets to or below 0. starting the counter
// at w-8 allow for this
- "sub %5, #8 \n"
+ "sub %[width], #8 \n"
- // handle 8x8 blocks. this should be the majority of the plane
- "1: \n"
- "mov %0, %1 \n"
-
- "vld1.8 {d0}, [%0], %2 \n"
- "vld1.8 {d1}, [%0], %2 \n"
- "vld1.8 {d2}, [%0], %2 \n"
- "vld1.8 {d3}, [%0], %2 \n"
- "vld1.8 {d4}, [%0], %2 \n"
- "vld1.8 {d5}, [%0], %2 \n"
- "vld1.8 {d6}, [%0], %2 \n"
- "vld1.8 {d7}, [%0] \n"
+ "1: \n"
+ "mov %[temp], %[src] \n"
+ "vld1.8 {d0}, [%[temp]], %[src_stride] \n"
+ "vld1.8 {d1}, [%[temp]], %[src_stride] \n"
+ "vld1.8 {d2}, [%[temp]], %[src_stride] \n"
+ "vld1.8 {d3}, [%[temp]], %[src_stride] \n"
+ "vld1.8 {d4}, [%[temp]], %[src_stride] \n"
+ "vld1.8 {d5}, [%[temp]], %[src_stride] \n"
+ "vld1.8 {d6}, [%[temp]], %[src_stride] \n"
+ "vld1.8 {d7}, [%[temp]] \n"
+ "add %[src], #8 \n"
"vtrn.8 d1, d0 \n"
"vtrn.8 d3, d2 \n"
"vtrn.8 d5, d4 \n"
"vtrn.8 d7, d6 \n"
+ "subs %[width], #8 \n"
"vtrn.16 d1, d3 \n"
"vtrn.16 d0, d2 \n"
@@ -69,129 +66,27 @@
"vrev16.8 q2, q2 \n"
"vrev16.8 q3, q3 \n"
- "mov %0, %3 \n"
+ "mov %[temp], %[dst] \n"
+ "vst1.8 {d1}, [%[temp]], %[dst_stride] \n"
+ "vst1.8 {d0}, [%[temp]], %[dst_stride] \n"
+ "vst1.8 {d3}, [%[temp]], %[dst_stride] \n"
+ "vst1.8 {d2}, [%[temp]], %[dst_stride] \n"
+ "vst1.8 {d5}, [%[temp]], %[dst_stride] \n"
+ "vst1.8 {d4}, [%[temp]], %[dst_stride] \n"
+ "vst1.8 {d7}, [%[temp]], %[dst_stride] \n"
+ "vst1.8 {d6}, [%[temp]] \n"
+ "add %[dst], %[dst], %[dst_stride], lsl #3 \n"
- "vst1.8 {d1}, [%0], %4 \n"
- "vst1.8 {d0}, [%0], %4 \n"
- "vst1.8 {d3}, [%0], %4 \n"
- "vst1.8 {d2}, [%0], %4 \n"
- "vst1.8 {d5}, [%0], %4 \n"
- "vst1.8 {d4}, [%0], %4 \n"
- "vst1.8 {d7}, [%0], %4 \n"
- "vst1.8 {d6}, [%0] \n"
-
- "add %1, #8 \n" // src += 8
- "add %3, %3, %4, lsl #3 \n" // dst += 8 * dst_stride
- "subs %5, #8 \n" // w -= 8
"bge 1b \n"
-
- // add 8 back to counter. if the result is 0 there are
- // no residuals.
- "adds %5, #8 \n"
- "beq 4f \n"
-
- // some residual, so between 1 and 7 lines left to transpose
- "cmp %5, #2 \n"
- "blt 3f \n"
-
- "cmp %5, #4 \n"
- "blt 2f \n"
-
- // 4x8 block
- "mov %0, %1 \n"
- "vld1.32 {d0[0]}, [%0], %2 \n"
- "vld1.32 {d0[1]}, [%0], %2 \n"
- "vld1.32 {d1[0]}, [%0], %2 \n"
- "vld1.32 {d1[1]}, [%0], %2 \n"
- "vld1.32 {d2[0]}, [%0], %2 \n"
- "vld1.32 {d2[1]}, [%0], %2 \n"
- "vld1.32 {d3[0]}, [%0], %2 \n"
- "vld1.32 {d3[1]}, [%0] \n"
-
- "mov %0, %3 \n"
-
- "vld1.8 {q3}, [%6] \n"
-
- "vtbl.8 d4, {d0, d1}, d6 \n"
- "vtbl.8 d5, {d0, d1}, d7 \n"
- "vtbl.8 d0, {d2, d3}, d6 \n"
- "vtbl.8 d1, {d2, d3}, d7 \n"
-
- // TODO(frkoenig): Rework shuffle above to
- // write out with 4 instead of 8 writes.
- "vst1.32 {d4[0]}, [%0], %4 \n"
- "vst1.32 {d4[1]}, [%0], %4 \n"
- "vst1.32 {d5[0]}, [%0], %4 \n"
- "vst1.32 {d5[1]}, [%0] \n"
-
- "add %0, %3, #4 \n"
- "vst1.32 {d0[0]}, [%0], %4 \n"
- "vst1.32 {d0[1]}, [%0], %4 \n"
- "vst1.32 {d1[0]}, [%0], %4 \n"
- "vst1.32 {d1[1]}, [%0] \n"
-
- "add %1, #4 \n" // src += 4
- "add %3, %3, %4, lsl #2 \n" // dst += 4 * dst_stride
- "subs %5, #4 \n" // w -= 4
- "beq 4f \n"
-
- // some residual, check to see if it includes a 2x8 block,
- // or less
- "cmp %5, #2 \n"
- "blt 3f \n"
-
- // 2x8 block
- "2: \n"
- "mov %0, %1 \n"
- "vld1.16 {d0[0]}, [%0], %2 \n"
- "vld1.16 {d1[0]}, [%0], %2 \n"
- "vld1.16 {d0[1]}, [%0], %2 \n"
- "vld1.16 {d1[1]}, [%0], %2 \n"
- "vld1.16 {d0[2]}, [%0], %2 \n"
- "vld1.16 {d1[2]}, [%0], %2 \n"
- "vld1.16 {d0[3]}, [%0], %2 \n"
- "vld1.16 {d1[3]}, [%0] \n"
-
- "vtrn.8 d0, d1 \n"
-
- "mov %0, %3 \n"
-
- "vst1.64 {d0}, [%0], %4 \n"
- "vst1.64 {d1}, [%0] \n"
-
- "add %1, #2 \n" // src += 2
- "add %3, %3, %4, lsl #1 \n" // dst += 2 * dst_stride
- "subs %5, #2 \n" // w -= 2
- "beq 4f \n"
-
- // 1x8 block
- "3: \n"
- "vld1.8 {d0[0]}, [%1], %2 \n"
- "vld1.8 {d0[1]}, [%1], %2 \n"
- "vld1.8 {d0[2]}, [%1], %2 \n"
- "vld1.8 {d0[3]}, [%1], %2 \n"
- "vld1.8 {d0[4]}, [%1], %2 \n"
- "vld1.8 {d0[5]}, [%1], %2 \n"
- "vld1.8 {d0[6]}, [%1], %2 \n"
- "vld1.8 {d0[7]}, [%1] \n"
-
- "vst1.64 {d0}, [%3] \n"
-
- "4: \n"
-
- : "=&r"(src_temp), // %0
- "+r"(src), // %1
- "+r"(src_stride), // %2
- "+r"(dst), // %3
- "+r"(dst_stride), // %4
- "+r"(width) // %5
- : "r"(&kVTbl4x4Transpose) // %6
+ : [temp] "=&r"(temp), // %[temp]
+ [src] "+r"(src), // %[src]
+ [dst] "+r"(dst), // %[dst]
+ [width] "+r"(width) // %[width]
+ : [src_stride] "r"(src_stride), // %[src_stride]
+ [dst_stride] "r"(dst_stride) // %[dst_stride]
: "memory", "cc", "q0", "q1", "q2", "q3");
}
-static const uvec8 kVTbl4x4TransposeDi = {0, 8, 1, 9, 2, 10, 3, 11,
- 4, 12, 5, 13, 6, 14, 7, 15};
-
void TransposeUVWx8_NEON(const uint8_t* src,
int src_stride,
uint8_t* dst_a,
@@ -199,30 +94,30 @@
uint8_t* dst_b,
int dst_stride_b,
int width) {
- const uint8_t* src_temp;
+ const uint8_t* temp;
asm volatile(
// loops are on blocks of 8. loop will stop when
// counter gets to or below 0. starting the counter
// at w-8 allow for this
- "sub %7, #8 \n"
+ "sub %[width], #8 \n"
- // handle 8x8 blocks. this should be the majority of the plane
- "1: \n"
- "mov %0, %1 \n"
-
- "vld2.8 {d0, d1}, [%0], %2 \n"
- "vld2.8 {d2, d3}, [%0], %2 \n"
- "vld2.8 {d4, d5}, [%0], %2 \n"
- "vld2.8 {d6, d7}, [%0], %2 \n"
- "vld2.8 {d16, d17}, [%0], %2 \n"
- "vld2.8 {d18, d19}, [%0], %2 \n"
- "vld2.8 {d20, d21}, [%0], %2 \n"
- "vld2.8 {d22, d23}, [%0] \n"
+ "1: \n"
+ "mov %[temp], %[src] \n"
+ "vld2.8 {d0, d1}, [%[temp]], %[src_stride] \n"
+ "vld2.8 {d2, d3}, [%[temp]], %[src_stride] \n"
+ "vld2.8 {d4, d5}, [%[temp]], %[src_stride] \n"
+ "vld2.8 {d6, d7}, [%[temp]], %[src_stride] \n"
+ "vld2.8 {d16, d17}, [%[temp]], %[src_stride] \n"
+ "vld2.8 {d18, d19}, [%[temp]], %[src_stride] \n"
+ "vld2.8 {d20, d21}, [%[temp]], %[src_stride] \n"
+ "vld2.8 {d22, d23}, [%[temp]] \n"
+ "add %[src], #8*2 \n"
"vtrn.8 q1, q0 \n"
"vtrn.8 q3, q2 \n"
"vtrn.8 q9, q8 \n"
"vtrn.8 q11, q10 \n"
+ "subs %[width], #8 \n"
"vtrn.16 q1, q3 \n"
"vtrn.16 q0, q2 \n"
@@ -243,171 +138,37 @@
"vrev16.8 q10, q10 \n"
"vrev16.8 q11, q11 \n"
- "mov %0, %3 \n"
+ "mov %[temp], %[dst_a] \n"
+ "vst1.8 {d2}, [%[temp]], %[dst_stride_a] \n"
+ "vst1.8 {d0}, [%[temp]], %[dst_stride_a] \n"
+ "vst1.8 {d6}, [%[temp]], %[dst_stride_a] \n"
+ "vst1.8 {d4}, [%[temp]], %[dst_stride_a] \n"
+ "vst1.8 {d18}, [%[temp]], %[dst_stride_a] \n"
+ "vst1.8 {d16}, [%[temp]], %[dst_stride_a] \n"
+ "vst1.8 {d22}, [%[temp]], %[dst_stride_a] \n"
+ "vst1.8 {d20}, [%[temp]] \n"
+ "add %[dst_a], %[dst_a], %[dst_stride_a], lsl #3 \n"
- "vst1.8 {d2}, [%0], %4 \n"
- "vst1.8 {d0}, [%0], %4 \n"
- "vst1.8 {d6}, [%0], %4 \n"
- "vst1.8 {d4}, [%0], %4 \n"
- "vst1.8 {d18}, [%0], %4 \n"
- "vst1.8 {d16}, [%0], %4 \n"
- "vst1.8 {d22}, [%0], %4 \n"
- "vst1.8 {d20}, [%0] \n"
+ "mov %[temp], %[dst_b] \n"
+ "vst1.8 {d3}, [%[temp]], %[dst_stride_b] \n"
+ "vst1.8 {d1}, [%[temp]], %[dst_stride_b] \n"
+ "vst1.8 {d7}, [%[temp]], %[dst_stride_b] \n"
+ "vst1.8 {d5}, [%[temp]], %[dst_stride_b] \n"
+ "vst1.8 {d19}, [%[temp]], %[dst_stride_b] \n"
+ "vst1.8 {d17}, [%[temp]], %[dst_stride_b] \n"
+ "vst1.8 {d23}, [%[temp]], %[dst_stride_b] \n"
+ "vst1.8 {d21}, [%[temp]] \n"
+ "add %[dst_b], %[dst_b], %[dst_stride_b], lsl #3 \n"
- "mov %0, %5 \n"
-
- "vst1.8 {d3}, [%0], %6 \n"
- "vst1.8 {d1}, [%0], %6 \n"
- "vst1.8 {d7}, [%0], %6 \n"
- "vst1.8 {d5}, [%0], %6 \n"
- "vst1.8 {d19}, [%0], %6 \n"
- "vst1.8 {d17}, [%0], %6 \n"
- "vst1.8 {d23}, [%0], %6 \n"
- "vst1.8 {d21}, [%0] \n"
-
- "add %1, #8*2 \n" // src += 8*2
- "add %3, %3, %4, lsl #3 \n" // dst_a += 8 *
- // dst_stride_a
- "add %5, %5, %6, lsl #3 \n" // dst_b += 8 *
- // dst_stride_b
- "subs %7, #8 \n" // w -= 8
"bge 1b \n"
-
- // add 8 back to counter. if the result is 0 there are
- // no residuals.
- "adds %7, #8 \n"
- "beq 4f \n"
-
- // some residual, so between 1 and 7 lines left to transpose
- "cmp %7, #2 \n"
- "blt 3f \n"
-
- "cmp %7, #4 \n"
- "blt 2f \n"
-
- // TODO(frkoenig): Clean this up
- // 4x8 block
- "mov %0, %1 \n"
- "vld1.64 {d0}, [%0], %2 \n"
- "vld1.64 {d1}, [%0], %2 \n"
- "vld1.64 {d2}, [%0], %2 \n"
- "vld1.64 {d3}, [%0], %2 \n"
- "vld1.64 {d4}, [%0], %2 \n"
- "vld1.64 {d5}, [%0], %2 \n"
- "vld1.64 {d6}, [%0], %2 \n"
- "vld1.64 {d7}, [%0] \n"
-
- "vld1.8 {q15}, [%8] \n"
-
- "vtrn.8 q0, q1 \n"
- "vtrn.8 q2, q3 \n"
-
- "vtbl.8 d16, {d0, d1}, d30 \n"
- "vtbl.8 d17, {d0, d1}, d31 \n"
- "vtbl.8 d18, {d2, d3}, d30 \n"
- "vtbl.8 d19, {d2, d3}, d31 \n"
- "vtbl.8 d20, {d4, d5}, d30 \n"
- "vtbl.8 d21, {d4, d5}, d31 \n"
- "vtbl.8 d22, {d6, d7}, d30 \n"
- "vtbl.8 d23, {d6, d7}, d31 \n"
-
- "mov %0, %3 \n"
-
- "vst1.32 {d16[0]}, [%0], %4 \n"
- "vst1.32 {d16[1]}, [%0], %4 \n"
- "vst1.32 {d17[0]}, [%0], %4 \n"
- "vst1.32 {d17[1]}, [%0], %4 \n"
-
- "add %0, %3, #4 \n"
- "vst1.32 {d20[0]}, [%0], %4 \n"
- "vst1.32 {d20[1]}, [%0], %4 \n"
- "vst1.32 {d21[0]}, [%0], %4 \n"
- "vst1.32 {d21[1]}, [%0] \n"
-
- "mov %0, %5 \n"
-
- "vst1.32 {d18[0]}, [%0], %6 \n"
- "vst1.32 {d18[1]}, [%0], %6 \n"
- "vst1.32 {d19[0]}, [%0], %6 \n"
- "vst1.32 {d19[1]}, [%0], %6 \n"
-
- "add %0, %5, #4 \n"
- "vst1.32 {d22[0]}, [%0], %6 \n"
- "vst1.32 {d22[1]}, [%0], %6 \n"
- "vst1.32 {d23[0]}, [%0], %6 \n"
- "vst1.32 {d23[1]}, [%0] \n"
-
- "add %1, #4*2 \n" // src += 4 * 2
- "add %3, %3, %4, lsl #2 \n" // dst_a += 4 *
- // dst_stride_a
- "add %5, %5, %6, lsl #2 \n" // dst_b += 4 *
- // dst_stride_b
- "subs %7, #4 \n" // w -= 4
- "beq 4f \n"
-
- // some residual, check to see if it includes a 2x8 block,
- // or less
- "cmp %7, #2 \n"
- "blt 3f \n"
-
- // 2x8 block
- "2: \n"
- "mov %0, %1 \n"
- "vld2.16 {d0[0], d2[0]}, [%0], %2 \n"
- "vld2.16 {d1[0], d3[0]}, [%0], %2 \n"
- "vld2.16 {d0[1], d2[1]}, [%0], %2 \n"
- "vld2.16 {d1[1], d3[1]}, [%0], %2 \n"
- "vld2.16 {d0[2], d2[2]}, [%0], %2 \n"
- "vld2.16 {d1[2], d3[2]}, [%0], %2 \n"
- "vld2.16 {d0[3], d2[3]}, [%0], %2 \n"
- "vld2.16 {d1[3], d3[3]}, [%0] \n"
-
- "vtrn.8 d0, d1 \n"
- "vtrn.8 d2, d3 \n"
-
- "mov %0, %3 \n"
-
- "vst1.64 {d0}, [%0], %4 \n"
- "vst1.64 {d2}, [%0] \n"
-
- "mov %0, %5 \n"
-
- "vst1.64 {d1}, [%0], %6 \n"
- "vst1.64 {d3}, [%0] \n"
-
- "add %1, #2*2 \n" // src += 2 * 2
- "add %3, %3, %4, lsl #1 \n" // dst_a += 2 *
- // dst_stride_a
- "add %5, %5, %6, lsl #1 \n" // dst_b += 2 *
- // dst_stride_b
- "subs %7, #2 \n" // w -= 2
- "beq 4f \n"
-
- // 1x8 block
- "3: \n"
- "vld2.8 {d0[0], d1[0]}, [%1], %2 \n"
- "vld2.8 {d0[1], d1[1]}, [%1], %2 \n"
- "vld2.8 {d0[2], d1[2]}, [%1], %2 \n"
- "vld2.8 {d0[3], d1[3]}, [%1], %2 \n"
- "vld2.8 {d0[4], d1[4]}, [%1], %2 \n"
- "vld2.8 {d0[5], d1[5]}, [%1], %2 \n"
- "vld2.8 {d0[6], d1[6]}, [%1], %2 \n"
- "vld2.8 {d0[7], d1[7]}, [%1] \n"
-
- "vst1.64 {d0}, [%3] \n"
- "vst1.64 {d1}, [%5] \n"
-
- "4: \n"
-
- : "=&r"(src_temp), // %0
- "+r"(src), // %1
- "+r"(src_stride), // %2
- "+r"(dst_a), // %3
- "+r"(dst_stride_a), // %4
- "+r"(dst_b), // %5
- "+r"(dst_stride_b), // %6
- "+r"(width) // %7
- : "r"(&kVTbl4x4TransposeDi) // %8
+ : [temp] "=&r"(temp), // %[temp]
+ [src] "+r"(src), // %[src]
+ [dst_a] "+r"(dst_a), // %[dst_a]
+ [dst_b] "+r"(dst_b), // %[dst_b]
+ [width] "+r"(width) // %[width]
+ : [src_stride] "r"(src_stride), // %[src_stride]
+ [dst_stride_a] "r"(dst_stride_a), // %[dst_stride_a]
+ [dst_stride_b] "r"(dst_stride_b) // %[dst_stride_b]
: "memory", "cc", "q0", "q1", "q2", "q3", "q8", "q9", "q10", "q11");
}
@@ -425,7 +186,7 @@
uint8_t* dst3 = dst2 + dst_stride;
asm volatile(
// Main loop transpose 4x4. Read a column, write a row.
- "1: \n"
+ "1: \n"
"vld4.32 {d0[0], d2[0], d4[0], d6[0]}, [%0], %9 \n"
"vld4.32 {d0[1], d2[1], d4[1], d6[1]}, [%1], %9 \n"
"vld4.32 {d1[0], d3[0], d5[0], d7[0]}, [%2], %9 \n"
diff --git a/source/rotate_neon64.cc b/source/rotate_neon64.cc
index 95047fa..e09bcb1 100644
--- a/source/rotate_neon64.cc
+++ b/source/rotate_neon64.cc
@@ -21,201 +21,122 @@
// This module is for GCC Neon armv8 64 bit.
#if !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__)
-static const uvec8 kVTbl4x4Transpose = {0, 4, 8, 12, 1, 5, 9, 13,
- 2, 6, 10, 14, 3, 7, 11, 15};
-
-void TransposeWx8_NEON(const uint8_t* src,
- int src_stride,
- uint8_t* dst,
- int dst_stride,
- int width) {
+void TransposeWx16_NEON(const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width) {
const uint8_t* src_temp;
asm volatile(
- // loops are on blocks of 8. loop will stop when
- // counter gets to or below 0. starting the counter
- // at w-8 allow for this
- "sub %w3, %w3, #8 \n"
+ "1: \n"
+ "mov %[src_temp], %[src] \n"
- // handle 8x8 blocks. this should be the majority of the plane
- "1: \n"
- "mov %0, %1 \n"
+ "ld1 {v16.16b}, [%[src_temp]], %[src_stride] \n"
+ "ld1 {v17.16b}, [%[src_temp]], %[src_stride] \n"
+ "ld1 {v18.16b}, [%[src_temp]], %[src_stride] \n"
+ "ld1 {v19.16b}, [%[src_temp]], %[src_stride] \n"
+ "ld1 {v20.16b}, [%[src_temp]], %[src_stride] \n"
+ "ld1 {v21.16b}, [%[src_temp]], %[src_stride] \n"
+ "ld1 {v22.16b}, [%[src_temp]], %[src_stride] \n"
+ "ld1 {v23.16b}, [%[src_temp]], %[src_stride] \n"
+ "ld1 {v24.16b}, [%[src_temp]], %[src_stride] \n"
+ "ld1 {v25.16b}, [%[src_temp]], %[src_stride] \n"
+ "ld1 {v26.16b}, [%[src_temp]], %[src_stride] \n"
+ "ld1 {v27.16b}, [%[src_temp]], %[src_stride] \n"
+ "ld1 {v28.16b}, [%[src_temp]], %[src_stride] \n"
+ "ld1 {v29.16b}, [%[src_temp]], %[src_stride] \n"
+ "ld1 {v30.16b}, [%[src_temp]], %[src_stride] \n"
+ "ld1 {v31.16b}, [%[src_temp]], %[src_stride] \n"
- "ld1 {v0.8b}, [%0], %5 \n"
- "ld1 {v1.8b}, [%0], %5 \n"
- "ld1 {v2.8b}, [%0], %5 \n"
- "ld1 {v3.8b}, [%0], %5 \n"
- "ld1 {v4.8b}, [%0], %5 \n"
- "ld1 {v5.8b}, [%0], %5 \n"
- "ld1 {v6.8b}, [%0], %5 \n"
- "ld1 {v7.8b}, [%0] \n"
- "mov %0, %1 \n"
+ "add %[src], %[src], #16 \n"
- "trn2 v16.8b, v0.8b, v1.8b \n"
- "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead
- "trn1 v17.8b, v0.8b, v1.8b \n"
- "add %0, %0, %5 \n"
- "trn2 v18.8b, v2.8b, v3.8b \n"
- "prfm pldl1keep, [%0, 448] \n" // row 1
- "trn1 v19.8b, v2.8b, v3.8b \n"
- "add %0, %0, %5 \n"
- "trn2 v20.8b, v4.8b, v5.8b \n"
- "prfm pldl1keep, [%0, 448] \n" // row 2
- "trn1 v21.8b, v4.8b, v5.8b \n"
- "add %0, %0, %5 \n"
- "trn2 v22.8b, v6.8b, v7.8b \n"
- "prfm pldl1keep, [%0, 448] \n" // row 3
- "trn1 v23.8b, v6.8b, v7.8b \n"
- "add %0, %0, %5 \n"
+ // Transpose bytes within each 2x2 block.
+ "trn1 v0.16b, v16.16b, v17.16b \n"
+ "trn2 v1.16b, v16.16b, v17.16b \n"
+ "trn1 v2.16b, v18.16b, v19.16b \n"
+ "trn2 v3.16b, v18.16b, v19.16b \n"
+ "trn1 v4.16b, v20.16b, v21.16b \n"
+ "trn2 v5.16b, v20.16b, v21.16b \n"
+ "trn1 v6.16b, v22.16b, v23.16b \n"
+ "trn2 v7.16b, v22.16b, v23.16b \n"
+ "trn1 v8.16b, v24.16b, v25.16b \n"
+ "trn2 v9.16b, v24.16b, v25.16b \n"
+ "trn1 v10.16b, v26.16b, v27.16b \n"
+ "trn2 v11.16b, v26.16b, v27.16b \n"
+ "trn1 v12.16b, v28.16b, v29.16b \n"
+ "trn2 v13.16b, v28.16b, v29.16b \n"
+ "trn1 v14.16b, v30.16b, v31.16b \n"
+ "trn2 v15.16b, v30.16b, v31.16b \n"
- "trn2 v3.4h, v17.4h, v19.4h \n"
- "prfm pldl1keep, [%0, 448] \n" // row 4
- "trn1 v1.4h, v17.4h, v19.4h \n"
- "add %0, %0, %5 \n"
- "trn2 v2.4h, v16.4h, v18.4h \n"
- "prfm pldl1keep, [%0, 448] \n" // row 5
- "trn1 v0.4h, v16.4h, v18.4h \n"
- "add %0, %0, %5 \n"
- "trn2 v7.4h, v21.4h, v23.4h \n"
- "prfm pldl1keep, [%0, 448] \n" // row 6
- "trn1 v5.4h, v21.4h, v23.4h \n"
- "add %0, %0, %5 \n"
- "trn2 v6.4h, v20.4h, v22.4h \n"
- "prfm pldl1keep, [%0, 448] \n" // row 7
- "trn1 v4.4h, v20.4h, v22.4h \n"
+ // Transpose 2x2-byte blocks within each 4x4 block.
+ "trn1 v16.8h, v0.8h, v2.8h \n"
+ "trn1 v17.8h, v1.8h, v3.8h \n"
+ "trn2 v18.8h, v0.8h, v2.8h \n"
+ "trn2 v19.8h, v1.8h, v3.8h \n"
+ "trn1 v20.8h, v4.8h, v6.8h \n"
+ "trn1 v21.8h, v5.8h, v7.8h \n"
+ "trn2 v22.8h, v4.8h, v6.8h \n"
+ "trn2 v23.8h, v5.8h, v7.8h \n"
+ "trn1 v24.8h, v8.8h, v10.8h \n"
+ "trn1 v25.8h, v9.8h, v11.8h \n"
+ "trn2 v26.8h, v8.8h, v10.8h \n"
+ "trn2 v27.8h, v9.8h, v11.8h \n"
+ "trn1 v28.8h, v12.8h, v14.8h \n"
+ "trn1 v29.8h, v13.8h, v15.8h \n"
+ "trn2 v30.8h, v12.8h, v14.8h \n"
+ "trn2 v31.8h, v13.8h, v15.8h \n"
- "trn2 v21.2s, v1.2s, v5.2s \n"
- "trn1 v17.2s, v1.2s, v5.2s \n"
- "trn2 v20.2s, v0.2s, v4.2s \n"
- "trn1 v16.2s, v0.2s, v4.2s \n"
- "trn2 v23.2s, v3.2s, v7.2s \n"
- "trn1 v19.2s, v3.2s, v7.2s \n"
- "trn2 v22.2s, v2.2s, v6.2s \n"
- "trn1 v18.2s, v2.2s, v6.2s \n"
+ "subs %w[width], %w[width], #16 \n"
- "mov %0, %2 \n"
+ // Transpose 4x4-byte blocks within each 8x8 block.
+ "trn1 v0.4s, v16.4s, v20.4s \n"
+ "trn1 v2.4s, v17.4s, v21.4s \n"
+ "trn1 v4.4s, v18.4s, v22.4s \n"
+ "trn1 v6.4s, v19.4s, v23.4s \n"
+ "trn2 v8.4s, v16.4s, v20.4s \n"
+ "trn2 v10.4s, v17.4s, v21.4s \n"
+ "trn2 v12.4s, v18.4s, v22.4s \n"
+ "trn2 v14.4s, v19.4s, v23.4s \n"
+ "trn1 v1.4s, v24.4s, v28.4s \n"
+ "trn1 v3.4s, v25.4s, v29.4s \n"
+ "trn1 v5.4s, v26.4s, v30.4s \n"
+ "trn1 v7.4s, v27.4s, v31.4s \n"
+ "trn2 v9.4s, v24.4s, v28.4s \n"
+ "trn2 v11.4s, v25.4s, v29.4s \n"
+ "trn2 v13.4s, v26.4s, v30.4s \n"
+ "trn2 v15.4s, v27.4s, v31.4s \n"
- "st1 {v17.8b}, [%0], %6 \n"
- "st1 {v16.8b}, [%0], %6 \n"
- "st1 {v19.8b}, [%0], %6 \n"
- "st1 {v18.8b}, [%0], %6 \n"
- "st1 {v21.8b}, [%0], %6 \n"
- "st1 {v20.8b}, [%0], %6 \n"
- "st1 {v23.8b}, [%0], %6 \n"
- "st1 {v22.8b}, [%0] \n"
+ // Transpose 8x8-byte blocks and store.
+ "st2 {v0.d, v1.d}[0], [%[dst]], %[dst_stride] \n"
+ "st2 {v2.d, v3.d}[0], [%[dst]], %[dst_stride] \n"
+ "st2 {v4.d, v5.d}[0], [%[dst]], %[dst_stride] \n"
+ "st2 {v6.d, v7.d}[0], [%[dst]], %[dst_stride] \n"
+ "st2 {v8.d, v9.d}[0], [%[dst]], %[dst_stride] \n"
+ "st2 {v10.d, v11.d}[0], [%[dst]], %[dst_stride] \n"
+ "st2 {v12.d, v13.d}[0], [%[dst]], %[dst_stride] \n"
+ "st2 {v14.d, v15.d}[0], [%[dst]], %[dst_stride] \n"
+ "st2 {v0.d, v1.d}[1], [%[dst]], %[dst_stride] \n"
+ "st2 {v2.d, v3.d}[1], [%[dst]], %[dst_stride] \n"
+ "st2 {v4.d, v5.d}[1], [%[dst]], %[dst_stride] \n"
+ "st2 {v6.d, v7.d}[1], [%[dst]], %[dst_stride] \n"
+ "st2 {v8.d, v9.d}[1], [%[dst]], %[dst_stride] \n"
+ "st2 {v10.d, v11.d}[1], [%[dst]], %[dst_stride] \n"
+ "st2 {v12.d, v13.d}[1], [%[dst]], %[dst_stride] \n"
+ "st2 {v14.d, v15.d}[1], [%[dst]], %[dst_stride] \n"
- "add %1, %1, #8 \n" // src += 8
- "add %2, %2, %6, lsl #3 \n" // dst += 8 * dst_stride
- "subs %w3, %w3, #8 \n" // w -= 8
- "b.ge 1b \n"
-
- // add 8 back to counter. if the result is 0 there are
- // no residuals.
- "adds %w3, %w3, #8 \n"
- "b.eq 4f \n"
-
- // some residual, so between 1 and 7 lines left to transpose
- "cmp %w3, #2 \n"
- "b.lt 3f \n"
-
- "cmp %w3, #4 \n"
- "b.lt 2f \n"
-
- // 4x8 block
- "mov %0, %1 \n"
- "ld1 {v0.s}[0], [%0], %5 \n"
- "ld1 {v0.s}[1], [%0], %5 \n"
- "ld1 {v0.s}[2], [%0], %5 \n"
- "ld1 {v0.s}[3], [%0], %5 \n"
- "ld1 {v1.s}[0], [%0], %5 \n"
- "ld1 {v1.s}[1], [%0], %5 \n"
- "ld1 {v1.s}[2], [%0], %5 \n"
- "ld1 {v1.s}[3], [%0] \n"
-
- "mov %0, %2 \n"
-
- "ld1 {v2.16b}, [%4] \n"
-
- "tbl v3.16b, {v0.16b}, v2.16b \n"
- "tbl v0.16b, {v1.16b}, v2.16b \n"
-
- // TODO(frkoenig): Rework shuffle above to
- // write out with 4 instead of 8 writes.
- "st1 {v3.s}[0], [%0], %6 \n"
- "st1 {v3.s}[1], [%0], %6 \n"
- "st1 {v3.s}[2], [%0], %6 \n"
- "st1 {v3.s}[3], [%0] \n"
-
- "add %0, %2, #4 \n"
- "st1 {v0.s}[0], [%0], %6 \n"
- "st1 {v0.s}[1], [%0], %6 \n"
- "st1 {v0.s}[2], [%0], %6 \n"
- "st1 {v0.s}[3], [%0] \n"
-
- "add %1, %1, #4 \n" // src += 4
- "add %2, %2, %6, lsl #2 \n" // dst += 4 * dst_stride
- "subs %w3, %w3, #4 \n" // w -= 4
- "b.eq 4f \n"
-
- // some residual, check to see if it includes a 2x8 block,
- // or less
- "cmp %w3, #2 \n"
- "b.lt 3f \n"
-
- // 2x8 block
- "2: \n"
- "mov %0, %1 \n"
- "ld1 {v0.h}[0], [%0], %5 \n"
- "ld1 {v1.h}[0], [%0], %5 \n"
- "ld1 {v0.h}[1], [%0], %5 \n"
- "ld1 {v1.h}[1], [%0], %5 \n"
- "ld1 {v0.h}[2], [%0], %5 \n"
- "ld1 {v1.h}[2], [%0], %5 \n"
- "ld1 {v0.h}[3], [%0], %5 \n"
- "ld1 {v1.h}[3], [%0] \n"
-
- "trn2 v2.8b, v0.8b, v1.8b \n"
- "trn1 v3.8b, v0.8b, v1.8b \n"
-
- "mov %0, %2 \n"
-
- "st1 {v3.8b}, [%0], %6 \n"
- "st1 {v2.8b}, [%0] \n"
-
- "add %1, %1, #2 \n" // src += 2
- "add %2, %2, %6, lsl #1 \n" // dst += 2 * dst_stride
- "subs %w3, %w3, #2 \n" // w -= 2
- "b.eq 4f \n"
-
- // 1x8 block
- "3: \n"
- "ld1 {v0.b}[0], [%1], %5 \n"
- "ld1 {v0.b}[1], [%1], %5 \n"
- "ld1 {v0.b}[2], [%1], %5 \n"
- "ld1 {v0.b}[3], [%1], %5 \n"
- "ld1 {v0.b}[4], [%1], %5 \n"
- "ld1 {v0.b}[5], [%1], %5 \n"
- "ld1 {v0.b}[6], [%1], %5 \n"
- "ld1 {v0.b}[7], [%1] \n"
-
- "st1 {v0.8b}, [%2] \n"
-
- "4: \n"
-
- : "=&r"(src_temp), // %0
- "+r"(src), // %1
- "+r"(dst), // %2
- "+r"(width) // %3
- : "r"(&kVTbl4x4Transpose), // %4
- "r"((ptrdiff_t)src_stride), // %5
- "r"((ptrdiff_t)dst_stride) // %6
- : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16",
- "v17", "v18", "v19", "v20", "v21", "v22", "v23");
+ "b.gt 1b \n"
+ : [src] "+r"(src), // %[src]
+ [src_temp] "=&r"(src_temp), // %[src_temp]
+ [dst] "+r"(dst), // %[dst]
+ [width] "+r"(width) // %[width]
+ : [src_stride] "r"((ptrdiff_t)src_stride), // %[src_stride]
+ [dst_stride] "r"((ptrdiff_t)dst_stride) // %[dst_stride]
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8",
+ "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18",
+ "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28",
+ "v29", "v30", "v31");
}
-static const uint8_t kVTbl4x4TransposeDi[32] = {
- 0, 16, 32, 48, 2, 18, 34, 50, 4, 20, 36, 52, 6, 22, 38, 54,
- 1, 17, 33, 49, 3, 19, 35, 51, 5, 21, 37, 53, 7, 23, 39, 55};
-
void TransposeUVWx8_NEON(const uint8_t* src,
int src_stride,
uint8_t* dst_a,
@@ -223,26 +144,24 @@
uint8_t* dst_b,
int dst_stride_b,
int width) {
- const uint8_t* src_temp;
+ const uint8_t* temp;
asm volatile(
// loops are on blocks of 8. loop will stop when
// counter gets to or below 0. starting the counter
// at w-8 allow for this
- "sub %w4, %w4, #8 \n"
+ "sub %w[width], %w[width], #8 \n"
- // handle 8x8 blocks. this should be the majority of the plane
- "1: \n"
- "mov %0, %1 \n"
-
- "ld1 {v0.16b}, [%0], %5 \n"
- "ld1 {v1.16b}, [%0], %5 \n"
- "ld1 {v2.16b}, [%0], %5 \n"
- "ld1 {v3.16b}, [%0], %5 \n"
- "ld1 {v4.16b}, [%0], %5 \n"
- "ld1 {v5.16b}, [%0], %5 \n"
- "ld1 {v6.16b}, [%0], %5 \n"
- "ld1 {v7.16b}, [%0] \n"
- "mov %0, %1 \n"
+ "1: \n"
+ "mov %[temp], %[src] \n"
+ "ld1 {v0.16b}, [%[temp]], %[src_stride] \n"
+ "ld1 {v1.16b}, [%[temp]], %[src_stride] \n"
+ "ld1 {v2.16b}, [%[temp]], %[src_stride] \n"
+ "ld1 {v3.16b}, [%[temp]], %[src_stride] \n"
+ "ld1 {v4.16b}, [%[temp]], %[src_stride] \n"
+ "ld1 {v5.16b}, [%[temp]], %[src_stride] \n"
+ "ld1 {v6.16b}, [%[temp]], %[src_stride] \n"
+ "ld1 {v7.16b}, [%[temp]] \n"
+ "add %[src], %[src], #16 \n"
"trn1 v16.16b, v0.16b, v1.16b \n"
"trn2 v17.16b, v0.16b, v1.16b \n"
@@ -253,6 +172,8 @@
"trn1 v22.16b, v6.16b, v7.16b \n"
"trn2 v23.16b, v6.16b, v7.16b \n"
+ "subs %w[width], %w[width], #8 \n"
+
"trn1 v0.8h, v16.8h, v18.8h \n"
"trn2 v1.8h, v16.8h, v18.8h \n"
"trn1 v2.8h, v20.8h, v22.8h \n"
@@ -271,167 +192,37 @@
"trn1 v22.4s, v5.4s, v7.4s \n"
"trn2 v23.4s, v5.4s, v7.4s \n"
- "mov %0, %2 \n"
+ "mov %[temp], %[dst_a] \n"
+ "st1 {v16.d}[0], [%[temp]], %[dst_stride_a] \n"
+ "st1 {v18.d}[0], [%[temp]], %[dst_stride_a] \n"
+ "st1 {v17.d}[0], [%[temp]], %[dst_stride_a] \n"
+ "st1 {v19.d}[0], [%[temp]], %[dst_stride_a] \n"
+ "st1 {v16.d}[1], [%[temp]], %[dst_stride_a] \n"
+ "st1 {v18.d}[1], [%[temp]], %[dst_stride_a] \n"
+ "st1 {v17.d}[1], [%[temp]], %[dst_stride_a] \n"
+ "st1 {v19.d}[1], [%[temp]] \n"
+ "add %[dst_a], %[dst_a], %[dst_stride_a], lsl #3 \n"
- "st1 {v16.d}[0], [%0], %6 \n"
- "st1 {v18.d}[0], [%0], %6 \n"
- "st1 {v17.d}[0], [%0], %6 \n"
- "st1 {v19.d}[0], [%0], %6 \n"
- "st1 {v16.d}[1], [%0], %6 \n"
- "st1 {v18.d}[1], [%0], %6 \n"
- "st1 {v17.d}[1], [%0], %6 \n"
- "st1 {v19.d}[1], [%0] \n"
+ "mov %[temp], %[dst_b] \n"
+ "st1 {v20.d}[0], [%[temp]], %[dst_stride_b] \n"
+ "st1 {v22.d}[0], [%[temp]], %[dst_stride_b] \n"
+ "st1 {v21.d}[0], [%[temp]], %[dst_stride_b] \n"
+ "st1 {v23.d}[0], [%[temp]], %[dst_stride_b] \n"
+ "st1 {v20.d}[1], [%[temp]], %[dst_stride_b] \n"
+ "st1 {v22.d}[1], [%[temp]], %[dst_stride_b] \n"
+ "st1 {v21.d}[1], [%[temp]], %[dst_stride_b] \n"
+ "st1 {v23.d}[1], [%[temp]] \n"
+ "add %[dst_b], %[dst_b], %[dst_stride_b], lsl #3 \n"
- "mov %0, %3 \n"
-
- "st1 {v20.d}[0], [%0], %7 \n"
- "st1 {v22.d}[0], [%0], %7 \n"
- "st1 {v21.d}[0], [%0], %7 \n"
- "st1 {v23.d}[0], [%0], %7 \n"
- "st1 {v20.d}[1], [%0], %7 \n"
- "st1 {v22.d}[1], [%0], %7 \n"
- "st1 {v21.d}[1], [%0], %7 \n"
- "st1 {v23.d}[1], [%0] \n"
-
- "add %1, %1, #16 \n" // src += 8*2
- "add %2, %2, %6, lsl #3 \n" // dst_a += 8 *
- // dst_stride_a
- "add %3, %3, %7, lsl #3 \n" // dst_b += 8 *
- // dst_stride_b
- "subs %w4, %w4, #8 \n" // w -= 8
"b.ge 1b \n"
-
- // add 8 back to counter. if the result is 0 there are
- // no residuals.
- "adds %w4, %w4, #8 \n"
- "b.eq 4f \n"
-
- // some residual, so between 1 and 7 lines left to transpose
- "cmp %w4, #2 \n"
- "b.lt 3f \n"
-
- "cmp %w4, #4 \n"
- "b.lt 2f \n"
-
- // TODO(frkoenig): Clean this up
- // 4x8 block
- "mov %0, %1 \n"
- "ld1 {v0.8b}, [%0], %5 \n"
- "ld1 {v1.8b}, [%0], %5 \n"
- "ld1 {v2.8b}, [%0], %5 \n"
- "ld1 {v3.8b}, [%0], %5 \n"
- "ld1 {v4.8b}, [%0], %5 \n"
- "ld1 {v5.8b}, [%0], %5 \n"
- "ld1 {v6.8b}, [%0], %5 \n"
- "ld1 {v7.8b}, [%0] \n"
-
- "ld1 {v30.16b}, [%8], #16 \n"
- "ld1 {v31.16b}, [%8] \n"
-
- "tbl v16.16b, {v0.16b, v1.16b, v2.16b, v3.16b}, v30.16b \n"
- "tbl v17.16b, {v0.16b, v1.16b, v2.16b, v3.16b}, v31.16b \n"
- "tbl v18.16b, {v4.16b, v5.16b, v6.16b, v7.16b}, v30.16b \n"
- "tbl v19.16b, {v4.16b, v5.16b, v6.16b, v7.16b}, v31.16b \n"
-
- "mov %0, %2 \n"
-
- "st1 {v16.s}[0], [%0], %6 \n"
- "st1 {v16.s}[1], [%0], %6 \n"
- "st1 {v16.s}[2], [%0], %6 \n"
- "st1 {v16.s}[3], [%0], %6 \n"
-
- "add %0, %2, #4 \n"
- "st1 {v18.s}[0], [%0], %6 \n"
- "st1 {v18.s}[1], [%0], %6 \n"
- "st1 {v18.s}[2], [%0], %6 \n"
- "st1 {v18.s}[3], [%0] \n"
-
- "mov %0, %3 \n"
-
- "st1 {v17.s}[0], [%0], %7 \n"
- "st1 {v17.s}[1], [%0], %7 \n"
- "st1 {v17.s}[2], [%0], %7 \n"
- "st1 {v17.s}[3], [%0], %7 \n"
-
- "add %0, %3, #4 \n"
- "st1 {v19.s}[0], [%0], %7 \n"
- "st1 {v19.s}[1], [%0], %7 \n"
- "st1 {v19.s}[2], [%0], %7 \n"
- "st1 {v19.s}[3], [%0] \n"
-
- "add %1, %1, #8 \n" // src += 4 * 2
- "add %2, %2, %6, lsl #2 \n" // dst_a += 4 *
- // dst_stride_a
- "add %3, %3, %7, lsl #2 \n" // dst_b += 4 *
- // dst_stride_b
- "subs %w4, %w4, #4 \n" // w -= 4
- "b.eq 4f \n"
-
- // some residual, check to see if it includes a 2x8 block,
- // or less
- "cmp %w4, #2 \n"
- "b.lt 3f \n"
-
- // 2x8 block
- "2: \n"
- "mov %0, %1 \n"
- "ld2 {v0.h, v1.h}[0], [%0], %5 \n"
- "ld2 {v2.h, v3.h}[0], [%0], %5 \n"
- "ld2 {v0.h, v1.h}[1], [%0], %5 \n"
- "ld2 {v2.h, v3.h}[1], [%0], %5 \n"
- "ld2 {v0.h, v1.h}[2], [%0], %5 \n"
- "ld2 {v2.h, v3.h}[2], [%0], %5 \n"
- "ld2 {v0.h, v1.h}[3], [%0], %5 \n"
- "ld2 {v2.h, v3.h}[3], [%0] \n"
-
- "trn1 v4.8b, v0.8b, v2.8b \n"
- "trn2 v5.8b, v0.8b, v2.8b \n"
- "trn1 v6.8b, v1.8b, v3.8b \n"
- "trn2 v7.8b, v1.8b, v3.8b \n"
-
- "mov %0, %2 \n"
-
- "st1 {v4.d}[0], [%0], %6 \n"
- "st1 {v6.d}[0], [%0] \n"
-
- "mov %0, %3 \n"
-
- "st1 {v5.d}[0], [%0], %7 \n"
- "st1 {v7.d}[0], [%0] \n"
-
- "add %1, %1, #4 \n" // src += 2 * 2
- "add %2, %2, %6, lsl #1 \n" // dst_a += 2 *
- // dst_stride_a
- "add %3, %3, %7, lsl #1 \n" // dst_b += 2 *
- // dst_stride_b
- "subs %w4, %w4, #2 \n" // w -= 2
- "b.eq 4f \n"
-
- // 1x8 block
- "3: \n"
- "ld2 {v0.b, v1.b}[0], [%1], %5 \n"
- "ld2 {v0.b, v1.b}[1], [%1], %5 \n"
- "ld2 {v0.b, v1.b}[2], [%1], %5 \n"
- "ld2 {v0.b, v1.b}[3], [%1], %5 \n"
- "ld2 {v0.b, v1.b}[4], [%1], %5 \n"
- "ld2 {v0.b, v1.b}[5], [%1], %5 \n"
- "ld2 {v0.b, v1.b}[6], [%1], %5 \n"
- "ld2 {v0.b, v1.b}[7], [%1] \n"
-
- "st1 {v0.d}[0], [%2] \n"
- "st1 {v1.d}[0], [%3] \n"
-
- "4: \n"
-
- : "=&r"(src_temp), // %0
- "+r"(src), // %1
- "+r"(dst_a), // %2
- "+r"(dst_b), // %3
- "+r"(width) // %4
- : "r"((ptrdiff_t)src_stride), // %5
- "r"((ptrdiff_t)dst_stride_a), // %6
- "r"((ptrdiff_t)dst_stride_b), // %7
- "r"(&kVTbl4x4TransposeDi) // %8
+ : [temp] "=&r"(temp), // %[temp]
+ [src] "+r"(src), // %[src]
+ [dst_a] "+r"(dst_a), // %[dst_a]
+ [dst_b] "+r"(dst_b), // %[dst_b]
+ [width] "+r"(width) // %[width]
+ : [src_stride] "r"((ptrdiff_t)src_stride), // %[src_stride]
+ [dst_stride_a] "r"((ptrdiff_t)dst_stride_a), // %[dst_stride_a]
+ [dst_stride_b] "r"((ptrdiff_t)dst_stride_b) // %[dst_stride_b]
: "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16",
"v17", "v18", "v19", "v20", "v21", "v22", "v23", "v30", "v31");
}
@@ -450,7 +241,7 @@
uint8_t* dst3 = dst2 + dst_stride;
asm volatile(
// Main loop transpose 4x4. Read a column, write a row.
- "1: \n"
+ "1: \n"
"ld4 {v0.s, v1.s, v2.s, v3.s}[0], [%0], %9 \n"
"ld4 {v0.s, v1.s, v2.s, v3.s}[1], [%1], %9 \n"
"ld4 {v0.s, v1.s, v2.s, v3.s}[2], [%2], %9 \n"
diff --git a/source/rotate_sme.cc b/source/rotate_sme.cc
new file mode 100644
index 0000000..dc9ab0c
--- /dev/null
+++ b/source/rotate_sme.cc
@@ -0,0 +1,174 @@
+/*
+ * Copyright 2024 The LibYuv Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "libyuv/rotate_row.h"
+#include "libyuv/row.h"
+
+#include "libyuv/basic_types.h"
+
+#ifdef __cplusplus
+namespace libyuv {
+extern "C" {
+#endif
+
+#if !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && \
+ defined(__aarch64__)
+
+__arm_locally_streaming __arm_new("za") void TransposeWxH_SME(
+ const uint8_t* src,
+ int src_stride,
+ uint8_t* dst,
+ int dst_stride,
+ int width,
+ int height) {
+ int vl;
+ asm("cntb %x0" : "=r"(vl));
+
+ do {
+ const uint8_t* src2 = src;
+ uint8_t* dst2 = dst;
+
+ // Process up to VL elements per iteration of the inner loop.
+ int block_height = height > vl ? vl : height;
+
+ int width2 = width;
+ do {
+ const uint8_t* src3 = src2;
+
+ // Process up to VL elements per iteration of the inner loop.
+ int block_width = width2 > vl ? vl : width2;
+
+ asm volatile(
+ "mov w12, #0 \n"
+
+ // Create a predicate to handle loading partial rows.
+ "whilelt p0.b, wzr, %w[block_width] \n"
+
+ // Load H <= VL rows into ZA0.
+ "1: \n"
+ "ld1b {za0h.b[w12, 0]}, p0/z, [%[src3]] \n"
+ "add %[src3], %[src3], %[src_stride] \n"
+ "add w12, w12, #1 \n"
+ "cmp w12, %w[block_height] \n"
+ "b.ne 1b \n"
+
+ // Create a predicate to handle storing partial columns.
+ "whilelt p0.b, wzr, %w[block_height] \n"
+ "mov w12, #0 \n"
+
+ // Store W <= VL columns from ZA0.
+ "2: \n"
+ "st1b {za0v.b[w12, 0]}, p0, [%[dst2]] \n"
+ "add %[dst2], %[dst2], %[dst_stride] \n"
+ "add w12, w12, #1 \n"
+ "cmp w12, %w[block_width] \n"
+ "b.ne 2b \n"
+ : [src3] "+r"(src3), // %[src3]
+ [dst2] "+r"(dst2) // %[dst2]
+ : [src_stride] "r"((ptrdiff_t)src_stride), // %[src_stride]
+ [dst_stride] "r"((ptrdiff_t)dst_stride), // %[dst_stride]
+ [block_width] "r"(block_width), // %[block_width]
+ [block_height] "r"(block_height) // %[block_height]
+ : "cc", "memory", "p0", "w12", "za");
+
+ src2 += vl;
+ width2 -= vl;
+ } while (width2 > 0);
+
+ src += vl * src_stride;
+ dst += vl;
+ height -= vl;
+ } while (height > 0);
+}
+
+__arm_locally_streaming __arm_new("za") void TransposeUVWxH_SME(
+ const uint8_t* src,
+ int src_stride,
+ uint8_t* dst_a,
+ int dst_stride_a,
+ uint8_t* dst_b,
+ int dst_stride_b,
+ int width,
+ int height) {
+ int vl;
+ asm("cnth %x0" : "=r"(vl));
+
+ do {
+ const uint8_t* src2 = src;
+ uint8_t* dst2_a = dst_a;
+ uint8_t* dst2_b = dst_b;
+
+ // Process up to VL bytes per iteration of the inner loop.
+ int block_height = height > vl * 2 ? vl * 2 : height;
+
+ int width2 = width;
+ do {
+ const uint8_t* src3 = src2;
+
+ // Process up to VL 16-bit elements per iteration of the inner loop.
+ int block_width = width2 > vl ? vl : width2;
+
+ asm volatile(
+ "mov w12, #0 \n"
+
+ // Create a predicate to handle loading partial rows,
+ // %[block_width] is always a multiple of two here.
+ "whilelt p0.b, wzr, %w[block_width] \n"
+
+ // Load H <= VL rows into ZA0, such that U/V components exist in
+ // alternating columns.
+ "1: \n"
+ "ld1b {za0h.b[w12, 0]}, p0/z, [%[src]] \n"
+ "add %[src], %[src], %[src_stride] \n"
+ "add w12, w12, #1 \n"
+ "cmp w12, %w[block_height] \n"
+ "b.ne 1b \n"
+
+ // Create a predicate to handle storing partial columns.
+ "whilelt p0.b, wzr, %w[block_height] \n"
+ "mov w12, #0 \n"
+
+ // Store alternating UV data from pairs of ZA0 columns.
+ "2: \n"
+ "st1b {za0v.b[w12, 0]}, p0, [%[dst_a]] \n"
+ "st1b {za0v.b[w12, 1]}, p0, [%[dst_b]] \n"
+ "add %[dst_a], %[dst_a], %[dst_stride_a] \n"
+ "add %[dst_b], %[dst_b], %[dst_stride_b] \n"
+ "add w12, w12, #2 \n"
+ "cmp w12, %w[block_width] \n"
+ "b.ne 2b \n"
+ : [src] "+r"(src3), // %[src]
+ [dst_a] "+r"(dst2_a), // %[dst_a]
+ [dst_b] "+r"(dst2_b) // %[dst_b]
+ : [src_stride] "r"((ptrdiff_t)src_stride), // %[src_stride]
+ [dst_stride_a] "r"((ptrdiff_t)dst_stride_a), // %[dst_stride_a]
+ [dst_stride_b] "r"((ptrdiff_t)dst_stride_b), // %[dst_stride_b]
+ [block_width] "r"(block_width * 2), // %[block_width]
+ [block_height] "r"(block_height) // %[block_height]
+ : "cc", "memory", "p0", "w12", "za");
+
+ src2 += 2 * vl;
+ width2 -= vl;
+ } while (width2 > 0);
+
+ src += 2 * vl * src_stride;
+ dst_a += 2 * vl;
+ dst_b += 2 * vl;
+ height -= 2 * vl;
+ } while (height > 0);
+}
+
+#endif // !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) &&
+ // defined(__aarch64__)
+
+#ifdef __cplusplus
+} // extern "C"
+} // namespace libyuv
+#endif
diff --git a/source/rotate_win.cc b/source/rotate_win.cc
index a78873f..03eeee3 100644
--- a/source/rotate_win.cc
+++ b/source/rotate_win.cc
@@ -17,8 +17,8 @@
#endif
// This module is for 32 bit Visual C x86
-#if !defined(LIBYUV_DISABLE_X86) && defined(_MSC_VER) && \
- !defined(__clang__) && defined(_M_IX86)
+#if !defined(LIBYUV_DISABLE_X86) && defined(_MSC_VER) && defined(_M_IX86) && \
+ (!defined(__clang__) || defined(LIBYUV_ENABLE_ROWWIN))
__declspec(naked) void TransposeWx8_SSSE3(const uint8_t* src,
int src_stride,
diff --git a/source/row_any.cc b/source/row_any.cc
index e574543..85fb6ff 100644
--- a/source/row_any.cc
+++ b/source/row_any.cc
@@ -142,6 +142,27 @@
memcpy(dst_ptr + (n >> DUVSHIFT) * BPP, vout, SS(r, DUVSHIFT) * BPP); \
}
+#ifdef HAS_I210ALPHATOARGBROW_NEON
+ANY41CT(I210AlphaToARGBRow_Any_NEON,
+ I210AlphaToARGBRow_NEON,
+ 1,
+ 0,
+ uint16_t,
+ 2,
+ 4,
+ 7)
+#endif
+#ifdef HAS_I410ALPHATOARGBROW_NEON
+ANY41CT(I410AlphaToARGBRow_Any_NEON,
+ I410AlphaToARGBRow_NEON,
+ 0,
+ 0,
+ uint16_t,
+ 2,
+ 4,
+ 7)
+#endif
+
#ifdef HAS_I210ALPHATOARGBROW_SSSE3
ANY41CT(I210AlphaToARGBRow_Any_SSSE3,
I210AlphaToARGBRow_SSSE3,
@@ -399,6 +420,9 @@
#ifdef HAS_I444TORGB24ROW_NEON
ANY31C(I444ToRGB24Row_Any_NEON, I444ToRGB24Row_NEON, 0, 0, 3, 7)
#endif
+#ifdef HAS_I422TOAR30ROW_NEON
+ANY31C(I422ToAR30Row_Any_NEON, I422ToAR30Row_NEON, 1, 0, 4, 7)
+#endif
#ifdef HAS_I422TOARGBROW_NEON
ANY31C(I444ToARGBRow_Any_NEON, I444ToARGBRow_NEON, 0, 0, 4, 7)
ANY31C(I422ToARGBRow_Any_NEON, I422ToARGBRow_NEON, 1, 0, 4, 7)
@@ -495,6 +519,24 @@
#ifdef HAS_I212TOAR30ROW_AVX2
ANY31CT(I212ToAR30Row_Any_AVX2, I212ToAR30Row_AVX2, 1, 0, uint16_t, 2, 4, 15)
#endif
+#ifdef HAS_I210TOARGBROW_NEON
+ANY31CT(I210ToARGBRow_Any_NEON, I210ToARGBRow_NEON, 1, 0, uint16_t, 2, 4, 7)
+#endif
+#ifdef HAS_I410TOARGBROW_NEON
+ANY31CT(I410ToARGBRow_Any_NEON, I410ToARGBRow_NEON, 0, 0, uint16_t, 2, 4, 7)
+#endif
+#ifdef HAS_I210TOAR30ROW_NEON
+ANY31CT(I210ToAR30Row_Any_NEON, I210ToAR30Row_NEON, 1, 0, uint16_t, 2, 4, 7)
+#endif
+#ifdef HAS_I410TOAR30ROW_NEON
+ANY31CT(I410ToAR30Row_Any_NEON, I410ToAR30Row_NEON, 0, 0, uint16_t, 2, 4, 7)
+#endif
+#ifdef HAS_I212TOARGBROW_NEON
+ANY31CT(I212ToARGBRow_Any_NEON, I212ToARGBRow_NEON, 1, 0, uint16_t, 2, 4, 7)
+#endif
+#ifdef HAS_I212TOAR30ROW_NEON
+ANY31CT(I212ToAR30Row_Any_NEON, I212ToAR30Row_NEON, 1, 0, uint16_t, 2, 4, 7)
+#endif
#undef ANY31CT
// Any 3 planes to 1 plane with parameter
@@ -528,7 +570,7 @@
2,
uint8_t,
4,
- 3)
+ 7)
#endif
#ifdef HAS_MERGEXR64ROW_AVX2
@@ -855,6 +897,12 @@
#ifdef HAS_P210TOAR30ROW_AVX2
ANY21CT(P210ToAR30Row_Any_AVX2, P210ToAR30Row_AVX2, 1, 0, uint16_t, 2, 4, 15)
#endif
+#ifdef HAS_P210TOAR30ROW_NEON
+ANY21CT(P210ToAR30Row_Any_NEON, P210ToAR30Row_NEON, 1, 0, uint16_t, 2, 4, 7)
+#endif
+#ifdef HAS_P210TOARGBROW_NEON
+ANY21CT(P210ToARGBRow_Any_NEON, P210ToARGBRow_NEON, 1, 0, uint16_t, 2, 4, 7)
+#endif
#ifdef HAS_P410TOAR30ROW_SSSE3
ANY21CT(P410ToAR30Row_Any_SSSE3, P410ToAR30Row_SSSE3, 0, 0, uint16_t, 2, 4, 7)
#endif
@@ -867,6 +915,12 @@
#ifdef HAS_P410TOAR30ROW_AVX2
ANY21CT(P410ToAR30Row_Any_AVX2, P410ToAR30Row_AVX2, 0, 0, uint16_t, 2, 4, 15)
#endif
+#ifdef HAS_P410TOAR30ROW_NEON
+ANY21CT(P410ToAR30Row_Any_NEON, P410ToAR30Row_NEON, 0, 0, uint16_t, 2, 4, 7)
+#endif
+#ifdef HAS_P410TOARGBROW_NEON
+ANY21CT(P410ToARGBRow_Any_NEON, P410ToARGBRow_NEON, 0, 0, uint16_t, 2, 4, 7)
+#endif
#undef ANY21CT
@@ -913,6 +967,9 @@
memcpy(dst_ptr + n * BPP, vout, r * BPP); \
}
+#ifdef HAS_COPYROW_AVX512BW
+ANY11(CopyRow_Any_AVX512BW, CopyRow_AVX512BW, 0, 1, 1, 127)
+#endif
#ifdef HAS_COPYROW_AVX
ANY11(CopyRow_Any_AVX, CopyRow_AVX, 0, 1, 1, 63)
#endif
@@ -948,6 +1005,12 @@
#if defined(HAS_ABGRTOAR30ROW_SSSE3)
ANY11(ABGRToAR30Row_Any_SSSE3, ABGRToAR30Row_SSSE3, 0, 4, 4, 3)
#endif
+#if defined(HAS_ABGRTOAR30ROW_NEON)
+ANY11(ABGRToAR30Row_Any_NEON, ABGRToAR30Row_NEON, 0, 4, 4, 7)
+#endif
+#if defined(HAS_ARGBTOAR30ROW_NEON)
+ANY11(ARGBToAR30Row_Any_NEON, ARGBToAR30Row_NEON, 0, 4, 4, 7)
+#endif
#if defined(HAS_ARGBTOAR30ROW_SSSE3)
ANY11(ARGBToAR30Row_Any_SSSE3, ARGBToAR30Row_SSSE3, 0, 4, 4, 3)
#endif
@@ -970,6 +1033,9 @@
ANY11(ARGB1555ToARGBRow_Any_SSE2, ARGB1555ToARGBRow_SSE2, 0, 2, 4, 7)
ANY11(ARGB4444ToARGBRow_Any_SSE2, ARGB4444ToARGBRow_SSE2, 0, 2, 4, 7)
#endif
+#if defined(HAS_RAWTOARGBROW_AVX2)
+ANY11(RAWToARGBRow_Any_AVX2, RAWToARGBRow_AVX2, 0, 3, 4, 31)
+#endif
#if defined(HAS_RAWTORGBAROW_SSSE3)
ANY11(RAWToRGBARow_Any_SSSE3, RAWToRGBARow_SSSE3, 0, 3, 4, 15)
#endif
@@ -1072,6 +1138,9 @@
#ifdef HAS_ARGBTOYROW_NEON
ANY11(ARGBToYRow_Any_NEON, ARGBToYRow_NEON, 0, 4, 1, 15)
#endif
+#ifdef HAS_ARGBTOYROW_NEON_DOTPROD
+ANY11(ARGBToYRow_Any_NEON_DotProd, ARGBToYRow_NEON_DotProd, 0, 4, 1, 15)
+#endif
#ifdef HAS_ARGBTOYROW_MSA
ANY11(ARGBToYRow_Any_MSA, ARGBToYRow_MSA, 0, 4, 1, 15)
#endif
@@ -1084,12 +1153,21 @@
#ifdef HAS_ARGBTOYJROW_NEON
ANY11(ARGBToYJRow_Any_NEON, ARGBToYJRow_NEON, 0, 4, 1, 15)
#endif
+#ifdef HAS_ARGBTOYJROW_NEON_DOTPROD
+ANY11(ARGBToYJRow_Any_NEON_DotProd, ARGBToYJRow_NEON_DotProd, 0, 4, 1, 15)
+#endif
#ifdef HAS_ABGRTOYJROW_NEON
ANY11(ABGRToYJRow_Any_NEON, ABGRToYJRow_NEON, 0, 4, 1, 15)
#endif
+#ifdef HAS_ABGRTOYJROW_NEON_DOTPROD
+ANY11(ABGRToYJRow_Any_NEON_DotProd, ABGRToYJRow_NEON_DotProd, 0, 4, 1, 15)
+#endif
#ifdef HAS_RGBATOYJROW_NEON
ANY11(RGBAToYJRow_Any_NEON, RGBAToYJRow_NEON, 0, 4, 1, 15)
#endif
+#ifdef HAS_RGBATOYJROW_NEON_DOTPROD
+ANY11(RGBAToYJRow_Any_NEON_DotProd, RGBAToYJRow_NEON_DotProd, 0, 4, 1, 15)
+#endif
#ifdef HAS_ARGBTOYJROW_MSA
ANY11(ARGBToYJRow_Any_MSA, ARGBToYJRow_MSA, 0, 4, 1, 15)
#endif
@@ -1114,6 +1192,9 @@
#ifdef HAS_BGRATOYROW_NEON
ANY11(BGRAToYRow_Any_NEON, BGRAToYRow_NEON, 0, 4, 1, 15)
#endif
+#ifdef HAS_BGRATOYROW_NEON_DOTPROD
+ANY11(BGRAToYRow_Any_NEON_DotProd, BGRAToYRow_NEON_DotProd, 0, 4, 1, 15)
+#endif
#ifdef HAS_BGRATOYROW_MSA
ANY11(BGRAToYRow_Any_MSA, BGRAToYRow_MSA, 0, 4, 1, 15)
#endif
@@ -1126,6 +1207,9 @@
#ifdef HAS_ABGRTOYROW_NEON
ANY11(ABGRToYRow_Any_NEON, ABGRToYRow_NEON, 0, 4, 1, 15)
#endif
+#ifdef HAS_ABGRTOYROW_NEON_DOTPROD
+ANY11(ABGRToYRow_Any_NEON_DotProd, ABGRToYRow_NEON_DotProd, 0, 4, 1, 15)
+#endif
#ifdef HAS_ABGRTOYROW_MSA
ANY11(ABGRToYRow_Any_MSA, ABGRToYRow_MSA, 0, 4, 1, 7)
#endif
@@ -1138,6 +1222,9 @@
#ifdef HAS_RGBATOYROW_NEON
ANY11(RGBAToYRow_Any_NEON, RGBAToYRow_NEON, 0, 4, 1, 15)
#endif
+#ifdef HAS_RGBATOYROW_NEON_DOTPROD
+ANY11(RGBAToYRow_Any_NEON_DotProd, RGBAToYRow_NEON_DotProd, 0, 4, 1, 15)
+#endif
#ifdef HAS_RGBATOYROW_MSA
ANY11(RGBAToYRow_Any_MSA, RGBAToYRow_MSA, 0, 4, 1, 15)
#endif
@@ -1202,7 +1289,7 @@
ANY11(RAWToYJRow_Any_LASX, RAWToYJRow_LASX, 0, 3, 1, 31)
#endif
#ifdef HAS_RGB565TOYROW_NEON
-ANY11(RGB565ToYRow_Any_NEON, RGB565ToYRow_NEON, 0, 2, 1, 7)
+ANY11(RGB565ToYRow_Any_NEON, RGB565ToYRow_NEON, 0, 2, 1, 15)
#endif
#ifdef HAS_RGB565TOYROW_MSA
ANY11(RGB565ToYRow_Any_MSA, RGB565ToYRow_MSA, 0, 2, 1, 15)
@@ -1214,8 +1301,12 @@
ANY11(RGB565ToYRow_Any_LASX, RGB565ToYRow_LASX, 0, 2, 1, 31)
#endif
#ifdef HAS_ARGB1555TOYROW_NEON
+#ifdef __aarch64__
+ANY11(ARGB1555ToYRow_Any_NEON, ARGB1555ToYRow_NEON, 0, 2, 1, 15)
+#else
ANY11(ARGB1555ToYRow_Any_NEON, ARGB1555ToYRow_NEON, 0, 2, 1, 7)
#endif
+#endif
#ifdef HAS_ARGB1555TOYROW_MSA
ANY11(ARGB1555ToYRow_Any_MSA, ARGB1555ToYRow_MSA, 0, 2, 1, 15)
#endif
@@ -1226,8 +1317,12 @@
ANY11(ARGB1555ToYRow_Any_LASX, ARGB1555ToYRow_LASX, 0, 2, 1, 31)
#endif
#ifdef HAS_ARGB4444TOYROW_NEON
+#ifdef __aarch64__
+ANY11(ARGB4444ToYRow_Any_NEON, ARGB4444ToYRow_NEON, 0, 2, 1, 15)
+#else
ANY11(ARGB4444ToYRow_Any_NEON, ARGB4444ToYRow_NEON, 0, 2, 1, 7)
#endif
+#endif
#ifdef HAS_YUY2TOYROW_NEON
ANY11(YUY2ToYRow_Any_NEON, YUY2ToYRow_NEON, 1, 4, 1, 15)
#endif
@@ -1292,7 +1387,7 @@
ANY11(RAWToARGBRow_Any_LASX, RAWToARGBRow_LASX, 0, 3, 4, 31)
#endif
#ifdef HAS_RGB565TOARGBROW_NEON
-ANY11(RGB565ToARGBRow_Any_NEON, RGB565ToARGBRow_NEON, 0, 2, 4, 7)
+ANY11(RGB565ToARGBRow_Any_NEON, RGB565ToARGBRow_NEON, 0, 2, 4, 15)
#endif
#ifdef HAS_RGB565TOARGBROW_MSA
ANY11(RGB565ToARGBRow_Any_MSA, RGB565ToARGBRow_MSA, 0, 2, 4, 15)
@@ -1304,7 +1399,7 @@
ANY11(RGB565ToARGBRow_Any_LASX, RGB565ToARGBRow_LASX, 0, 2, 4, 31)
#endif
#ifdef HAS_ARGB1555TOARGBROW_NEON
-ANY11(ARGB1555ToARGBRow_Any_NEON, ARGB1555ToARGBRow_NEON, 0, 2, 4, 7)
+ANY11(ARGB1555ToARGBRow_Any_NEON, ARGB1555ToARGBRow_NEON, 0, 2, 4, 15)
#endif
#ifdef HAS_ARGB1555TOARGBROW_MSA
ANY11(ARGB1555ToARGBRow_Any_MSA, ARGB1555ToARGBRow_MSA, 0, 2, 4, 15)
@@ -1595,8 +1690,8 @@
// Any 1 to 1 with parameter and shorts. BPP measures in shorts.
#define ANY11C(NAMEANY, ANY_SIMD, SBPP, BPP, STYPE, DTYPE, MASK) \
void NAMEANY(const STYPE* src_ptr, DTYPE* dst_ptr, int scale, int width) { \
- SIMD_ALIGNED(STYPE vin[32]); \
- SIMD_ALIGNED(DTYPE vout[32]); \
+ SIMD_ALIGNED(STYPE vin[64]); \
+ SIMD_ALIGNED(DTYPE vout[64]); \
memset(vin, 0, sizeof(vin)); /* for msan */ \
int r = width & MASK; \
int n = width & ~MASK; \
@@ -1626,6 +1721,15 @@
uint8_t,
31)
#endif
+#ifdef HAS_CONVERT16TO8ROW_AVX512BW
+ANY11C(Convert16To8Row_Any_AVX512BW,
+ Convert16To8Row_AVX512BW,
+ 2,
+ 1,
+ uint16_t,
+ uint8_t,
+ 63)
+#endif
#ifdef HAS_CONVERT16TO8ROW_NEON
ANY11C(Convert16To8Row_Any_NEON,
Convert16To8Row_NEON,
@@ -1679,6 +1783,43 @@
#endif
#undef ANY11C
+// Any 1 to 1 with parameter and shorts. BPP measures in shorts.
+#define ANY11SB(NAMEANY, ANY_SIMD, SBPP, BPP, STYPE, DTYPE, MASK) \
+ void NAMEANY(const STYPE* src_ptr, DTYPE* dst_ptr, int scale, int bias, \
+ int width) { \
+ SIMD_ALIGNED(STYPE vin[64]); \
+ SIMD_ALIGNED(DTYPE vout[64]); \
+ memset(vin, 0, sizeof(vin)); /* for msan */ \
+ int r = width & MASK; \
+ int n = width & ~MASK; \
+ if (n > 0) { \
+ ANY_SIMD(src_ptr, dst_ptr, scale, bias, n); \
+ } \
+ memcpy(vin, src_ptr + n, r * SBPP); \
+ ANY_SIMD(vin, vout, scale, bias, MASK + 1); \
+ memcpy(dst_ptr + n, vout, r * BPP); \
+ }
+
+#ifdef HAS_CONVERT8TO8ROW_NEON
+ANY11SB(Convert8To8Row_Any_NEON,
+ Convert8To8Row_NEON,
+ 1,
+ 1,
+ uint8_t,
+ uint8_t,
+ 31)
+#endif
+#ifdef HAS_CONVERT8TO8ROW_AVX2
+ANY11SB(Convert8To8Row_Any_AVX2,
+ Convert8To8Row_AVX2,
+ 1,
+ 1,
+ uint8_t,
+ uint8_t,
+ 31)
+#endif
+#undef ANY11B
+
// Any 1 to 1 with parameter and shorts to byte. BPP measures in shorts.
#define ANY11P16(NAMEANY, ANY_SIMD, ST, T, SBPP, BPP, MASK) \
void NAMEANY(const ST* src_ptr, T* dst_ptr, float param, int width) { \
@@ -1712,14 +1853,7 @@
15)
#endif
#ifdef HAS_HALFFLOATROW_NEON
-ANY11P16(HalfFloatRow_Any_NEON, HalfFloatRow_NEON, uint16_t, uint16_t, 2, 2, 7)
-ANY11P16(HalfFloat1Row_Any_NEON,
- HalfFloat1Row_NEON,
- uint16_t,
- uint16_t,
- 2,
- 2,
- 7)
+ANY11P16(HalfFloatRow_Any_NEON, HalfFloatRow_NEON, uint16_t, uint16_t, 2, 2, 15)
#endif
#ifdef HAS_HALFFLOATROW_MSA
ANY11P16(HalfFloatRow_Any_MSA, HalfFloatRow_MSA, uint16_t, uint16_t, 2, 2, 31)
@@ -2014,6 +2148,15 @@
#ifdef HAS_ARGBTOUV444ROW_SSSE3
ANY12(ARGBToUV444Row_Any_SSSE3, ARGBToUV444Row_SSSE3, 0, 4, 0, 15)
#endif
+#ifdef HAS_ARGBTOUVJ444ROW_SSSE3
+ANY12(ARGBToUVJ444Row_Any_SSSE3, ARGBToUVJ444Row_SSSE3, 0, 4, 0, 15)
+#endif
+#ifdef HAS_ARGBTOUV444ROW_AVX2
+ANY12(ARGBToUV444Row_Any_AVX2, ARGBToUV444Row_AVX2, 0, 4, 0, 31)
+#endif
+#ifdef HAS_ARGBTOUVJ444ROW_AVX2
+ANY12(ARGBToUVJ444Row_Any_AVX2, ARGBToUVJ444Row_AVX2, 0, 4, 0, 31)
+#endif
#ifdef HAS_YUY2TOUV422ROW_AVX2
ANY12(YUY2ToUV422Row_Any_AVX2, YUY2ToUV422Row_AVX2, 1, 4, 1, 31)
ANY12(UYVYToUV422Row_Any_AVX2, UYVYToUV422Row_AVX2, 1, 4, 1, 31)
@@ -2024,9 +2167,14 @@
#endif
#ifdef HAS_YUY2TOUV422ROW_NEON
ANY12(ARGBToUV444Row_Any_NEON, ARGBToUV444Row_NEON, 0, 4, 0, 7)
+ANY12(ARGBToUVJ444Row_Any_NEON, ARGBToUVJ444Row_NEON, 0, 4, 0, 7)
ANY12(YUY2ToUV422Row_Any_NEON, YUY2ToUV422Row_NEON, 1, 4, 1, 15)
ANY12(UYVYToUV422Row_Any_NEON, UYVYToUV422Row_NEON, 1, 4, 1, 15)
#endif
+#ifdef HAS_ARGBTOUV444ROW_NEON_I8MM
+ANY12(ARGBToUV444Row_Any_NEON_I8MM, ARGBToUV444Row_NEON_I8MM, 0, 4, 0, 7)
+ANY12(ARGBToUVJ444Row_Any_NEON_I8MM, ARGBToUVJ444Row_NEON_I8MM, 0, 4, 0, 7)
+#endif
#ifdef HAS_YUY2TOUV422ROW_MSA
ANY12(ARGBToUV444Row_Any_MSA, ARGBToUV444Row_MSA, 0, 4, 0, 15)
ANY12(YUY2ToUV422Row_Any_MSA, YUY2ToUV422Row_MSA, 1, 4, 1, 31)
@@ -2093,6 +2241,12 @@
#ifdef HAS_SPLITRGBROW_SSSE3
ANY13(SplitRGBRow_Any_SSSE3, SplitRGBRow_SSSE3, 3, 15)
#endif
+#ifdef HAS_SPLITRGBROW_SSE41
+ANY13(SplitRGBRow_Any_SSE41, SplitRGBRow_SSE41, 3, 15)
+#endif
+#ifdef HAS_SPLITRGBROW_AVX2
+ANY13(SplitRGBRow_Any_AVX2, SplitRGBRow_AVX2, 3, 31)
+#endif
#ifdef HAS_SPLITRGBROW_NEON
ANY13(SplitRGBRow_Any_NEON, SplitRGBRow_NEON, 3, 15)
#endif
@@ -2204,6 +2358,9 @@
#ifdef HAS_ARGBTOUVROW_NEON
ANY12S(ARGBToUVRow_Any_NEON, ARGBToUVRow_NEON, 0, 4, 15)
#endif
+#ifdef HAS_ARGBTOUVROW_SVE2
+ANY12S(ARGBToUVRow_Any_SVE2, ARGBToUVRow_SVE2, 0, 4, 1)
+#endif
#ifdef HAS_ARGBTOUVROW_MSA
ANY12S(ARGBToUVRow_Any_MSA, ARGBToUVRow_MSA, 0, 4, 31)
#endif
@@ -2216,9 +2373,15 @@
#ifdef HAS_ARGBTOUVJROW_NEON
ANY12S(ARGBToUVJRow_Any_NEON, ARGBToUVJRow_NEON, 0, 4, 15)
#endif
+#ifdef HAS_ARGBTOUVJROW_SVE2
+ANY12S(ARGBToUVJRow_Any_SVE2, ARGBToUVJRow_SVE2, 0, 4, 1)
+#endif
#ifdef HAS_ABGRTOUVJROW_NEON
ANY12S(ABGRToUVJRow_Any_NEON, ABGRToUVJRow_NEON, 0, 4, 15)
#endif
+#ifdef HAS_ABGRTOUVJROW_SVE2
+ANY12S(ABGRToUVJRow_Any_SVE2, ABGRToUVJRow_SVE2, 0, 4, 1)
+#endif
#ifdef HAS_ARGBTOUVJROW_MSA
ANY12S(ARGBToUVJRow_Any_MSA, ARGBToUVJRow_MSA, 0, 4, 31)
#endif
@@ -2231,6 +2394,9 @@
#ifdef HAS_BGRATOUVROW_NEON
ANY12S(BGRAToUVRow_Any_NEON, BGRAToUVRow_NEON, 0, 4, 15)
#endif
+#ifdef HAS_BGRATOUVROW_SVE2
+ANY12S(BGRAToUVRow_Any_SVE2, BGRAToUVRow_SVE2, 0, 4, 1)
+#endif
#ifdef HAS_BGRATOUVROW_MSA
ANY12S(BGRAToUVRow_Any_MSA, BGRAToUVRow_MSA, 0, 4, 15)
#endif
@@ -2240,6 +2406,9 @@
#ifdef HAS_ABGRTOUVROW_NEON
ANY12S(ABGRToUVRow_Any_NEON, ABGRToUVRow_NEON, 0, 4, 15)
#endif
+#ifdef HAS_ABGRTOUVROW_SVE2
+ANY12S(ABGRToUVRow_Any_SVE2, ABGRToUVRow_SVE2, 0, 4, 1)
+#endif
#ifdef HAS_ABGRTOUVROW_MSA
ANY12S(ABGRToUVRow_Any_MSA, ABGRToUVRow_MSA, 0, 4, 15)
#endif
@@ -2249,6 +2418,9 @@
#ifdef HAS_RGBATOUVROW_NEON
ANY12S(RGBAToUVRow_Any_NEON, RGBAToUVRow_NEON, 0, 4, 15)
#endif
+#ifdef HAS_RGBATOUVROW_SVE2
+ANY12S(RGBAToUVRow_Any_SVE2, RGBAToUVRow_SVE2, 0, 4, 1)
+#endif
#ifdef HAS_RGBATOUVROW_MSA
ANY12S(RGBAToUVRow_Any_MSA, RGBAToUVRow_MSA, 0, 4, 15)
#endif
@@ -2368,6 +2540,12 @@
ANY11S(AYUVToUVRow_Any_NEON, AYUVToUVRow_NEON, 0, 4, 15)
ANY11S(AYUVToVURow_Any_NEON, AYUVToVURow_NEON, 0, 4, 15)
#endif
+#ifdef HAS_AYUVTOUVROW_SVE2
+ANY11S(AYUVToUVRow_Any_SVE2, AYUVToUVRow_SVE2, 0, 4, 1)
+#endif
+#ifdef HAS_AYUVTOVUROW_SVE2
+ANY11S(AYUVToVURow_Any_SVE2, AYUVToVURow_SVE2, 0, 4, 1)
+#endif
#undef ANY11S
#define ANYDETILE(NAMEANY, ANY_SIMD, T, BPP, MASK) \
diff --git a/source/row_common.cc b/source/row_common.cc
index 3afc4b4..7101ec3 100644
--- a/source/row_common.cc
+++ b/source/row_common.cc
@@ -36,18 +36,9 @@
// LIBYUV_UNLIMITED_BT709
// LIBYUV_UNLIMITED_BT2020
-// The following macro from row_win makes the C code match the row_win code,
-// which is 7 bit fixed point for ARGBToI420:
-#if !defined(LIBYUV_BIT_EXACT) && !defined(LIBYUV_DISABLE_X86) && \
- defined(_MSC_VER) && !defined(__clang__) && \
- (defined(_M_IX86) || defined(_M_X64))
-#define LIBYUV_RGB7 1
-#endif
-
#if !defined(LIBYUV_BIT_EXACT) && (defined(__x86_64__) || defined(_M_X64) || \
defined(__i386__) || defined(_M_IX86))
#define LIBYUV_ARGBTOUV_PAVGB 1
-#define LIBYUV_RGBTOU_TRUNCATE 1
#endif
#if defined(LIBYUV_BIT_EXACT)
#define LIBYUV_UNATTENUATE_DUP 1
@@ -56,7 +47,7 @@
// llvm x86 is poor at ternary operator, so use branchless min/max.
#define USE_BRANCHLESS 1
-#if USE_BRANCHLESS
+#if defined(USE_BRANCHLESS)
static __inline int32_t clamp0(int32_t v) {
return -(v >= 0) & v;
}
@@ -624,59 +615,33 @@
dst_ar64_16 += 4;
}
}
-
-#ifdef LIBYUV_RGB7
-// Old 7 bit math for compatibility on unsupported platforms.
-static __inline uint8_t RGBToY(uint8_t r, uint8_t g, uint8_t b) {
- return STATIC_CAST(uint8_t, ((33 * r + 65 * g + 13 * b) >> 7) + 16);
-}
-#else
-// 8 bit
-// Intel SSE/AVX uses the following equivalent formula
-// 0x7e80 = (66 + 129 + 25) * -128 + 0x1000 (for +16) and 0x0080 for round.
-// return (66 * ((int)r - 128) + 129 * ((int)g - 128) + 25 * ((int)b - 128) +
-// 0x7e80) >> 8;
+// BT601 8 bit Y:
+// b 0.114 * 219 = 24.966 = 25
+// g 0.587 * 219 = 128.553 = 129
+// r 0.299 * 219 = 65.481 = 66
+// BT601 8 bit U:
+// b 0.875 * 128 = 112.0 = 112
+// g -0.5781 * 128 = −73.9968 = -74
+// r -0.2969 * 128 = −38.0032 = -38
+// BT601 8 bit V:
+// b -0.1406 * 128 = −17.9968 = -18
+// g -0.7344 * 128 = −94.0032 = -94
+// r 0.875 * 128 = 112.0 = 112
static __inline uint8_t RGBToY(uint8_t r, uint8_t g, uint8_t b) {
return STATIC_CAST(uint8_t, (66 * r + 129 * g + 25 * b + 0x1080) >> 8);
}
-#endif
-
-#define AVGB(a, b) (((a) + (b) + 1) >> 1)
-
-// LIBYUV_RGBTOU_TRUNCATE mimics x86 code that does not round.
-#ifdef LIBYUV_RGBTOU_TRUNCATE
static __inline uint8_t RGBToU(uint8_t r, uint8_t g, uint8_t b) {
return STATIC_CAST(uint8_t, (112 * b - 74 * g - 38 * r + 0x8000) >> 8);
}
static __inline uint8_t RGBToV(uint8_t r, uint8_t g, uint8_t b) {
return STATIC_CAST(uint8_t, (112 * r - 94 * g - 18 * b + 0x8000) >> 8);
}
-#else
-// TODO(fbarchard): Add rounding to x86 SIMD and use this
-static __inline uint8_t RGBToU(uint8_t r, uint8_t g, uint8_t b) {
- return STATIC_CAST(uint8_t, (112 * b - 74 * g - 38 * r + 0x8080) >> 8);
-}
-static __inline uint8_t RGBToV(uint8_t r, uint8_t g, uint8_t b) {
- return STATIC_CAST(uint8_t, (112 * r - 94 * g - 18 * b + 0x8080) >> 8);
-}
-#endif
-
-// LIBYUV_ARGBTOUV_PAVGB mimics x86 code that subsamples with 2 pavgb.
-#if !defined(LIBYUV_ARGBTOUV_PAVGB)
-static __inline int RGB2xToU(uint16_t r, uint16_t g, uint16_t b) {
- return STATIC_CAST(
- uint8_t, ((112 / 2) * b - (74 / 2) * g - (38 / 2) * r + 0x8080) >> 8);
-}
-static __inline int RGB2xToV(uint16_t r, uint16_t g, uint16_t b) {
- return STATIC_CAST(
- uint8_t, ((112 / 2) * r - (94 / 2) * g - (18 / 2) * b + 0x8080) >> 8);
-}
-#endif
+#define AVGB(a, b) (((a) + (b) + 1) >> 1)
// ARGBToY_C and ARGBToUV_C
-// Intel version mimic SSE/AVX which does 2 pavgb
-#if LIBYUV_ARGBTOUV_PAVGB
+// Intel version of UV mimic SSE/AVX which does 2 pavgb
+#if defined(LIBYUV_ARGBTOUV_PAVGB)
#define MAKEROWY(NAME, R, G, B, BPP) \
void NAME##ToYRow_C(const uint8_t* src_rgb, uint8_t* dst_y, int width) { \
int x; \
@@ -713,7 +678,7 @@
} \
}
#else
-// ARM version does sum / 2 then multiply by 2x smaller coefficients
+// ARM version does average of 4 pixels with rounding
#define MAKEROWY(NAME, R, G, B, BPP) \
void NAME##ToYRow_C(const uint8_t* src_rgb, uint8_t* dst_y, int width) { \
int x; \
@@ -728,28 +693,28 @@
const uint8_t* src_rgb1 = src_rgb + src_stride_rgb; \
int x; \
for (x = 0; x < width - 1; x += 2) { \
- uint16_t ab = (src_rgb[B] + src_rgb[B + BPP] + src_rgb1[B] + \
- src_rgb1[B + BPP] + 1) >> \
- 1; \
- uint16_t ag = (src_rgb[G] + src_rgb[G + BPP] + src_rgb1[G] + \
- src_rgb1[G + BPP] + 1) >> \
- 1; \
- uint16_t ar = (src_rgb[R] + src_rgb[R + BPP] + src_rgb1[R] + \
- src_rgb1[R + BPP] + 1) >> \
- 1; \
- dst_u[0] = RGB2xToU(ar, ag, ab); \
- dst_v[0] = RGB2xToV(ar, ag, ab); \
+ uint8_t ab = (src_rgb[B] + src_rgb[B + BPP] + src_rgb1[B] + \
+ src_rgb1[B + BPP] + 2) >> \
+ 2; \
+ uint8_t ag = (src_rgb[G] + src_rgb[G + BPP] + src_rgb1[G] + \
+ src_rgb1[G + BPP] + 2) >> \
+ 2; \
+ uint8_t ar = (src_rgb[R] + src_rgb[R + BPP] + src_rgb1[R] + \
+ src_rgb1[R + BPP] + 2) >> \
+ 2; \
+ dst_u[0] = RGBToU(ar, ag, ab); \
+ dst_v[0] = RGBToV(ar, ag, ab); \
src_rgb += BPP * 2; \
src_rgb1 += BPP * 2; \
dst_u += 1; \
dst_v += 1; \
} \
if (width & 1) { \
- uint16_t ab = src_rgb[B] + src_rgb1[B]; \
- uint16_t ag = src_rgb[G] + src_rgb1[G]; \
- uint16_t ar = src_rgb[R] + src_rgb1[R]; \
- dst_u[0] = RGB2xToU(ar, ag, ab); \
- dst_v[0] = RGB2xToV(ar, ag, ab); \
+ uint8_t ab = (src_rgb[B] + src_rgb1[B] + 1) >> 1; \
+ uint8_t ag = (src_rgb[G] + src_rgb1[G] + 1) >> 1; \
+ uint8_t ar = (src_rgb[R] + src_rgb1[R] + 1) >> 1; \
+ dst_u[0] = RGBToU(ar, ag, ab); \
+ dst_v[0] = RGBToV(ar, ag, ab); \
} \
}
#endif
@@ -762,62 +727,37 @@
MAKEROWY(RAW, 0, 1, 2, 3)
#undef MAKEROWY
-// JPeg uses a variation on BT.601-1 full range
+// JPeg uses BT.601-1 full range
// y = 0.29900 * r + 0.58700 * g + 0.11400 * b
// u = -0.16874 * r - 0.33126 * g + 0.50000 * b + center
// v = 0.50000 * r - 0.41869 * g - 0.08131 * b + center
-// BT.601 Mpeg range uses:
-// b 0.1016 * 255 = 25.908 = 25
-// g 0.5078 * 255 = 129.489 = 129
-// r 0.2578 * 255 = 65.739 = 66
-// JPeg 7 bit Y (deprecated)
-// b 0.11400 * 128 = 14.592 = 15
-// g 0.58700 * 128 = 75.136 = 75
-// r 0.29900 * 128 = 38.272 = 38
// JPeg 8 bit Y:
// b 0.11400 * 256 = 29.184 = 29
// g 0.58700 * 256 = 150.272 = 150
// r 0.29900 * 256 = 76.544 = 77
// JPeg 8 bit U:
-// b 0.50000 * 255 = 127.5 = 127
-// g -0.33126 * 255 = -84.4713 = -84
-// r -0.16874 * 255 = -43.0287 = -43
+// b 0.50000 * 256 = 128.0 = 128
+// g -0.33126 * 256 = −84.80256 = -85
+// r -0.16874 * 256 = −43.19744 = -43
// JPeg 8 bit V:
-// b -0.08131 * 255 = -20.73405 = -20
-// g -0.41869 * 255 = -106.76595 = -107
-// r 0.50000 * 255 = 127.5 = 127
+// b -0.08131 * 256 = −20.81536 = -21
+// g -0.41869 * 256 = −107.18464 = -107
+// r 0.50000 * 256 = 128.0 = 128
-#ifdef LIBYUV_RGB7
-// Old 7 bit math for compatibility on unsupported platforms.
-static __inline uint8_t RGBToYJ(uint8_t r, uint8_t g, uint8_t b) {
- return (38 * r + 75 * g + 15 * b + 64) >> 7;
-}
-#else
// 8 bit
static __inline uint8_t RGBToYJ(uint8_t r, uint8_t g, uint8_t b) {
return (77 * r + 150 * g + 29 * b + 128) >> 8;
}
-#endif
-
-#if defined(LIBYUV_ARGBTOUV_PAVGB)
static __inline uint8_t RGBToUJ(uint8_t r, uint8_t g, uint8_t b) {
- return (127 * b - 84 * g - 43 * r + 0x8080) >> 8;
+ return (128 * b - 85 * g - 43 * r + 0x8000) >> 8;
}
static __inline uint8_t RGBToVJ(uint8_t r, uint8_t g, uint8_t b) {
- return (127 * r - 107 * g - 20 * b + 0x8080) >> 8;
+ return (128 * r - 107 * g - 21 * b + 0x8000) >> 8;
}
-#else
-static __inline uint8_t RGB2xToUJ(uint16_t r, uint16_t g, uint16_t b) {
- return ((127 / 2) * b - (84 / 2) * g - (43 / 2) * r + 0x8080) >> 8;
-}
-static __inline uint8_t RGB2xToVJ(uint16_t r, uint16_t g, uint16_t b) {
- return ((127 / 2) * r - (107 / 2) * g - (20 / 2) * b + 0x8080) >> 8;
-}
-#endif
// ARGBToYJ_C and ARGBToUVJ_C
// Intel version mimic SSE/AVX which does 2 pavgb
-#if LIBYUV_ARGBTOUV_PAVGB
+#if defined(LIBYUV_ARGBTOUV_PAVGB)
#define MAKEROWYJ(NAME, R, G, B, BPP) \
void NAME##ToYJRow_C(const uint8_t* src_rgb, uint8_t* dst_y, int width) { \
int x; \
@@ -854,7 +794,7 @@
} \
}
#else
-// ARM version does sum / 2 then multiply by 2x smaller coefficients
+// ARM version does average of 4 pixels with rounding
#define MAKEROWYJ(NAME, R, G, B, BPP) \
void NAME##ToYJRow_C(const uint8_t* src_rgb, uint8_t* dst_y, int width) { \
int x; \
@@ -869,28 +809,28 @@
const uint8_t* src_rgb1 = src_rgb + src_stride_rgb; \
int x; \
for (x = 0; x < width - 1; x += 2) { \
- uint16_t ab = (src_rgb[B] + src_rgb[B + BPP] + src_rgb1[B] + \
- src_rgb1[B + BPP] + 1) >> \
- 1; \
- uint16_t ag = (src_rgb[G] + src_rgb[G + BPP] + src_rgb1[G] + \
- src_rgb1[G + BPP] + 1) >> \
- 1; \
- uint16_t ar = (src_rgb[R] + src_rgb[R + BPP] + src_rgb1[R] + \
- src_rgb1[R + BPP] + 1) >> \
- 1; \
- dst_u[0] = RGB2xToUJ(ar, ag, ab); \
- dst_v[0] = RGB2xToVJ(ar, ag, ab); \
+ uint8_t ab = (src_rgb[B] + src_rgb[B + BPP] + src_rgb1[B] + \
+ src_rgb1[B + BPP] + 2) >> \
+ 2; \
+ uint8_t ag = (src_rgb[G] + src_rgb[G + BPP] + src_rgb1[G] + \
+ src_rgb1[G + BPP] + 2) >> \
+ 2; \
+ uint8_t ar = (src_rgb[R] + src_rgb[R + BPP] + src_rgb1[R] + \
+ src_rgb1[R + BPP] + 2) >> \
+ 2; \
+ dst_u[0] = RGBToUJ(ar, ag, ab); \
+ dst_v[0] = RGBToVJ(ar, ag, ab); \
src_rgb += BPP * 2; \
src_rgb1 += BPP * 2; \
dst_u += 1; \
dst_v += 1; \
} \
if (width & 1) { \
- uint16_t ab = (src_rgb[B] + src_rgb1[B]); \
- uint16_t ag = (src_rgb[G] + src_rgb1[G]); \
- uint16_t ar = (src_rgb[R] + src_rgb1[R]); \
- dst_u[0] = RGB2xToUJ(ar, ag, ab); \
- dst_v[0] = RGB2xToVJ(ar, ag, ab); \
+ uint16_t ab = (src_rgb[B] + src_rgb1[B] + 1) >> 1; \
+ uint16_t ag = (src_rgb[G] + src_rgb1[G] + 1) >> 1; \
+ uint16_t ar = (src_rgb[R] + src_rgb1[R] + 1) >> 1; \
+ dst_u[0] = RGBToUJ(ar, ag, ab); \
+ dst_v[0] = RGBToVJ(ar, ag, ab); \
} \
}
@@ -988,18 +928,18 @@
g3 = STATIC_CAST(uint8_t, (g3 << 2) | (g3 >> 4));
r3 = STATIC_CAST(uint8_t, (r3 << 3) | (r3 >> 2));
-#if LIBYUV_ARGBTOUV_PAVGB
+#if defined(LIBYUV_ARGBTOUV_PAVGB)
uint8_t ab = AVGB(AVGB(b0, b2), AVGB(b1, b3));
uint8_t ag = AVGB(AVGB(g0, g2), AVGB(g1, g3));
uint8_t ar = AVGB(AVGB(r0, r2), AVGB(r1, r3));
dst_u[0] = RGBToU(ar, ag, ab);
dst_v[0] = RGBToV(ar, ag, ab);
#else
- uint16_t b = (b0 + b1 + b2 + b3 + 1) >> 1;
- uint16_t g = (g0 + g1 + g2 + g3 + 1) >> 1;
- uint16_t r = (r0 + r1 + r2 + r3 + 1) >> 1;
- dst_u[0] = RGB2xToU(r, g, b);
- dst_v[0] = RGB2xToV(r, g, b);
+ uint8_t b = (b0 + b1 + b2 + b3 + 2) >> 2;
+ uint8_t g = (g0 + g1 + g2 + g3 + 2) >> 2;
+ uint8_t r = (r0 + r1 + r2 + r3 + 2) >> 2;
+ dst_u[0] = RGBToU(r, g, b);
+ dst_v[0] = RGBToV(r, g, b);
#endif
src_rgb565 += 4;
@@ -1023,19 +963,11 @@
g2 = STATIC_CAST(uint8_t, (g2 << 2) | (g2 >> 4));
r2 = STATIC_CAST(uint8_t, (r2 << 3) | (r2 >> 2));
-#if LIBYUV_ARGBTOUV_PAVGB
uint8_t ab = AVGB(b0, b2);
uint8_t ag = AVGB(g0, g2);
uint8_t ar = AVGB(r0, r2);
dst_u[0] = RGBToU(ar, ag, ab);
dst_v[0] = RGBToV(ar, ag, ab);
-#else
- uint16_t b = b0 + b2;
- uint16_t g = g0 + g2;
- uint16_t r = r0 + r2;
- dst_u[0] = RGB2xToU(r, g, b);
- dst_v[0] = RGB2xToV(r, g, b);
-#endif
}
}
@@ -1077,18 +1009,18 @@
g3 = STATIC_CAST(uint8_t, (g3 << 3) | (g3 >> 2));
r3 = STATIC_CAST(uint8_t, (r3 << 3) | (r3 >> 2));
-#if LIBYUV_ARGBTOUV_PAVGB
+#if defined(LIBYUV_ARGBTOUV_PAVGB)
uint8_t ab = AVGB(AVGB(b0, b2), AVGB(b1, b3));
uint8_t ag = AVGB(AVGB(g0, g2), AVGB(g1, g3));
uint8_t ar = AVGB(AVGB(r0, r2), AVGB(r1, r3));
dst_u[0] = RGBToU(ar, ag, ab);
dst_v[0] = RGBToV(ar, ag, ab);
#else
- uint16_t b = (b0 + b1 + b2 + b3 + 1) >> 1;
- uint16_t g = (g0 + g1 + g2 + g3 + 1) >> 1;
- uint16_t r = (r0 + r1 + r2 + r3 + 1) >> 1;
- dst_u[0] = RGB2xToU(r, g, b);
- dst_v[0] = RGB2xToV(r, g, b);
+ uint8_t b = (b0 + b1 + b2 + b3 + 2) >> 2;
+ uint8_t g = (g0 + g1 + g2 + g3 + 2) >> 2;
+ uint8_t r = (r0 + r1 + r2 + r3 + 2) >> 2;
+ dst_u[0] = RGBToU(r, g, b);
+ dst_v[0] = RGBToV(r, g, b);
#endif
src_argb1555 += 4;
@@ -1113,19 +1045,11 @@
g2 = STATIC_CAST(uint8_t, (g2 << 3) | (g2 >> 2));
r2 = STATIC_CAST(uint8_t, (r2 << 3) | (r2 >> 2));
-#if LIBYUV_ARGBTOUV_PAVGB
uint8_t ab = AVGB(b0, b2);
uint8_t ag = AVGB(g0, g2);
uint8_t ar = AVGB(r0, r2);
dst_u[0] = RGBToU(ar, ag, ab);
dst_v[0] = RGBToV(ar, ag, ab);
-#else
- uint16_t b = b0 + b2;
- uint16_t g = g0 + g2;
- uint16_t r = r0 + r2;
- dst_u[0] = RGB2xToU(r, g, b);
- dst_v[0] = RGB2xToV(r, g, b);
-#endif
}
}
@@ -1163,18 +1087,18 @@
g3 = STATIC_CAST(uint8_t, (g3 << 4) | g3);
r3 = STATIC_CAST(uint8_t, (r3 << 4) | r3);
-#if LIBYUV_ARGBTOUV_PAVGB
+#if defined(LIBYUV_ARGBTOUV_PAVGB)
uint8_t ab = AVGB(AVGB(b0, b2), AVGB(b1, b3));
uint8_t ag = AVGB(AVGB(g0, g2), AVGB(g1, g3));
uint8_t ar = AVGB(AVGB(r0, r2), AVGB(r1, r3));
dst_u[0] = RGBToU(ar, ag, ab);
dst_v[0] = RGBToV(ar, ag, ab);
#else
- uint16_t b = (b0 + b1 + b2 + b3 + 1) >> 1;
- uint16_t g = (g0 + g1 + g2 + g3 + 1) >> 1;
- uint16_t r = (r0 + r1 + r2 + r3 + 1) >> 1;
- dst_u[0] = RGB2xToU(r, g, b);
- dst_v[0] = RGB2xToV(r, g, b);
+ uint8_t b = (b0 + b1 + b2 + b3 + 2) >> 2;
+ uint8_t g = (g0 + g1 + g2 + g3 + 2) >> 2;
+ uint8_t r = (r0 + r1 + r2 + r3 + 2) >> 2;
+ dst_u[0] = RGBToU(r, g, b);
+ dst_v[0] = RGBToV(r, g, b);
#endif
src_argb4444 += 4;
@@ -1197,19 +1121,11 @@
g2 = STATIC_CAST(uint8_t, (g2 << 4) | g2);
r2 = STATIC_CAST(uint8_t, (r2 << 4) | r2);
-#if LIBYUV_ARGBTOUV_PAVGB
uint8_t ab = AVGB(b0, b2);
uint8_t ag = AVGB(g0, g2);
uint8_t ar = AVGB(r0, r2);
dst_u[0] = RGBToU(ar, ag, ab);
dst_v[0] = RGBToV(ar, ag, ab);
-#else
- uint16_t b = b0 + b2;
- uint16_t g = g0 + g2;
- uint16_t r = r0 + r2;
- dst_u[0] = RGB2xToU(r, g, b);
- dst_v[0] = RGB2xToV(r, g, b);
-#endif
}
}
@@ -1230,6 +1146,23 @@
}
}
+void ARGBToUVJ444Row_C(const uint8_t* src_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ int x;
+ for (x = 0; x < width; ++x) {
+ uint8_t ab = src_argb[0];
+ uint8_t ag = src_argb[1];
+ uint8_t ar = src_argb[2];
+ dst_u[0] = RGBToUJ(ar, ag, ab);
+ dst_v[0] = RGBToVJ(ar, ag, ab);
+ src_argb += 4;
+ dst_u += 1;
+ dst_v += 1;
+ }
+}
+
void ARGBGrayRow_C(const uint8_t* src_argb, uint8_t* dst_argb, int width) {
int x;
for (x = 0; x < width; ++x) {
@@ -1375,34 +1308,29 @@
#undef REPEAT8
#undef SHADE
-#define REPEAT8(v) (v) | ((v) << 8)
-#define SHADE(f, v) v* f >> 16
-
void ARGBMultiplyRow_C(const uint8_t* src_argb,
const uint8_t* src_argb1,
uint8_t* dst_argb,
int width) {
int i;
for (i = 0; i < width; ++i) {
- const uint32_t b = REPEAT8(src_argb[0]);
- const uint32_t g = REPEAT8(src_argb[1]);
- const uint32_t r = REPEAT8(src_argb[2]);
- const uint32_t a = REPEAT8(src_argb[3]);
+ const uint32_t b = src_argb[0];
+ const uint32_t g = src_argb[1];
+ const uint32_t r = src_argb[2];
+ const uint32_t a = src_argb[3];
const uint32_t b_scale = src_argb1[0];
const uint32_t g_scale = src_argb1[1];
const uint32_t r_scale = src_argb1[2];
const uint32_t a_scale = src_argb1[3];
- dst_argb[0] = STATIC_CAST(uint8_t, SHADE(b, b_scale));
- dst_argb[1] = STATIC_CAST(uint8_t, SHADE(g, g_scale));
- dst_argb[2] = STATIC_CAST(uint8_t, SHADE(r, r_scale));
- dst_argb[3] = STATIC_CAST(uint8_t, SHADE(a, a_scale));
+ dst_argb[0] = STATIC_CAST(uint8_t, (b * b_scale + 128) >> 8);
+ dst_argb[1] = STATIC_CAST(uint8_t, (g * g_scale + 128) >> 8);
+ dst_argb[2] = STATIC_CAST(uint8_t, (r * r_scale + 128) >> 8);
+ dst_argb[3] = STATIC_CAST(uint8_t, (a * a_scale + 128) >> 8);
src_argb += 4;
src_argb1 += 4;
dst_argb += 4;
}
}
-#undef REPEAT8
-#undef SHADE
#define SHADE(f, v) clamp255(v + f)
@@ -3245,6 +3173,24 @@
}
}
+// Use scale to convert J420 to I420
+// scale parameter is 8.8 fixed point but limited to 0 to 255
+// Function is based on DivideRow, but adds a bias
+// Does not clamp
+void Convert8To8Row_C(const uint8_t* src_y,
+ uint8_t* dst_y,
+ int scale,
+ int bias,
+ int width) {
+ int x;
+ assert(scale >= 0);
+ assert(scale <= 255);
+
+ for (x = 0; x < width; ++x) {
+ dst_y[x] = ((src_y[x] * scale) >> 8) + bias;
+ }
+}
+
void CopyRow_C(const uint8_t* src, uint8_t* dst, int count) {
memcpy(dst, src, count);
}
@@ -3535,7 +3481,7 @@
T(0xfc), T(0xfd), T(0xfe), 0x01000100};
#undef T
-#if LIBYUV_UNATTENUATE_DUP
+#if defined(LIBYUV_UNATTENUATE_DUP)
// This code mimics the Intel SIMD version for better testability.
#define UNATTENUATE(f, ia) clamp255(((f | (f << 8)) * ia) >> 16)
#else
@@ -4008,7 +3954,7 @@
#define MAXTWIDTH 2048
#if !(defined(_MSC_VER) && !defined(__clang__) && defined(_M_IX86)) && \
- defined(HAS_I422TORGB565ROW_SSSE3)
+ defined(HAS_I422TORGB565ROW_SSSE3) && !defined(LIBYUV_ENABLE_ROWWIN)
// row_win.cc has asm version, but GCC uses 2 step wrapper.
void I422ToRGB565Row_SSSE3(const uint8_t* src_y,
const uint8_t* src_u,
@@ -4352,13 +4298,17 @@
#endif // HAS_RGB24TOYJROW_AVX2
#ifdef HAS_RAWTOYJROW_AVX2
-// Convert 16 RAW pixels (64 bytes) to 16 YJ values.
+// Convert 32 RAW pixels (128 bytes) to 32 YJ values.
void RAWToYJRow_AVX2(const uint8_t* src_raw, uint8_t* dst_yj, int width) {
// Row buffer for intermediate ARGB pixels.
SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4]);
while (width > 0) {
int twidth = width > MAXTWIDTH ? MAXTWIDTH : width;
+#ifdef HAS_RAWTOARGBROW_AVX2
+ RAWToARGBRow_AVX2(src_raw, row, twidth);
+#else
RAWToARGBRow_SSSE3(src_raw, row, twidth);
+#endif
ARGBToYJRow_AVX2(row, dst_yj, twidth);
src_raw += twidth * 3;
dst_yj += twidth;
diff --git a/source/row_gcc.cc b/source/row_gcc.cc
index d807498..8f79803 100644
--- a/source/row_gcc.cc
+++ b/source/row_gcc.cc
@@ -15,7 +15,9 @@
#endif
// This module is for GCC x86 and x64.
-#if !defined(LIBYUV_DISABLE_X86) && (defined(__x86_64__) || defined(__i386__))
+#if !defined(LIBYUV_DISABLE_X86) && \
+ (defined(__x86_64__) || defined(__i386__)) && \
+ !defined(LIBYUV_ENABLE_ROWWIN)
#if defined(HAS_ARGBTOYROW_SSSE3) || defined(HAS_ARGBGRAYROW_SSSE3)
@@ -35,60 +37,25 @@
#endif // defined(HAS_ARGBTOYROW_SSSE3) || defined(HAS_ARGBGRAYROW_SSSE3)
#if defined(HAS_ARGBTOYROW_SSSE3) || defined(HAS_I422TOARGBROW_SSSE3)
-
-static const vec8 kARGBToU = {112, -74, -38, 0, 112, -74, -38, 0,
- 112, -74, -38, 0, 112, -74, -38, 0};
-
-static const vec8 kARGBToUJ = {127, -84, -43, 0, 127, -84, -43, 0,
- 127, -84, -43, 0, 127, -84, -43, 0};
-
-static const vec8 kABGRToUJ = {-43, -84, 127, 0, -43, -84, 127, 0,
- -43, -84, 127, 0, -43, -84, 127, 0};
-
-static const vec8 kARGBToV = {-18, -94, 112, 0, -18, -94, 112, 0,
- -18, -94, 112, 0, -18, -94, 112, 0};
-
-static const vec8 kARGBToVJ = {-20, -107, 127, 0, -20, -107, 127, 0,
- -20, -107, 127, 0, -20, -107, 127, 0};
-
-static const vec8 kABGRToVJ = {127, -107, -20, 0, 127, -107, -20, 0,
- 127, -107, -20, 0, 127, -107, -20, 0};
-
// Constants for BGRA
static const uvec8 kBGRAToY = {0u, 66u, 129u, 25u, 0u, 66u, 129u, 25u,
0u, 66u, 129u, 25u, 0u, 66u, 129u, 25u};
-static const vec8 kBGRAToU = {0, -38, -74, 112, 0, -38, -74, 112,
- 0, -38, -74, 112, 0, -38, -74, 112};
-
-static const vec8 kBGRAToV = {0, 112, -94, -18, 0, 112, -94, -18,
- 0, 112, -94, -18, 0, 112, -94, -18};
-
// Constants for ABGR
static const uvec8 kABGRToY = {66u, 129u, 25u, 0u, 66u, 129u, 25u, 0u,
66u, 129u, 25u, 0u, 66u, 129u, 25u, 0u};
-static const vec8 kABGRToU = {-38, -74, 112, 0, -38, -74, 112, 0,
- -38, -74, 112, 0, -38, -74, 112, 0};
-
-static const vec8 kABGRToV = {112, -94, -18, 0, 112, -94, -18, 0,
- 112, -94, -18, 0, 112, -94, -18, 0};
-
// Constants for RGBA.
static const uvec8 kRGBAToY = {0u, 25u, 129u, 66u, 0u, 25u, 129u, 66u,
0u, 25u, 129u, 66u, 0u, 25u, 129u, 66u};
-
-static const vec8 kRGBAToU = {0, 112, -74, -38, 0, 112, -74, -38,
- 0, 112, -74, -38, 0, 112, -74, -38};
-
-static const vec8 kRGBAToV = {0, -18, -94, 112, 0, -18, -94, 112,
- 0, -18, -94, 112, 0, -18, -94, 112};
-
+// 126 (7e) - (-109..110) = 16..235
static const uvec16 kAddY16 = {0x7e80u, 0x7e80u, 0x7e80u, 0x7e80u,
0x7e80u, 0x7e80u, 0x7e80u, 0x7e80u};
+static const uvec16 kAddY0 = {0x8080u, 0x8080u, 0x8080u, 0x8080u,
+ 0x8080u, 0x8080u, 0x8080u, 0x8080u};
-static const uvec8 kAddUV128 = {128u, 128u, 128u, 128u, 128u, 128u, 128u, 128u,
- 128u, 128u, 128u, 128u, 128u, 128u, 128u, 128u};
+static const uvec16 kAddUV128 = {0x8000u, 0x8000u, 0x8000u, 0x8000u,
+ 0x8000u, 0x8000u, 0x8000u, 0x8000u};
static const uvec16 kSub128 = {0x8080u, 0x8080u, 0x8080u, 0x8080u,
0x8080u, 0x8080u, 0x8080u, 0x8080u};
@@ -102,12 +69,16 @@
0u, 1u, 2u, 12u, 3u, 4u, 5u, 13u, 6u, 7u, 8u, 14u, 9u, 10u, 11u, 15u};
// Shuffle table for converting RAW to ARGB.
-static const uvec8 kShuffleMaskRAWToARGB = {2u, 1u, 0u, 12u, 5u, 4u, 3u, 13u,
- 8u, 7u, 6u, 14u, 11u, 10u, 9u, 15u};
+static const uvec8 kShuffleMaskRAWToARGB = {
+ 2u, 1u, 0u, 128u, 5u, 4u, 3u, 128u, 8u, 7u, 6u, 128u, 11u, 10u, 9u, 128u};
+// Shuffle table for converting RAW to ARGB. Last 12
+static const uvec8 kShuffleMaskRAWToARGB_0 = {6u, 5u, 4u, 128u, 9u, 8u,
+ 7u, 128u, 12u, 11u, 10u, 128u,
+ 15u, 14u, 13u, 128u};
// Shuffle table for converting RAW to RGBA.
-static const uvec8 kShuffleMaskRAWToRGBA = {12u, 2u, 1u, 0u, 13u, 5u, 4u, 3u,
- 14u, 8u, 7u, 6u, 15u, 11u, 10u, 9u};
+static const uvec8 kShuffleMaskRAWToRGBA = {
+ 128u, 2u, 1u, 0u, 128u, 5u, 4u, 3u, 128u, 8u, 7u, 6u, 128u, 11u, 10u, 9u};
// Shuffle table for converting RAW to RGB24. First 8.
static const uvec8 kShuffleMaskRAWToRGB24_0 = {
@@ -137,24 +108,20 @@
0u, 1u, 2u, 4u, 5u, 6u, 8u, 9u, 128u, 128u, 128u, 128u, 10u, 12u, 13u, 14u};
// YUY2 shuf 16 Y to 32 Y.
-static const lvec8 kShuffleYUY2Y = {0, 0, 2, 2, 4, 4, 6, 6, 8, 8, 10,
- 10, 12, 12, 14, 14, 0, 0, 2, 2, 4, 4,
- 6, 6, 8, 8, 10, 10, 12, 12, 14, 14};
+static const vec8 kShuffleYUY2Y = {0, 0, 2, 2, 4, 4, 6, 6,
+ 8, 8, 10, 10, 12, 12, 14, 14};
// YUY2 shuf 8 UV to 16 UV.
-static const lvec8 kShuffleYUY2UV = {1, 3, 1, 3, 5, 7, 5, 7, 9, 11, 9,
- 11, 13, 15, 13, 15, 1, 3, 1, 3, 5, 7,
- 5, 7, 9, 11, 9, 11, 13, 15, 13, 15};
+static const vec8 kShuffleYUY2UV = {1, 3, 1, 3, 5, 7, 5, 7,
+ 9, 11, 9, 11, 13, 15, 13, 15};
// UYVY shuf 16 Y to 32 Y.
-static const lvec8 kShuffleUYVYY = {1, 1, 3, 3, 5, 5, 7, 7, 9, 9, 11,
- 11, 13, 13, 15, 15, 1, 1, 3, 3, 5, 5,
- 7, 7, 9, 9, 11, 11, 13, 13, 15, 15};
+static const vec8 kShuffleUYVYY = {1, 1, 3, 3, 5, 5, 7, 7,
+ 9, 9, 11, 11, 13, 13, 15, 15};
// UYVY shuf 8 UV to 16 UV.
-static const lvec8 kShuffleUYVYUV = {0, 2, 0, 2, 4, 6, 4, 6, 8, 10, 8,
- 10, 12, 14, 12, 14, 0, 2, 0, 2, 4, 6,
- 4, 6, 8, 10, 8, 10, 12, 14, 12, 14};
+static const vec8 kShuffleUYVYUV = {0, 2, 0, 2, 4, 6, 4, 6,
+ 8, 10, 8, 10, 12, 14, 12, 14};
// NV21 shuf 8 VU to 16 UV.
static const lvec8 kShuffleNV21 = {
@@ -170,7 +137,7 @@
"pslld $0x18,%%xmm5 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"movq (%0),%%xmm0 \n"
"lea 0x8(%0),%0 \n"
"punpcklbw %%xmm0,%%xmm0 \n"
@@ -202,7 +169,7 @@
"movdqa %3,%%xmm4 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n"
"movdqu 0x10(%0),%%xmm1 \n"
"movdqu 0x20(%0),%%xmm3 \n"
@@ -235,40 +202,81 @@
void RAWToARGBRow_SSSE3(const uint8_t* src_raw, uint8_t* dst_argb, int width) {
asm volatile(
- "pcmpeqb %%xmm5,%%xmm5 \n" // 0xff000000
- "pslld $0x18,%%xmm5 \n"
+ "pcmpeqb %%xmm6,%%xmm6 \n" // 0xff000000
+ "pslld $0x18,%%xmm6 \n"
"movdqa %3,%%xmm4 \n"
+ "movdqa %4,%%xmm5 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n"
- "movdqu 0x10(%0),%%xmm1 \n"
- "movdqu 0x20(%0),%%xmm3 \n"
+ "movdqu 12(%0),%%xmm1 \n"
+ "movdqu 24(%0),%%xmm2 \n"
+ "movdqu 32(%0),%%xmm3 \n"
"lea 0x30(%0),%0 \n"
- "movdqa %%xmm3,%%xmm2 \n"
- "palignr $0x8,%%xmm1,%%xmm2 \n"
- "pshufb %%xmm4,%%xmm2 \n"
- "por %%xmm5,%%xmm2 \n"
- "palignr $0xc,%%xmm0,%%xmm1 \n"
"pshufb %%xmm4,%%xmm0 \n"
- "movdqu %%xmm2,0x20(%1) \n"
- "por %%xmm5,%%xmm0 \n"
"pshufb %%xmm4,%%xmm1 \n"
- "movdqu %%xmm0,(%1) \n"
- "por %%xmm5,%%xmm1 \n"
- "palignr $0x4,%%xmm3,%%xmm3 \n"
- "pshufb %%xmm4,%%xmm3 \n"
+ "pshufb %%xmm4,%%xmm2 \n"
+ "pshufb %%xmm5,%%xmm3 \n"
+ "por %%xmm6,%%xmm0 \n"
+ "por %%xmm6,%%xmm1 \n"
+ "por %%xmm6,%%xmm2 \n"
+ "por %%xmm6,%%xmm3 \n"
+ "movdqu %%xmm0,0x00(%1) \n"
"movdqu %%xmm1,0x10(%1) \n"
- "por %%xmm5,%%xmm3 \n"
+ "movdqu %%xmm2,0x20(%1) \n"
"movdqu %%xmm3,0x30(%1) \n"
"lea 0x40(%1),%1 \n"
"sub $0x10,%2 \n"
"jg 1b \n"
- : "+r"(src_raw), // %0
- "+r"(dst_argb), // %1
- "+r"(width) // %2
- : "m"(kShuffleMaskRAWToARGB) // %3
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
+ : "+r"(src_raw), // %0
+ "+r"(dst_argb), // %1
+ "+r"(width) // %2
+ : "m"(kShuffleMaskRAWToARGB), // %3
+ "m"(kShuffleMaskRAWToARGB_0) // %4
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6");
+}
+
+void RAWToARGBRow_AVX2(const uint8_t* src_raw, uint8_t* dst_argb, int width) {
+ asm volatile(
+ "vpcmpeqb %%ymm6,%%ymm6,%%ymm6 \n" // 0xff000000
+ "vpslld $0x18,%%ymm6,%%ymm6 \n"
+ "vbroadcastf128 %3,%%ymm4 \n" //
+ "vbroadcastf128 %4,%%ymm5 \n" //
+
+ LABELALIGN //
+ "1: \n"
+ "vmovdqu (%0),%%xmm0 \n" // first 12
+ "vinserti128 $1,12(%0),%%ymm0,%%ymm0 \n" // second 12
+ "vmovdqu 24(%0),%%xmm1 \n" // third 12
+ "vinserti128 $1,36(%0),%%ymm1,%%ymm1 \n" // forth 12
+ "vmovdqu 48(%0),%%xmm2 \n" // fifth 12
+ "vinserti128 $1,60(%0),%%ymm2,%%ymm2 \n" // sixth 12
+ "vmovdqu 68(%0),%%xmm3 \n" // seventh 12
+ "vinserti128 $1,80(%0),%%ymm3,%%ymm3 \n" // eighth 12
+ "lea 96(%0),%0 \n"
+ "vpshufb %%ymm4,%%ymm0,%%ymm0 \n"
+ "vpshufb %%ymm4,%%ymm1,%%ymm1 \n"
+ "vpshufb %%ymm4,%%ymm2,%%ymm2 \n"
+ "vpshufb %%ymm5,%%ymm3,%%ymm3 \n"
+ "vpor %%ymm6,%%ymm0,%%ymm0 \n"
+ "vpor %%ymm6,%%ymm1,%%ymm1 \n"
+ "vpor %%ymm6,%%ymm2,%%ymm2 \n"
+ "vpor %%ymm6,%%ymm3,%%ymm3 \n"
+ "vmovdqu %%ymm0,(%1) \n"
+ "vmovdqu %%ymm1,0x20(%1) \n"
+ "vmovdqu %%ymm2,0x40(%1) \n"
+ "vmovdqu %%ymm3,0x60(%1) \n"
+ "lea 0x80(%1),%1 \n"
+ "sub $0x20,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_raw), // %0
+ "+r"(dst_argb), // %1
+ "+r"(width) // %2
+ : "m"(kShuffleMaskRAWToARGB), // %3
+ "m"(kShuffleMaskRAWToARGB_0) // %4
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6");
}
// Same code as RAWToARGB with different shuffler and A in low bits
@@ -279,7 +287,7 @@
"movdqa %3,%%xmm4 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n"
"movdqu 0x10(%0),%%xmm1 \n"
"movdqu 0x20(%0),%%xmm3 \n"
@@ -319,7 +327,7 @@
"movdqa %5,%%xmm5 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n"
"movdqu 0x4(%0),%%xmm1 \n"
"movdqu 0x8(%0),%%xmm2 \n"
@@ -361,7 +369,7 @@
"sub %0,%1 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n"
"movdqa %%xmm0,%%xmm1 \n"
"movdqa %%xmm0,%%xmm2 \n"
@@ -408,7 +416,7 @@
"sub %0,%1 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n"
"movdqa %%xmm0,%%xmm1 \n"
"movdqa %%xmm0,%%xmm2 \n"
@@ -452,7 +460,7 @@
"sub %0,%1 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n"
"movdqa %%xmm0,%%xmm2 \n"
"pand %%xmm4,%%xmm0 \n"
@@ -479,81 +487,79 @@
}
void ARGBToRGB24Row_SSSE3(const uint8_t* src, uint8_t* dst, int width) {
- asm volatile(
+ asm volatile("movdqa %3,%%xmm6 \n"
- "movdqa %3,%%xmm6 \n"
-
- LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x10(%0),%%xmm1 \n"
- "movdqu 0x20(%0),%%xmm2 \n"
- "movdqu 0x30(%0),%%xmm3 \n"
- "lea 0x40(%0),%0 \n"
- "pshufb %%xmm6,%%xmm0 \n"
- "pshufb %%xmm6,%%xmm1 \n"
- "pshufb %%xmm6,%%xmm2 \n"
- "pshufb %%xmm6,%%xmm3 \n"
- "movdqa %%xmm1,%%xmm4 \n"
- "psrldq $0x4,%%xmm1 \n"
- "pslldq $0xc,%%xmm4 \n"
- "movdqa %%xmm2,%%xmm5 \n"
- "por %%xmm4,%%xmm0 \n"
- "pslldq $0x8,%%xmm5 \n"
- "movdqu %%xmm0,(%1) \n"
- "por %%xmm5,%%xmm1 \n"
- "psrldq $0x8,%%xmm2 \n"
- "pslldq $0x4,%%xmm3 \n"
- "por %%xmm3,%%xmm2 \n"
- "movdqu %%xmm1,0x10(%1) \n"
- "movdqu %%xmm2,0x20(%1) \n"
- "lea 0x30(%1),%1 \n"
- "sub $0x10,%2 \n"
- "jg 1b \n"
- : "+r"(src), // %0
- "+r"(dst), // %1
- "+r"(width) // %2
- : "m"(kShuffleMaskARGBToRGB24) // %3
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6");
+ LABELALIGN
+ "1: \n"
+ "movdqu (%0),%%xmm0 \n"
+ "movdqu 0x10(%0),%%xmm1 \n"
+ "movdqu 0x20(%0),%%xmm2 \n"
+ "movdqu 0x30(%0),%%xmm3 \n"
+ "lea 0x40(%0),%0 \n"
+ "pshufb %%xmm6,%%xmm0 \n"
+ "pshufb %%xmm6,%%xmm1 \n"
+ "pshufb %%xmm6,%%xmm2 \n"
+ "pshufb %%xmm6,%%xmm3 \n"
+ "movdqa %%xmm1,%%xmm4 \n"
+ "psrldq $0x4,%%xmm1 \n"
+ "pslldq $0xc,%%xmm4 \n"
+ "movdqa %%xmm2,%%xmm5 \n"
+ "por %%xmm4,%%xmm0 \n"
+ "pslldq $0x8,%%xmm5 \n"
+ "movdqu %%xmm0,(%1) \n"
+ "por %%xmm5,%%xmm1 \n"
+ "psrldq $0x8,%%xmm2 \n"
+ "pslldq $0x4,%%xmm3 \n"
+ "por %%xmm3,%%xmm2 \n"
+ "movdqu %%xmm1,0x10(%1) \n"
+ "movdqu %%xmm2,0x20(%1) \n"
+ "lea 0x30(%1),%1 \n"
+ "sub $0x10,%2 \n"
+ "jg 1b \n"
+ : "+r"(src), // %0
+ "+r"(dst), // %1
+ "+r"(width) // %2
+ : "m"(kShuffleMaskARGBToRGB24) // %3
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
+ "xmm6");
}
void ARGBToRAWRow_SSSE3(const uint8_t* src, uint8_t* dst, int width) {
- asm volatile(
+ asm volatile("movdqa %3,%%xmm6 \n"
- "movdqa %3,%%xmm6 \n"
-
- LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x10(%0),%%xmm1 \n"
- "movdqu 0x20(%0),%%xmm2 \n"
- "movdqu 0x30(%0),%%xmm3 \n"
- "lea 0x40(%0),%0 \n"
- "pshufb %%xmm6,%%xmm0 \n"
- "pshufb %%xmm6,%%xmm1 \n"
- "pshufb %%xmm6,%%xmm2 \n"
- "pshufb %%xmm6,%%xmm3 \n"
- "movdqa %%xmm1,%%xmm4 \n"
- "psrldq $0x4,%%xmm1 \n"
- "pslldq $0xc,%%xmm4 \n"
- "movdqa %%xmm2,%%xmm5 \n"
- "por %%xmm4,%%xmm0 \n"
- "pslldq $0x8,%%xmm5 \n"
- "movdqu %%xmm0,(%1) \n"
- "por %%xmm5,%%xmm1 \n"
- "psrldq $0x8,%%xmm2 \n"
- "pslldq $0x4,%%xmm3 \n"
- "por %%xmm3,%%xmm2 \n"
- "movdqu %%xmm1,0x10(%1) \n"
- "movdqu %%xmm2,0x20(%1) \n"
- "lea 0x30(%1),%1 \n"
- "sub $0x10,%2 \n"
- "jg 1b \n"
- : "+r"(src), // %0
- "+r"(dst), // %1
- "+r"(width) // %2
- : "m"(kShuffleMaskARGBToRAW) // %3
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6");
+ LABELALIGN
+ "1: \n"
+ "movdqu (%0),%%xmm0 \n"
+ "movdqu 0x10(%0),%%xmm1 \n"
+ "movdqu 0x20(%0),%%xmm2 \n"
+ "movdqu 0x30(%0),%%xmm3 \n"
+ "lea 0x40(%0),%0 \n"
+ "pshufb %%xmm6,%%xmm0 \n"
+ "pshufb %%xmm6,%%xmm1 \n"
+ "pshufb %%xmm6,%%xmm2 \n"
+ "pshufb %%xmm6,%%xmm3 \n"
+ "movdqa %%xmm1,%%xmm4 \n"
+ "psrldq $0x4,%%xmm1 \n"
+ "pslldq $0xc,%%xmm4 \n"
+ "movdqa %%xmm2,%%xmm5 \n"
+ "por %%xmm4,%%xmm0 \n"
+ "pslldq $0x8,%%xmm5 \n"
+ "movdqu %%xmm0,(%1) \n"
+ "por %%xmm5,%%xmm1 \n"
+ "psrldq $0x8,%%xmm2 \n"
+ "pslldq $0x4,%%xmm3 \n"
+ "por %%xmm3,%%xmm2 \n"
+ "movdqu %%xmm1,0x10(%1) \n"
+ "movdqu %%xmm2,0x20(%1) \n"
+ "lea 0x30(%1),%1 \n"
+ "sub $0x10,%2 \n"
+ "jg 1b \n"
+ : "+r"(src), // %0
+ "+r"(dst), // %1
+ "+r"(width) // %2
+ : "m"(kShuffleMaskARGBToRAW) // %3
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
+ "xmm6");
}
#ifdef HAS_ARGBTORGB24ROW_AVX2
@@ -566,7 +572,7 @@
"vmovdqa %4,%%ymm7 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"vmovdqu (%0),%%ymm0 \n"
"vmovdqu 0x20(%0),%%ymm1 \n"
"vmovdqu 0x40(%0),%%ymm2 \n"
@@ -594,7 +600,7 @@
"lea 0x60(%1),%1 \n"
"sub $0x20,%2 \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: "+r"(src), // %0
"+r"(dst), // %1
"+r"(width) // %2
@@ -627,7 +633,7 @@
"vmovdqa %5,%%ymm7 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"vmovdqu (%0),%%ymm0 \n"
"vmovdqu 0x20(%0),%%ymm1 \n"
"vmovdqu 0x40(%0),%%ymm2 \n"
@@ -642,7 +648,7 @@
"lea 0x60(%1),%1 \n"
"sub $0x20,%2 \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: "+r"(src), // %0
"+r"(dst), // %1
"+r"(width) // %2
@@ -660,7 +666,7 @@
"vmovdqa %4,%%ymm7 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"vmovdqu (%0),%%ymm0 \n"
"vmovdqu 0x20(%0),%%ymm1 \n"
"vmovdqu 0x40(%0),%%ymm2 \n"
@@ -688,7 +694,7 @@
"lea 0x60(%1),%1 \n"
"sub $0x20,%2 \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: "+r"(src), // %0
"+r"(dst), // %1
"+r"(width) // %2
@@ -710,7 +716,7 @@
"pslld $0xb,%%xmm5 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n"
"movdqa %%xmm0,%%xmm1 \n"
"movdqa %%xmm0,%%xmm2 \n"
@@ -755,7 +761,7 @@
"pslld $0xb,%%xmm5 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n"
"paddusb %%xmm6,%%xmm0 \n"
"movdqa %%xmm0,%%xmm1 \n"
@@ -801,7 +807,7 @@
"vpslld $0xb,%%ymm3,%%ymm5 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"vmovdqu (%0),%%ymm0 \n"
"vpaddusb %%ymm6,%%ymm0,%%ymm0 \n"
"vpsrld $0x5,%%ymm0,%%ymm2 \n"
@@ -819,7 +825,7 @@
"lea 0x10(%1),%1 \n"
"sub $0x8,%2 \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: "+r"(src), // %0
"+r"(dst), // %1
"+r"(width) // %2
@@ -841,7 +847,7 @@
"pslld $0xf,%%xmm7 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n"
"movdqa %%xmm0,%%xmm1 \n"
"movdqa %%xmm0,%%xmm2 \n"
@@ -878,7 +884,7 @@
"psrlw $0x8,%%xmm3 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n"
"movdqa %%xmm0,%%xmm1 \n"
"pand %%xmm3,%%xmm0 \n"
@@ -946,7 +952,7 @@
"pshufd $0x0,%%xmm6,%%xmm6 \n"
"sub %0,%1 \n"
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n" // fetch 4 ARGB pixels
"movdqa %%xmm0,%%xmm1 \n"
"pshufb %%xmm2,%%xmm1 \n" // R0B0
@@ -985,7 +991,7 @@
"pshufd $0x0,%%xmm6,%%xmm6 \n"
"sub %0,%1 \n"
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n" // fetch 4 ABGR pixels
"movdqa %%xmm0,%%xmm1 \n"
"pshufb %%xmm2,%%xmm1 \n" // R0B0
@@ -1021,7 +1027,7 @@
"vbroadcastss %7,%%ymm6 \n" // multipler for AG
"sub %0,%1 \n"
- "1: \n"
+ "1: \n"
"vmovdqu (%0),%%ymm0 \n" // fetch 8 ARGB pixels
"vpshufb %%ymm2,%%ymm0,%%ymm1 \n" // R0B0
"vpand %%ymm5,%%ymm0,%%ymm0 \n" // A0G0
@@ -1034,7 +1040,7 @@
"add $0x20,%0 \n"
"sub $0x8,%2 \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: "+r"(src), // %0
"+r"(dst), // %1
@@ -1058,7 +1064,7 @@
"vbroadcastss %7,%%ymm6 \n" // multipler for AG
"sub %0,%1 \n"
- "1: \n"
+ "1: \n"
"vmovdqu (%0),%%ymm0 \n" // fetch 8 ABGR pixels
"vpshufb %%ymm2,%%ymm0,%%ymm1 \n" // R0B0
"vpand %%ymm5,%%ymm0,%%ymm0 \n" // A0G0
@@ -1071,7 +1077,7 @@
"add $0x20,%0 \n"
"sub $0x8,%2 \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: "+r"(src), // %0
"+r"(dst), // %1
@@ -1097,9 +1103,7 @@
uint16_t* dst_ar64,
int width) {
asm volatile(
-
- LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n"
"movdqa %%xmm0,%%xmm1 \n"
"punpcklbw %%xmm0,%%xmm0 \n"
@@ -1113,18 +1117,17 @@
: "+r"(src_argb), // %0
"+r"(dst_ar64), // %1
"+r"(width) // %2
- :
- : "memory", "cc", "xmm0", "xmm1");
+ ::"memory",
+ "cc", "xmm0", "xmm1");
}
void ARGBToAB64Row_SSSE3(const uint8_t* src_argb,
uint16_t* dst_ab64,
int width) {
asm volatile(
-
"movdqa %3,%%xmm2 \n"
"movdqa %4,%%xmm3 \n" LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n"
"movdqa %%xmm0,%%xmm1 \n"
"pshufb %%xmm2,%%xmm0 \n"
@@ -1147,9 +1150,7 @@
uint8_t* dst_argb,
int width) {
asm volatile(
-
- LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n"
"movdqu 0x10(%0),%%xmm1 \n"
"psrlw $8,%%xmm0 \n"
@@ -1163,33 +1164,33 @@
: "+r"(src_ar64), // %0
"+r"(dst_argb), // %1
"+r"(width) // %2
- :
- : "memory", "cc", "xmm0", "xmm1");
+ ::"memory",
+ "cc", "xmm0", "xmm1");
}
void AB64ToARGBRow_SSSE3(const uint16_t* src_ab64,
uint8_t* dst_argb,
int width) {
- asm volatile(
+ asm volatile("movdqa %3,%%xmm2 \n"
- "movdqa %3,%%xmm2 \n" LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x10(%0),%%xmm1 \n"
- "psrlw $8,%%xmm0 \n"
- "psrlw $8,%%xmm1 \n"
- "packuswb %%xmm1,%%xmm0 \n"
- "pshufb %%xmm2,%%xmm0 \n"
- "movdqu %%xmm0,(%1) \n"
- "lea 0x20(%0),%0 \n"
- "lea 0x10(%1),%1 \n"
- "sub $0x4,%2 \n"
- "jg 1b \n"
- : "+r"(src_ab64), // %0
- "+r"(dst_argb), // %1
- "+r"(width) // %2
- : "m"(kShuffleARGBToABGR) // %3
- : "memory", "cc", "xmm0", "xmm1", "xmm2");
+ LABELALIGN
+ "1: \n"
+ "movdqu (%0),%%xmm0 \n"
+ "movdqu 0x10(%0),%%xmm1 \n"
+ "psrlw $8,%%xmm0 \n"
+ "psrlw $8,%%xmm1 \n"
+ "packuswb %%xmm1,%%xmm0 \n"
+ "pshufb %%xmm2,%%xmm0 \n"
+ "movdqu %%xmm0,(%1) \n"
+ "lea 0x20(%0),%0 \n"
+ "lea 0x10(%1),%1 \n"
+ "sub $0x4,%2 \n"
+ "jg 1b \n"
+ : "+r"(src_ab64), // %0
+ "+r"(dst_argb), // %1
+ "+r"(width) // %2
+ : "m"(kShuffleARGBToABGR) // %3
+ : "memory", "cc", "xmm0", "xmm1", "xmm2");
}
#ifdef HAS_ARGBTOAR64ROW_AVX2
@@ -1197,9 +1198,7 @@
uint16_t* dst_ar64,
int width) {
asm volatile(
-
- LABELALIGN
- "1: \n"
+ "1: \n"
"vmovdqu (%0),%%ymm0 \n"
"vpermq $0xd8,%%ymm0,%%ymm0 \n"
"vpunpckhbw %%ymm0,%%ymm0,%%ymm1 \n"
@@ -1210,12 +1209,12 @@
"lea 0x40(%1),%1 \n"
"sub $0x8,%2 \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: "+r"(src_argb), // %0
"+r"(dst_ar64), // %1
"+r"(width) // %2
- :
- : "memory", "cc", "xmm0", "xmm1");
+ ::"memory",
+ "cc", "xmm0", "xmm1");
}
#endif
@@ -1224,10 +1223,9 @@
uint16_t* dst_ab64,
int width) {
asm volatile(
-
"vbroadcastf128 %3,%%ymm2 \n"
"vbroadcastf128 %4,%%ymm3 \n" LABELALIGN
- "1: \n"
+ "1: \n"
"vmovdqu (%0),%%ymm0 \n"
"vpermq $0xd8,%%ymm0,%%ymm0 \n"
"vpshufb %%ymm3,%%ymm0,%%ymm1 \n"
@@ -1238,7 +1236,7 @@
"lea 0x40(%1),%1 \n"
"sub $0x8,%2 \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: "+r"(src_argb), // %0
"+r"(dst_ab64), // %1
"+r"(width) // %2
@@ -1253,9 +1251,7 @@
uint8_t* dst_argb,
int width) {
asm volatile(
-
- LABELALIGN
- "1: \n"
+ "1: \n"
"vmovdqu (%0),%%ymm0 \n"
"vmovdqu 0x20(%0),%%ymm1 \n"
"vpsrlw $8,%%ymm0,%%ymm0 \n"
@@ -1267,12 +1263,12 @@
"lea 0x20(%1),%1 \n"
"sub $0x8,%2 \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: "+r"(src_ar64), // %0
"+r"(dst_argb), // %1
"+r"(width) // %2
- :
- : "memory", "cc", "xmm0", "xmm1");
+ ::"memory",
+ "cc", "xmm0", "xmm1");
}
#endif
@@ -1280,28 +1276,26 @@
void AB64ToARGBRow_AVX2(const uint16_t* src_ab64,
uint8_t* dst_argb,
int width) {
- asm volatile(
-
- "vbroadcastf128 %3,%%ymm2 \n" LABELALIGN
- "1: \n"
- "vmovdqu (%0),%%ymm0 \n"
- "vmovdqu 0x20(%0),%%ymm1 \n"
- "vpsrlw $8,%%ymm0,%%ymm0 \n"
- "vpsrlw $8,%%ymm1,%%ymm1 \n"
- "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n"
- "vpermq $0xd8,%%ymm0,%%ymm0 \n"
- "vpshufb %%ymm2,%%ymm0,%%ymm0 \n"
- "vmovdqu %%ymm0,(%1) \n"
- "lea 0x40(%0),%0 \n"
- "lea 0x20(%1),%1 \n"
- "sub $0x8,%2 \n"
- "jg 1b \n"
- "vzeroupper \n"
- : "+r"(src_ab64), // %0
- "+r"(dst_argb), // %1
- "+r"(width) // %2
- : "m"(kShuffleARGBToABGR) // %3
- : "memory", "cc", "xmm0", "xmm1", "xmm2");
+ asm volatile("vbroadcastf128 %3,%%ymm2 \n" LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vmovdqu 0x20(%0),%%ymm1 \n"
+ "vpsrlw $8,%%ymm0,%%ymm0 \n"
+ "vpsrlw $8,%%ymm1,%%ymm1 \n"
+ "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "vpshufb %%ymm2,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm0,(%1) \n"
+ "lea 0x40(%0),%0 \n"
+ "lea 0x20(%1),%1 \n"
+ "sub $0x8,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_ab64), // %0
+ "+r"(dst_argb), // %1
+ "+r"(width) // %2
+ : "m"(kShuffleARGBToABGR) // %3
+ : "memory", "cc", "xmm0", "xmm1", "xmm2");
}
#endif
@@ -1359,8 +1353,8 @@
"vphaddw %%ymm1,%%ymm0,%%ymm0 \n" /* mutates. */ \
"vphaddw %%ymm3,%%ymm2,%%ymm2 \n" \
"prefetcht0 1280(%0) \n" \
- "vpaddw %%" #round ",%%ymm0,%%ymm0 \n" /* Add .5 for rounding. */ \
- "vpaddw %%" #round ",%%ymm2,%%ymm2 \n" \
+ "vpaddw %%" #round ",%%ymm0,%%ymm0 \n" /* Add 16 */ \
+ "vpaddw %%" #round ",%%ymm2,%%ymm2 \n" \
"vpsrlw $0x8,%%ymm0,%%ymm0 \n" \
"vpsrlw $0x8,%%ymm2,%%ymm2 \n" \
"vpackuswb %%ymm2,%%ymm0,%%ymm0 \n" /* mutates. */ \
@@ -1368,8 +1362,7 @@
"vmovdqu %%ymm0,(%1) \n" \
"lea 0x20(%1),%1 \n" \
"sub $0x20,%2 \n" \
- "jg 1b \n" \
- "vzeroupper \n"
+ "jg 1b \n"
// clang-format on
@@ -1379,9 +1372,10 @@
asm volatile(
"movdqa %3,%%xmm4 \n"
"movdqa %4,%%xmm5 \n"
- "movdqa %5,%%xmm7 \n"
+ "movdqa %5,%%xmm7 \n" //
- LABELALIGN RGBTOY(xmm7)
+ LABELALIGN "" //
+ RGBTOY(xmm7) //
: "+r"(src_argb), // %0
"+r"(dst_y), // %1
"+r"(width) // %2
@@ -1400,14 +1394,18 @@
asm volatile(
"movdqa %3,%%xmm4 \n"
"movdqa %4,%%xmm5 \n"
+ "movdqa %5,%%xmm7 \n" //
- LABELALIGN RGBTOY(xmm5)
+ LABELALIGN "" //
+ RGBTOY(xmm7) //
: "+r"(src_argb), // %0
"+r"(dst_y), // %1
"+r"(width) // %2
: "m"(kARGBToYJ), // %3
- "m"(kSub128) // %4
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6");
+ "m"(kSub128), // %4
+ "m"(kAddY0) // %5
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ "xmm7");
}
#endif // HAS_ARGBTOYJROW_SSSE3
@@ -1418,14 +1416,18 @@
asm volatile(
"movdqa %3,%%xmm4 \n"
"movdqa %4,%%xmm5 \n"
+ "movdqa %5,%%xmm7 \n" //
- LABELALIGN RGBTOY(xmm5)
+ LABELALIGN "" //
+ RGBTOY(xmm7) //
: "+r"(src_abgr), // %0
"+r"(dst_y), // %1
"+r"(width) // %2
: "m"(kABGRToYJ), // %3
- "m"(kSub128) // %4
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6");
+ "m"(kSub128), // %4
+ "m"(kAddY0) // %5
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ "xmm7");
}
#endif // HAS_ABGRTOYJROW_SSSE3
@@ -1436,14 +1438,18 @@
asm volatile(
"movdqa %3,%%xmm4 \n"
"movdqa %4,%%xmm5 \n"
+ "movdqa %5,%%xmm7 \n" //
- LABELALIGN RGBTOY(xmm5)
+ LABELALIGN "" //
+ RGBTOY(xmm7) //
: "+r"(src_rgba), // %0
"+r"(dst_y), // %1
"+r"(width) // %2
: "m"(kRGBAToYJ), // %3
- "m"(kSub128) // %4
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6");
+ "m"(kSub128), // %4
+ "m"(kAddY0) // %5
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ "xmm7");
}
#endif // HAS_RGBATOYJROW_SSSE3
@@ -1461,8 +1467,11 @@
"vbroadcastf128 %3,%%ymm4 \n"
"vbroadcastf128 %4,%%ymm5 \n"
"vbroadcastf128 %5,%%ymm7 \n"
- "vmovdqu %6,%%ymm6 \n" LABELALIGN RGBTOY_AVX2(
- ymm7) "vzeroupper \n"
+ "vmovdqa %6,%%ymm6 \n" //
+
+ LABELALIGN "" //
+ RGBTOY_AVX2(ymm7) //
+ "vzeroupper \n"
: "+r"(src_argb), // %0
"+r"(dst_y), // %1
"+r"(width) // %2
@@ -1482,8 +1491,11 @@
"vbroadcastf128 %3,%%ymm4 \n"
"vbroadcastf128 %4,%%ymm5 \n"
"vbroadcastf128 %5,%%ymm7 \n"
- "vmovdqu %6,%%ymm6 \n" LABELALIGN RGBTOY_AVX2(
- ymm7) "vzeroupper \n"
+ "vmovdqa %6,%%ymm6 \n" //
+
+ LABELALIGN "" //
+ RGBTOY_AVX2(ymm7) //
+ "vzeroupper \n"
: "+r"(src_abgr), // %0
"+r"(dst_y), // %1
"+r"(width) // %2
@@ -1502,17 +1514,23 @@
asm volatile(
"vbroadcastf128 %3,%%ymm4 \n"
"vbroadcastf128 %4,%%ymm5 \n"
- "vmovdqu %5,%%ymm6 \n" LABELALIGN RGBTOY_AVX2(
- ymm5) "vzeroupper \n"
+ "vbroadcastf128 %5,%%ymm7 \n"
+ "vmovdqa %6,%%ymm6 \n" //
+
+ LABELALIGN "" //
+ RGBTOY_AVX2(ymm7) //
+ "vzeroupper \n"
: "+r"(src_argb), // %0
"+r"(dst_y), // %1
"+r"(width) // %2
: "m"(kARGBToYJ), // %3
"m"(kSub128), // %4
- "m"(kPermdARGBToY_AVX) // %5
+ "m"(kAddY0), // %5
+ "m"(kPermdARGBToY_AVX) // %6
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
"xmm7");
}
+
#endif // HAS_ARGBTOYJROW_AVX2
#ifdef HAS_ABGRTOYJROW_AVX2
@@ -1521,14 +1539,19 @@
asm volatile(
"vbroadcastf128 %3,%%ymm4 \n"
"vbroadcastf128 %4,%%ymm5 \n"
- "vmovdqu %5,%%ymm6 \n" LABELALIGN RGBTOY_AVX2(
- ymm5) "vzeroupper \n"
+ "vbroadcastf128 %5,%%ymm7 \n"
+ "vmovdqa %6,%%ymm6 \n" //
+
+ LABELALIGN "" //
+ RGBTOY_AVX2(ymm7) //
+ "vzeroupper \n"
: "+r"(src_abgr), // %0
"+r"(dst_y), // %1
"+r"(width) // %2
: "m"(kABGRToYJ), // %3
"m"(kSub128), // %4
- "m"(kPermdARGBToY_AVX) // %5
+ "m"(kAddY0), // %5
+ "m"(kPermdARGBToY_AVX) // %6
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
"xmm7");
}
@@ -1540,483 +1563,37 @@
asm volatile(
"vbroadcastf128 %3,%%ymm4 \n"
"vbroadcastf128 %4,%%ymm5 \n"
- "vmovdqu %5,%%ymm6 \n" LABELALIGN RGBTOY_AVX2(
- ymm5) "vzeroupper \n"
+ "vbroadcastf128 %5,%%ymm7 \n"
+ "vmovdqa %6,%%ymm6 \n" //
+
+ LABELALIGN "" //
+ RGBTOY_AVX2(ymm7) //
+ "vzeroupper \n"
: "+r"(src_rgba), // %0
"+r"(dst_y), // %1
"+r"(width) // %2
: "m"(kRGBAToYJ), // %3
"m"(kSub128), // %4
- "m"(kPermdARGBToY_AVX) // %5
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6");
+ "m"(kAddY0), // %5
+ "m"(kPermdARGBToY_AVX) // %6
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ "xmm7");
}
#endif // HAS_RGBATOYJROW_AVX2
-#ifdef HAS_ARGBTOUVROW_SSSE3
-void ARGBToUVRow_SSSE3(const uint8_t* src_argb,
- int src_stride_argb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- asm volatile(
- "movdqa %5,%%xmm3 \n"
- "movdqa %6,%%xmm4 \n"
- "movdqa %7,%%xmm5 \n"
- "sub %1,%2 \n"
-
- LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x00(%0,%4,1),%%xmm7 \n"
- "pavgb %%xmm7,%%xmm0 \n"
- "movdqu 0x10(%0),%%xmm1 \n"
- "movdqu 0x10(%0,%4,1),%%xmm7 \n"
- "pavgb %%xmm7,%%xmm1 \n"
- "movdqu 0x20(%0),%%xmm2 \n"
- "movdqu 0x20(%0,%4,1),%%xmm7 \n"
- "pavgb %%xmm7,%%xmm2 \n"
- "movdqu 0x30(%0),%%xmm6 \n"
- "movdqu 0x30(%0,%4,1),%%xmm7 \n"
- "pavgb %%xmm7,%%xmm6 \n"
-
- "lea 0x40(%0),%0 \n"
- "movdqa %%xmm0,%%xmm7 \n"
- "shufps $0x88,%%xmm1,%%xmm0 \n"
- "shufps $0xdd,%%xmm1,%%xmm7 \n"
- "pavgb %%xmm7,%%xmm0 \n"
- "movdqa %%xmm2,%%xmm7 \n"
- "shufps $0x88,%%xmm6,%%xmm2 \n"
- "shufps $0xdd,%%xmm6,%%xmm7 \n"
- "pavgb %%xmm7,%%xmm2 \n"
- "movdqa %%xmm0,%%xmm1 \n"
- "movdqa %%xmm2,%%xmm6 \n"
- "pmaddubsw %%xmm4,%%xmm0 \n"
- "pmaddubsw %%xmm4,%%xmm2 \n"
- "pmaddubsw %%xmm3,%%xmm1 \n"
- "pmaddubsw %%xmm3,%%xmm6 \n"
- "phaddw %%xmm2,%%xmm0 \n"
- "phaddw %%xmm6,%%xmm1 \n"
- "psraw $0x8,%%xmm0 \n"
- "psraw $0x8,%%xmm1 \n"
- "packsswb %%xmm1,%%xmm0 \n"
- "paddb %%xmm5,%%xmm0 \n"
- "movlps %%xmm0,(%1) \n"
- "movhps %%xmm0,0x00(%1,%2,1) \n"
- "lea 0x8(%1),%1 \n"
- "sub $0x10,%3 \n"
- "jg 1b \n"
- : "+r"(src_argb), // %0
- "+r"(dst_u), // %1
- "+r"(dst_v), // %2
- "+rm"(width) // %3
- : "r"((intptr_t)(src_stride_argb)), // %4
- "m"(kARGBToV), // %5
- "m"(kARGBToU), // %6
- "m"(kAddUV128) // %7
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm6", "xmm7");
-}
-#endif // HAS_ARGBTOUVROW_SSSE3
-
-#if defined(HAS_ARGBTOUVROW_AVX2) || defined(HAS_ABGRTOUVROW_AVX2) || \
- defined(HAS_ARGBTOUVJROW_AVX2) || defined(HAS_ABGRTOUVJROW_AVX2)
-// vpshufb for vphaddw + vpackuswb packed to shorts.
-static const lvec8 kShufARGBToUV_AVX = {
- 0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15,
- 0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15};
-#endif
-
-#if defined(HAS_ARGBTOUVROW_AVX2)
-void ARGBToUVRow_AVX2(const uint8_t* src_argb,
- int src_stride_argb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- asm volatile(
- "vbroadcastf128 %5,%%ymm5 \n"
- "vbroadcastf128 %6,%%ymm6 \n"
- "vbroadcastf128 %7,%%ymm7 \n"
- "sub %1,%2 \n"
-
- LABELALIGN
- "1: \n"
- "vmovdqu (%0),%%ymm0 \n"
- "vmovdqu 0x20(%0),%%ymm1 \n"
- "vmovdqu 0x40(%0),%%ymm2 \n"
- "vmovdqu 0x60(%0),%%ymm3 \n"
- "vpavgb 0x00(%0,%4,1),%%ymm0,%%ymm0 \n"
- "vpavgb 0x20(%0,%4,1),%%ymm1,%%ymm1 \n"
- "vpavgb 0x40(%0,%4,1),%%ymm2,%%ymm2 \n"
- "vpavgb 0x60(%0,%4,1),%%ymm3,%%ymm3 \n"
- "lea 0x80(%0),%0 \n"
- "vshufps $0x88,%%ymm1,%%ymm0,%%ymm4 \n"
- "vshufps $0xdd,%%ymm1,%%ymm0,%%ymm0 \n"
- "vpavgb %%ymm4,%%ymm0,%%ymm0 \n"
- "vshufps $0x88,%%ymm3,%%ymm2,%%ymm4 \n"
- "vshufps $0xdd,%%ymm3,%%ymm2,%%ymm2 \n"
- "vpavgb %%ymm4,%%ymm2,%%ymm2 \n"
-
- "vpmaddubsw %%ymm7,%%ymm0,%%ymm1 \n"
- "vpmaddubsw %%ymm7,%%ymm2,%%ymm3 \n"
- "vpmaddubsw %%ymm6,%%ymm0,%%ymm0 \n"
- "vpmaddubsw %%ymm6,%%ymm2,%%ymm2 \n"
- "vphaddw %%ymm3,%%ymm1,%%ymm1 \n"
- "vphaddw %%ymm2,%%ymm0,%%ymm0 \n"
- "vpsraw $0x8,%%ymm1,%%ymm1 \n"
- "vpsraw $0x8,%%ymm0,%%ymm0 \n"
- "vpacksswb %%ymm0,%%ymm1,%%ymm0 \n"
- "vpermq $0xd8,%%ymm0,%%ymm0 \n"
- "vpshufb %8,%%ymm0,%%ymm0 \n"
- "vpaddb %%ymm5,%%ymm0,%%ymm0 \n"
-
- "vextractf128 $0x0,%%ymm0,(%1) \n"
- "vextractf128 $0x1,%%ymm0,0x0(%1,%2,1) \n"
- "lea 0x10(%1),%1 \n"
- "sub $0x20,%3 \n"
- "jg 1b \n"
- "vzeroupper \n"
- : "+r"(src_argb), // %0
- "+r"(dst_u), // %1
- "+r"(dst_v), // %2
- "+rm"(width) // %3
- : "r"((intptr_t)(src_stride_argb)), // %4
- "m"(kAddUV128), // %5
- "m"(kARGBToV), // %6
- "m"(kARGBToU), // %7
- "m"(kShufARGBToUV_AVX) // %8
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
- "xmm7");
-}
-#endif // HAS_ARGBTOUVROW_AVX2
-
-#ifdef HAS_ABGRTOUVROW_AVX2
-void ABGRToUVRow_AVX2(const uint8_t* src_abgr,
- int src_stride_abgr,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- asm volatile(
- "vbroadcastf128 %5,%%ymm5 \n"
- "vbroadcastf128 %6,%%ymm6 \n"
- "vbroadcastf128 %7,%%ymm7 \n"
- "sub %1,%2 \n"
-
- LABELALIGN
- "1: \n"
- "vmovdqu (%0),%%ymm0 \n"
- "vmovdqu 0x20(%0),%%ymm1 \n"
- "vmovdqu 0x40(%0),%%ymm2 \n"
- "vmovdqu 0x60(%0),%%ymm3 \n"
- "vpavgb 0x00(%0,%4,1),%%ymm0,%%ymm0 \n"
- "vpavgb 0x20(%0,%4,1),%%ymm1,%%ymm1 \n"
- "vpavgb 0x40(%0,%4,1),%%ymm2,%%ymm2 \n"
- "vpavgb 0x60(%0,%4,1),%%ymm3,%%ymm3 \n"
- "lea 0x80(%0),%0 \n"
- "vshufps $0x88,%%ymm1,%%ymm0,%%ymm4 \n"
- "vshufps $0xdd,%%ymm1,%%ymm0,%%ymm0 \n"
- "vpavgb %%ymm4,%%ymm0,%%ymm0 \n"
- "vshufps $0x88,%%ymm3,%%ymm2,%%ymm4 \n"
- "vshufps $0xdd,%%ymm3,%%ymm2,%%ymm2 \n"
- "vpavgb %%ymm4,%%ymm2,%%ymm2 \n"
-
- "vpmaddubsw %%ymm7,%%ymm0,%%ymm1 \n"
- "vpmaddubsw %%ymm7,%%ymm2,%%ymm3 \n"
- "vpmaddubsw %%ymm6,%%ymm0,%%ymm0 \n"
- "vpmaddubsw %%ymm6,%%ymm2,%%ymm2 \n"
- "vphaddw %%ymm3,%%ymm1,%%ymm1 \n"
- "vphaddw %%ymm2,%%ymm0,%%ymm0 \n"
- "vpsraw $0x8,%%ymm1,%%ymm1 \n"
- "vpsraw $0x8,%%ymm0,%%ymm0 \n"
- "vpacksswb %%ymm0,%%ymm1,%%ymm0 \n"
- "vpermq $0xd8,%%ymm0,%%ymm0 \n"
- "vpshufb %8,%%ymm0,%%ymm0 \n"
- "vpaddb %%ymm5,%%ymm0,%%ymm0 \n"
-
- "vextractf128 $0x0,%%ymm0,(%1) \n"
- "vextractf128 $0x1,%%ymm0,0x0(%1,%2,1) \n"
- "lea 0x10(%1),%1 \n"
- "sub $0x20,%3 \n"
- "jg 1b \n"
- "vzeroupper \n"
- : "+r"(src_abgr), // %0
- "+r"(dst_u), // %1
- "+r"(dst_v), // %2
- "+rm"(width) // %3
- : "r"((intptr_t)(src_stride_abgr)), // %4
- "m"(kAddUV128), // %5
- "m"(kABGRToV), // %6
- "m"(kABGRToU), // %7
- "m"(kShufARGBToUV_AVX) // %8
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
- "xmm7");
-}
-#endif // HAS_ABGRTOUVROW_AVX2
-
-#ifdef HAS_ARGBTOUVJROW_AVX2
-void ARGBToUVJRow_AVX2(const uint8_t* src_argb,
- int src_stride_argb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- asm volatile(
- "vbroadcastf128 %5,%%ymm5 \n"
- "vbroadcastf128 %6,%%ymm6 \n"
- "vbroadcastf128 %7,%%ymm7 \n"
- "sub %1,%2 \n"
-
- LABELALIGN
- "1: \n"
- "vmovdqu (%0),%%ymm0 \n"
- "vmovdqu 0x20(%0),%%ymm1 \n"
- "vmovdqu 0x40(%0),%%ymm2 \n"
- "vmovdqu 0x60(%0),%%ymm3 \n"
- "vpavgb 0x00(%0,%4,1),%%ymm0,%%ymm0 \n"
- "vpavgb 0x20(%0,%4,1),%%ymm1,%%ymm1 \n"
- "vpavgb 0x40(%0,%4,1),%%ymm2,%%ymm2 \n"
- "vpavgb 0x60(%0,%4,1),%%ymm3,%%ymm3 \n"
- "lea 0x80(%0),%0 \n"
- "vshufps $0x88,%%ymm1,%%ymm0,%%ymm4 \n"
- "vshufps $0xdd,%%ymm1,%%ymm0,%%ymm0 \n"
- "vpavgb %%ymm4,%%ymm0,%%ymm0 \n"
- "vshufps $0x88,%%ymm3,%%ymm2,%%ymm4 \n"
- "vshufps $0xdd,%%ymm3,%%ymm2,%%ymm2 \n"
- "vpavgb %%ymm4,%%ymm2,%%ymm2 \n"
-
- "vpmaddubsw %%ymm7,%%ymm0,%%ymm1 \n"
- "vpmaddubsw %%ymm7,%%ymm2,%%ymm3 \n"
- "vpmaddubsw %%ymm6,%%ymm0,%%ymm0 \n"
- "vpmaddubsw %%ymm6,%%ymm2,%%ymm2 \n"
- "vphaddw %%ymm3,%%ymm1,%%ymm1 \n"
- "vphaddw %%ymm2,%%ymm0,%%ymm0 \n"
- "vpaddw %%ymm5,%%ymm0,%%ymm0 \n"
- "vpaddw %%ymm5,%%ymm1,%%ymm1 \n"
- "vpsraw $0x8,%%ymm1,%%ymm1 \n"
- "vpsraw $0x8,%%ymm0,%%ymm0 \n"
- "vpacksswb %%ymm0,%%ymm1,%%ymm0 \n"
- "vpermq $0xd8,%%ymm0,%%ymm0 \n"
- "vpshufb %8,%%ymm0,%%ymm0 \n"
-
- "vextractf128 $0x0,%%ymm0,(%1) \n"
- "vextractf128 $0x1,%%ymm0,0x0(%1,%2,1) \n"
- "lea 0x10(%1),%1 \n"
- "sub $0x20,%3 \n"
- "jg 1b \n"
- "vzeroupper \n"
- : "+r"(src_argb), // %0
- "+r"(dst_u), // %1
- "+r"(dst_v), // %2
- "+rm"(width) // %3
- : "r"((intptr_t)(src_stride_argb)), // %4
- "m"(kSub128), // %5
- "m"(kARGBToVJ), // %6
- "m"(kARGBToUJ), // %7
- "m"(kShufARGBToUV_AVX) // %8
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
- "xmm7");
-}
-#endif // HAS_ARGBTOUVJROW_AVX2
-
-// TODO(fbarchard): Pass kABGRToVJ / kABGRToUJ as matrix
-#ifdef HAS_ABGRTOUVJROW_AVX2
-void ABGRToUVJRow_AVX2(const uint8_t* src_abgr,
- int src_stride_abgr,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- asm volatile(
- "vbroadcastf128 %5,%%ymm5 \n"
- "vbroadcastf128 %6,%%ymm6 \n"
- "vbroadcastf128 %7,%%ymm7 \n"
- "sub %1,%2 \n"
-
- LABELALIGN
- "1: \n"
- "vmovdqu (%0),%%ymm0 \n"
- "vmovdqu 0x20(%0),%%ymm1 \n"
- "vmovdqu 0x40(%0),%%ymm2 \n"
- "vmovdqu 0x60(%0),%%ymm3 \n"
- "vpavgb 0x00(%0,%4,1),%%ymm0,%%ymm0 \n"
- "vpavgb 0x20(%0,%4,1),%%ymm1,%%ymm1 \n"
- "vpavgb 0x40(%0,%4,1),%%ymm2,%%ymm2 \n"
- "vpavgb 0x60(%0,%4,1),%%ymm3,%%ymm3 \n"
- "lea 0x80(%0),%0 \n"
- "vshufps $0x88,%%ymm1,%%ymm0,%%ymm4 \n"
- "vshufps $0xdd,%%ymm1,%%ymm0,%%ymm0 \n"
- "vpavgb %%ymm4,%%ymm0,%%ymm0 \n"
- "vshufps $0x88,%%ymm3,%%ymm2,%%ymm4 \n"
- "vshufps $0xdd,%%ymm3,%%ymm2,%%ymm2 \n"
- "vpavgb %%ymm4,%%ymm2,%%ymm2 \n"
-
- "vpmaddubsw %%ymm7,%%ymm0,%%ymm1 \n"
- "vpmaddubsw %%ymm7,%%ymm2,%%ymm3 \n"
- "vpmaddubsw %%ymm6,%%ymm0,%%ymm0 \n"
- "vpmaddubsw %%ymm6,%%ymm2,%%ymm2 \n"
- "vphaddw %%ymm3,%%ymm1,%%ymm1 \n"
- "vphaddw %%ymm2,%%ymm0,%%ymm0 \n"
- "vpaddw %%ymm5,%%ymm0,%%ymm0 \n"
- "vpaddw %%ymm5,%%ymm1,%%ymm1 \n"
- "vpsraw $0x8,%%ymm1,%%ymm1 \n"
- "vpsraw $0x8,%%ymm0,%%ymm0 \n"
- "vpacksswb %%ymm0,%%ymm1,%%ymm0 \n"
- "vpermq $0xd8,%%ymm0,%%ymm0 \n"
- "vpshufb %8,%%ymm0,%%ymm0 \n"
-
- "vextractf128 $0x0,%%ymm0,(%1) \n"
- "vextractf128 $0x1,%%ymm0,0x0(%1,%2,1) \n"
- "lea 0x10(%1),%1 \n"
- "sub $0x20,%3 \n"
- "jg 1b \n"
- "vzeroupper \n"
- : "+r"(src_abgr), // %0
- "+r"(dst_u), // %1
- "+r"(dst_v), // %2
- "+rm"(width) // %3
- : "r"((intptr_t)(src_stride_abgr)), // %4
- "m"(kSub128), // %5
- "m"(kABGRToVJ), // %6
- "m"(kABGRToUJ), // %7
- "m"(kShufARGBToUV_AVX) // %8
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
- "xmm7");
-}
-#endif // HAS_ABGRTOUVJROW_AVX2
-
-#ifdef HAS_ARGBTOUVJROW_SSSE3
-void ARGBToUVJRow_SSSE3(const uint8_t* src_argb,
- int src_stride_argb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- asm volatile(
- "movdqa %5,%%xmm3 \n"
- "movdqa %6,%%xmm4 \n"
- "movdqa %7,%%xmm5 \n"
- "sub %1,%2 \n"
-
- LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x00(%0,%4,1),%%xmm7 \n"
- "pavgb %%xmm7,%%xmm0 \n"
- "movdqu 0x10(%0),%%xmm1 \n"
- "movdqu 0x10(%0,%4,1),%%xmm7 \n"
- "pavgb %%xmm7,%%xmm1 \n"
- "movdqu 0x20(%0),%%xmm2 \n"
- "movdqu 0x20(%0,%4,1),%%xmm7 \n"
- "pavgb %%xmm7,%%xmm2 \n"
- "movdqu 0x30(%0),%%xmm6 \n"
- "movdqu 0x30(%0,%4,1),%%xmm7 \n"
- "pavgb %%xmm7,%%xmm6 \n"
-
- "lea 0x40(%0),%0 \n"
- "movdqa %%xmm0,%%xmm7 \n"
- "shufps $0x88,%%xmm1,%%xmm0 \n"
- "shufps $0xdd,%%xmm1,%%xmm7 \n"
- "pavgb %%xmm7,%%xmm0 \n"
- "movdqa %%xmm2,%%xmm7 \n"
- "shufps $0x88,%%xmm6,%%xmm2 \n"
- "shufps $0xdd,%%xmm6,%%xmm7 \n"
- "pavgb %%xmm7,%%xmm2 \n"
- "movdqa %%xmm0,%%xmm1 \n"
- "movdqa %%xmm2,%%xmm6 \n"
- "pmaddubsw %%xmm4,%%xmm0 \n"
- "pmaddubsw %%xmm4,%%xmm2 \n"
- "pmaddubsw %%xmm3,%%xmm1 \n"
- "pmaddubsw %%xmm3,%%xmm6 \n"
- "phaddw %%xmm2,%%xmm0 \n"
- "phaddw %%xmm6,%%xmm1 \n"
- "paddw %%xmm5,%%xmm0 \n"
- "paddw %%xmm5,%%xmm1 \n"
- "psraw $0x8,%%xmm0 \n"
- "psraw $0x8,%%xmm1 \n"
- "packsswb %%xmm1,%%xmm0 \n"
- "movlps %%xmm0,(%1) \n"
- "movhps %%xmm0,0x00(%1,%2,1) \n"
- "lea 0x8(%1),%1 \n"
- "sub $0x10,%3 \n"
- "jg 1b \n"
- : "+r"(src_argb), // %0
- "+r"(dst_u), // %1
- "+r"(dst_v), // %2
- "+rm"(width) // %3
- : "r"((intptr_t)(src_stride_argb)), // %4
- "m"(kARGBToVJ), // %5
- "m"(kARGBToUJ), // %6
- "m"(kSub128) // %7
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm6", "xmm7");
-}
-#endif // HAS_ARGBTOUVJROW_SSSE3
-
-#ifdef HAS_ABGRTOUVJROW_SSSE3
-void ABGRToUVJRow_SSSE3(const uint8_t* src_abgr,
- int src_stride_abgr,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- asm volatile(
- "movdqa %5,%%xmm3 \n"
- "movdqa %6,%%xmm4 \n"
- "movdqa %7,%%xmm5 \n"
- "sub %1,%2 \n"
-
- LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x00(%0,%4,1),%%xmm7 \n"
- "pavgb %%xmm7,%%xmm0 \n"
- "movdqu 0x10(%0),%%xmm1 \n"
- "movdqu 0x10(%0,%4,1),%%xmm7 \n"
- "pavgb %%xmm7,%%xmm1 \n"
- "movdqu 0x20(%0),%%xmm2 \n"
- "movdqu 0x20(%0,%4,1),%%xmm7 \n"
- "pavgb %%xmm7,%%xmm2 \n"
- "movdqu 0x30(%0),%%xmm6 \n"
- "movdqu 0x30(%0,%4,1),%%xmm7 \n"
- "pavgb %%xmm7,%%xmm6 \n"
-
- "lea 0x40(%0),%0 \n"
- "movdqa %%xmm0,%%xmm7 \n"
- "shufps $0x88,%%xmm1,%%xmm0 \n"
- "shufps $0xdd,%%xmm1,%%xmm7 \n"
- "pavgb %%xmm7,%%xmm0 \n"
- "movdqa %%xmm2,%%xmm7 \n"
- "shufps $0x88,%%xmm6,%%xmm2 \n"
- "shufps $0xdd,%%xmm6,%%xmm7 \n"
- "pavgb %%xmm7,%%xmm2 \n"
- "movdqa %%xmm0,%%xmm1 \n"
- "movdqa %%xmm2,%%xmm6 \n"
- "pmaddubsw %%xmm4,%%xmm0 \n"
- "pmaddubsw %%xmm4,%%xmm2 \n"
- "pmaddubsw %%xmm3,%%xmm1 \n"
- "pmaddubsw %%xmm3,%%xmm6 \n"
- "phaddw %%xmm2,%%xmm0 \n"
- "phaddw %%xmm6,%%xmm1 \n"
- "paddw %%xmm5,%%xmm0 \n"
- "paddw %%xmm5,%%xmm1 \n"
- "psraw $0x8,%%xmm0 \n"
- "psraw $0x8,%%xmm1 \n"
- "packsswb %%xmm1,%%xmm0 \n"
- "movlps %%xmm0,(%1) \n"
- "movhps %%xmm0,0x00(%1,%2,1) \n"
- "lea 0x8(%1),%1 \n"
- "sub $0x10,%3 \n"
- "jg 1b \n"
- : "+r"(src_abgr), // %0
- "+r"(dst_u), // %1
- "+r"(dst_v), // %2
- "+rm"(width) // %3
- : "r"((intptr_t)(src_stride_abgr)), // %4
- "m"(kABGRToVJ), // %5
- "m"(kABGRToUJ), // %6
- "m"(kSub128) // %7
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm6", "xmm7");
-}
-#endif // HAS_ABGRTOUVJROW_SSSE3
-
#ifdef HAS_ARGBTOUV444ROW_SSSE3
-void ARGBToUV444Row_SSSE3(const uint8_t* src_argb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
+
+// Coefficients expressed as negatives to allow 128
+struct RgbUVConstants {
+ vec8 kRGBToU;
+ vec8 kRGBToV;
+};
+
+void ARGBToUV444MatrixRow_SSSE3(const uint8_t* src_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct RgbUVConstants* rgbuvconstants) {
asm volatile(
"movdqa %4,%%xmm3 \n"
"movdqa %5,%%xmm4 \n"
@@ -2024,22 +1601,7 @@
"sub %1,%2 \n"
LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x10(%0),%%xmm1 \n"
- "movdqu 0x20(%0),%%xmm2 \n"
- "movdqu 0x30(%0),%%xmm6 \n"
- "pmaddubsw %%xmm4,%%xmm0 \n"
- "pmaddubsw %%xmm4,%%xmm1 \n"
- "pmaddubsw %%xmm4,%%xmm2 \n"
- "pmaddubsw %%xmm4,%%xmm6 \n"
- "phaddw %%xmm1,%%xmm0 \n"
- "phaddw %%xmm6,%%xmm2 \n"
- "psraw $0x8,%%xmm0 \n"
- "psraw $0x8,%%xmm2 \n"
- "packsswb %%xmm2,%%xmm0 \n"
- "paddb %%xmm5,%%xmm0 \n"
- "movdqu %%xmm0,(%1) \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n"
"movdqu 0x10(%0),%%xmm1 \n"
"movdqu 0x20(%0),%%xmm2 \n"
@@ -2050,48 +1612,198 @@
"pmaddubsw %%xmm3,%%xmm6 \n"
"phaddw %%xmm1,%%xmm0 \n"
"phaddw %%xmm6,%%xmm2 \n"
- "psraw $0x8,%%xmm0 \n"
- "psraw $0x8,%%xmm2 \n"
- "packsswb %%xmm2,%%xmm0 \n"
- "paddb %%xmm5,%%xmm0 \n"
+ "movdqa %%xmm5,%%xmm1 \n"
+ "movdqa %%xmm5,%%xmm6 \n"
+ "psubw %%xmm0,%%xmm1 \n"
+ "psubw %%xmm2,%%xmm6 \n"
+ "psrlw $0x8,%%xmm1 \n"
+ "psrlw $0x8,%%xmm6 \n"
+ "packuswb %%xmm6,%%xmm1 \n"
+ "movdqu %%xmm1,(%1) \n"
+
+ "movdqu (%0),%%xmm0 \n"
+ "movdqu 0x10(%0),%%xmm1 \n"
+ "movdqu 0x20(%0),%%xmm2 \n"
+ "movdqu 0x30(%0),%%xmm6 \n"
+ "pmaddubsw %%xmm4,%%xmm0 \n"
+ "pmaddubsw %%xmm4,%%xmm1 \n"
+ "pmaddubsw %%xmm4,%%xmm2 \n"
+ "pmaddubsw %%xmm4,%%xmm6 \n"
+ "phaddw %%xmm1,%%xmm0 \n"
+ "phaddw %%xmm6,%%xmm2 \n"
+ "movdqa %%xmm5,%%xmm1 \n"
+ "movdqa %%xmm5,%%xmm6 \n"
+ "psubw %%xmm0,%%xmm1 \n"
+ "psubw %%xmm2,%%xmm6 \n"
+ "psrlw $0x8,%%xmm1 \n"
+ "psrlw $0x8,%%xmm6 \n"
+ "packuswb %%xmm6,%%xmm1 \n"
+ "movdqu %%xmm1,0x00(%1,%2,1) \n"
+
"lea 0x40(%0),%0 \n"
- "movdqu %%xmm0,0x00(%1,%2,1) \n"
"lea 0x10(%1),%1 \n"
"sub $0x10,%3 \n"
"jg 1b \n"
- : "+r"(src_argb), // %0
- "+r"(dst_u), // %1
- "+r"(dst_v), // %2
- "+rm"(width) // %3
- : "m"(kARGBToV), // %4
- "m"(kARGBToU), // %5
- "m"(kAddUV128) // %6
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm6");
+ : "+r"(src_argb), // %0
+ "+r"(dst_u), // %1
+ "+r"(dst_v), // %2
+ "+rm"(width) // %3
+ : "m"(rgbuvconstants->kRGBToU), // %4
+ "m"(rgbuvconstants->kRGBToV), // %5
+ "m"(kAddUV128) // %6
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6");
}
#endif // HAS_ARGBTOUV444ROW_SSSE3
-void BGRAToYRow_SSSE3(const uint8_t* src_bgra, uint8_t* dst_y, int width) {
- asm volatile(
- "movdqa %3,%%xmm4 \n"
- "movdqa %4,%%xmm5 \n"
- "movdqa %5,%%xmm7 \n"
+#ifdef HAS_ARGBTOUV444ROW_AVX2
- LABELALIGN RGBTOY(xmm7)
- : "+r"(src_bgra), // %0
- "+r"(dst_y), // %1
- "+r"(width) // %2
- : "m"(kBGRAToY), // %3
- "m"(kSub128), // %4
- "m"(kAddY16) // %5
+void ARGBToUV444MatrixRow_AVX2(const uint8_t* src_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct RgbUVConstants* rgbuvconstants) {
+ asm volatile(
+ "vbroadcastf128 %4,%%ymm3 \n"
+ "vbroadcastf128 %5,%%ymm4 \n"
+ "vbroadcastf128 %6,%%ymm5 \n"
+ "vmovdqa %7,%%ymm7 \n"
+ "sub %1,%2 \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vmovdqu 0x20(%0),%%ymm1 \n"
+ "vmovdqu 0x40(%0),%%ymm2 \n"
+ "vmovdqu 0x60(%0),%%ymm6 \n"
+ "vpmaddubsw %%ymm3,%%ymm0,%%ymm0 \n"
+ "vpmaddubsw %%ymm3,%%ymm1,%%ymm1 \n"
+ "vpmaddubsw %%ymm3,%%ymm2,%%ymm2 \n"
+ "vpmaddubsw %%ymm3,%%ymm6,%%ymm6 \n"
+ "vphaddw %%ymm1,%%ymm0,%%ymm0 \n" // mutates
+ "vphaddw %%ymm6,%%ymm2,%%ymm2 \n"
+ "vpsubw %%ymm0,%%ymm5,%%ymm0 \n"
+ "vpsubw %%ymm2,%%ymm5,%%ymm2 \n"
+ "vpsrlw $0x8,%%ymm0,%%ymm0 \n"
+ "vpsrlw $0x8,%%ymm2,%%ymm2 \n"
+ "vpackuswb %%ymm2,%%ymm0,%%ymm0 \n" // mutates
+ "vpermd %%ymm0,%%ymm7,%%ymm0 \n" // unmutate.
+ "vmovdqu %%ymm0,(%1) \n"
+
+ "vmovdqu (%0),%%ymm0 \n"
+ "vmovdqu 0x20(%0),%%ymm1 \n"
+ "vmovdqu 0x40(%0),%%ymm2 \n"
+ "vmovdqu 0x60(%0),%%ymm6 \n"
+ "vpmaddubsw %%ymm4,%%ymm0,%%ymm0 \n"
+ "vpmaddubsw %%ymm4,%%ymm1,%%ymm1 \n"
+ "vpmaddubsw %%ymm4,%%ymm2,%%ymm2 \n"
+ "vpmaddubsw %%ymm4,%%ymm6,%%ymm6 \n"
+ "vphaddw %%ymm1,%%ymm0,%%ymm0 \n" // mutates
+ "vphaddw %%ymm6,%%ymm2,%%ymm2 \n"
+ "vpsubw %%ymm0,%%ymm5,%%ymm0 \n"
+ "vpsubw %%ymm2,%%ymm5,%%ymm2 \n"
+ "vpsrlw $0x8,%%ymm0,%%ymm0 \n"
+ "vpsrlw $0x8,%%ymm2,%%ymm2 \n"
+ "vpackuswb %%ymm2,%%ymm0,%%ymm0 \n" // mutates
+ "vpermd %%ymm0,%%ymm7,%%ymm0 \n" // unmutate.
+ "vmovdqu %%ymm0,(%1,%2,1) \n"
+ "lea 0x80(%0),%0 \n"
+ "lea 0x20(%1),%1 \n"
+ "sub $0x20,%3 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_argb), // %0
+ "+r"(dst_u), // %1
+ "+r"(dst_v), // %2
+ "+rm"(width) // %3
+ : "m"(rgbuvconstants->kRGBToU), // %4
+ "m"(rgbuvconstants->kRGBToV), // %5
+ "m"(kAddUV128), // %6
+ "m"(kPermdARGBToY_AVX) // %7
+ : "memory", "cc", "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6",
+ "ymm7");
+}
+#endif // HAS_ARGBTOUV444ROW_AVX2
+
+// vpshufb for vphaddw + vpackuswb packed to shorts.
+static const lvec8 kShufARGBToUV_AVX = {
+ 0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15,
+ 0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15};
+
+void ARGBToUVMatrixRow_AVX2(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct RgbUVConstants* rgbuvconstants) {
+ asm volatile(
+ "vbroadcastf128 %5,%%ymm5 \n"
+ "vbroadcastf128 %6,%%ymm6 \n"
+ "vbroadcastf128 %7,%%ymm7 \n"
+ "sub %1,%2 \n"
+
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vmovdqu 0x20(%0),%%ymm1 \n"
+ "vmovdqu 0x40(%0),%%ymm2 \n"
+ "vmovdqu 0x60(%0),%%ymm3 \n"
+ "vpavgb 0x00(%0,%4,1),%%ymm0,%%ymm0 \n"
+ "vpavgb 0x20(%0,%4,1),%%ymm1,%%ymm1 \n"
+ "vpavgb 0x40(%0,%4,1),%%ymm2,%%ymm2 \n"
+ "vpavgb 0x60(%0,%4,1),%%ymm3,%%ymm3 \n"
+ "lea 0x80(%0),%0 \n"
+ "vshufps $0x88,%%ymm1,%%ymm0,%%ymm4 \n"
+ "vshufps $0xdd,%%ymm1,%%ymm0,%%ymm0 \n"
+ "vpavgb %%ymm4,%%ymm0,%%ymm0 \n"
+ "vshufps $0x88,%%ymm3,%%ymm2,%%ymm4 \n"
+ "vshufps $0xdd,%%ymm3,%%ymm2,%%ymm2 \n"
+ "vpavgb %%ymm4,%%ymm2,%%ymm2 \n"
+
+ "vpmaddubsw %%ymm6,%%ymm0,%%ymm1 \n"
+ "vpmaddubsw %%ymm6,%%ymm2,%%ymm3 \n"
+ "vpmaddubsw %%ymm7,%%ymm0,%%ymm0 \n"
+ "vpmaddubsw %%ymm7,%%ymm2,%%ymm2 \n"
+ "vphaddw %%ymm3,%%ymm1,%%ymm1 \n"
+ "vphaddw %%ymm2,%%ymm0,%%ymm0 \n"
+ "vpsubw %%ymm0,%%ymm5,%%ymm0 \n"
+ "vpsubw %%ymm1,%%ymm5,%%ymm1 \n"
+ "vpsrlw $0x8,%%ymm1,%%ymm1 \n"
+ "vpsrlw $0x8,%%ymm0,%%ymm0 \n"
+ "vpackuswb %%ymm0,%%ymm1,%%ymm0 \n"
+ "vpermq $0xd8,%%ymm0,%%ymm0 \n"
+ "vpshufb %8,%%ymm0,%%ymm0 \n"
+
+ "vextractf128 $0x0,%%ymm0,(%1) \n"
+ "vextractf128 $0x1,%%ymm0,0x0(%1,%2,1) \n"
+ "lea 0x10(%1),%1 \n"
+ "subl $0x20,%3 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_argb), // %0
+ "+r"(dst_u), // %1
+ "+r"(dst_v), // %2
+#if defined(__i386__)
+ "+m"(width) // %3
+#else
+ "+rm"(width) // %3
+#endif
+ : "r"((intptr_t)(src_stride_argb)), // %4
+ "m"(kAddUV128), // %5
+ "m"(rgbuvconstants->kRGBToU), // %6
+ "m"(rgbuvconstants->kRGBToV), // %7
+ "m"(kShufARGBToUV_AVX) // %8
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
"xmm7");
}
-void BGRAToUVRow_SSSE3(const uint8_t* src_bgra,
- int src_stride_bgra,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
+#ifdef HAS_ARGBTOUVROW_SSSE3
+
+void ARGBToUVMatrixRow_SSSE3(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct RgbUVConstants* rgbuvconstants) {
asm volatile(
"movdqa %5,%%xmm3 \n"
"movdqa %6,%%xmm4 \n"
@@ -2099,7 +1811,7 @@
"sub %1,%2 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n"
"movdqu 0x00(%0,%4,1),%%xmm7 \n"
"pavgb %%xmm7,%%xmm0 \n"
@@ -2112,7 +1824,6 @@
"movdqu 0x30(%0),%%xmm6 \n"
"movdqu 0x30(%0,%4,1),%%xmm7 \n"
"pavgb %%xmm7,%%xmm6 \n"
-
"lea 0x40(%0),%0 \n"
"movdqa %%xmm0,%%xmm7 \n"
"shufps $0x88,%%xmm1,%%xmm0 \n"
@@ -2122,32 +1833,236 @@
"shufps $0x88,%%xmm6,%%xmm2 \n"
"shufps $0xdd,%%xmm6,%%xmm7 \n"
"pavgb %%xmm7,%%xmm2 \n"
+
"movdqa %%xmm0,%%xmm1 \n"
"movdqa %%xmm2,%%xmm6 \n"
- "pmaddubsw %%xmm4,%%xmm0 \n"
- "pmaddubsw %%xmm4,%%xmm2 \n"
- "pmaddubsw %%xmm3,%%xmm1 \n"
- "pmaddubsw %%xmm3,%%xmm6 \n"
+ "pmaddubsw %%xmm3,%%xmm0 \n"
+ "pmaddubsw %%xmm3,%%xmm2 \n"
+ "pmaddubsw %%xmm4,%%xmm1 \n"
+ "pmaddubsw %%xmm4,%%xmm6 \n"
"phaddw %%xmm2,%%xmm0 \n"
"phaddw %%xmm6,%%xmm1 \n"
- "psraw $0x8,%%xmm0 \n"
- "psraw $0x8,%%xmm1 \n"
- "packsswb %%xmm1,%%xmm0 \n"
- "paddb %%xmm5,%%xmm0 \n"
- "movlps %%xmm0,(%1) \n"
- "movhps %%xmm0,0x00(%1,%2,1) \n"
+ "movdqa %%xmm5,%%xmm2 \n"
+ "movdqa %%xmm5,%%xmm6 \n"
+ "psubw %%xmm0,%%xmm2 \n"
+ "psubw %%xmm1,%%xmm6 \n"
+ "psrlw $0x8,%%xmm2 \n"
+ "psrlw $0x8,%%xmm6 \n"
+ "packuswb %%xmm6,%%xmm2 \n"
+ "movlps %%xmm2,(%1) \n"
+ "movhps %%xmm2,0x00(%1,%2,1) \n"
"lea 0x8(%1),%1 \n"
- "sub $0x10,%3 \n"
+ "subl $0x10,%3 \n"
"jg 1b \n"
- : "+r"(src_bgra), // %0
- "+r"(dst_u), // %1
- "+r"(dst_v), // %2
- "+rm"(width) // %3
- : "r"((intptr_t)(src_stride_bgra)), // %4
- "m"(kBGRAToV), // %5
- "m"(kBGRAToU), // %6
+ : "+r"(src_argb), // %0
+ "+r"(dst_u), // %1
+ "+r"(dst_v), // %2
+#if defined(__i386__)
+ "+m"(width) // %3
+#else
+ "+rm"(width) // %3
+#endif
+ : "r"((intptr_t)(src_stride_argb)), // %4
+ "m"(rgbuvconstants->kRGBToU), // %5
+ "m"(rgbuvconstants->kRGBToV), // %6
"m"(kAddUV128) // %7
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm6", "xmm7");
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ "xmm7");
+}
+
+#endif // HAS_ARGBTOUVROW_SSSE3
+
+#ifdef HAS_ARGBTOUV444ROW_SSSE3
+
+// RGB to BT601 coefficients
+// UB 0.875 coefficient = 112
+// UG -0.5781 coefficient = -74
+// UR -0.2969 coefficient = -38
+// VB -0.1406 coefficient = -18
+// VG -0.7344 coefficient = -94
+// VR 0.875 coefficient = 112
+
+static const struct RgbUVConstants kARGBI601UVConstants = {
+ {-112, 74, 38, 0, -112, 74, 38, 0, -112, 74, 38, 0, -112, 74, 38, 0},
+ {18, 94, -112, 0, 18, 94, -112, 0, 18, 94, -112, 0, 18, 94, -112, 0}};
+
+void ARGBToUV444Row_SSSE3(const uint8_t* src_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ ARGBToUV444MatrixRow_SSSE3(src_argb, dst_u, dst_v, width,
+ &kARGBI601UVConstants);
+}
+#endif // HAS_ARGBTOUV444ROW_SSSE3
+
+#ifdef HAS_ARGBTOUV444ROW_AVX2
+void ARGBToUV444Row_AVX2(const uint8_t* src_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ ARGBToUV444MatrixRow_AVX2(src_argb, dst_u, dst_v, width,
+ &kARGBI601UVConstants);
+}
+#endif // HAS_ARGBTOUV444ROW_AVX2
+
+#ifdef HAS_ARGBTOUVROW_SSSE3
+void ARGBToUVRow_SSSE3(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ ARGBToUVMatrixRow_SSSE3(src_argb, src_stride_argb, dst_u, dst_v, width,
+ &kARGBI601UVConstants);
+}
+void ARGBToUVRow_AVX2(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ ARGBToUVMatrixRow_AVX2(src_argb, src_stride_argb, dst_u, dst_v, width,
+ &kARGBI601UVConstants);
+}
+
+static const struct RgbUVConstants kABGRI601UVConstants = {
+ {38, 74, -112, 0, 38, 74, -112, 0, 38, 74, -112, 0, 38, 74, -112, 0},
+ {-112, 94, 18, 0, -112, 94, 18, 0, -112, 94, 18, 0, -112, 94, 18, 0}};
+
+void ABGRToUVRow_SSSE3(const uint8_t* src_abgr,
+ int src_stride_abgr,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ ARGBToUVMatrixRow_SSSE3(src_abgr, src_stride_abgr, dst_u, dst_v, width,
+ &kABGRI601UVConstants);
+}
+
+void ABGRToUVRow_AVX2(const uint8_t* src_abgr,
+ int src_stride_abgr,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ ARGBToUVMatrixRow_AVX2(src_abgr, src_stride_abgr, dst_u, dst_v, width,
+ &kABGRI601UVConstants);
+}
+
+static const struct RgbUVConstants kBGRAI601UVConstants = {
+ {0, 38, 74, -112, 0, 38, 74, -112, 0, 38, 74, -112, 0, 38, 74, -112},
+ {0, -112, 94, 18, 0, -112, 94, 18, 0, -112, 94, 18, 0, -112, 94, 18}};
+
+void BGRAToUVRow_SSSE3(const uint8_t* src_bgra,
+ int src_stride_bgra,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ ARGBToUVMatrixRow_SSSE3(src_bgra, src_stride_bgra, dst_u, dst_v, width,
+ &kBGRAI601UVConstants);
+}
+
+static const struct RgbUVConstants kRGBAI601UVConstants = {
+ {0, -112, 74, 38, 0, -112, 74, 38, 0, -112, 74, 38, 0, -112, 74, 38},
+ {0, 18, 94, -112, 0, 18, 94, -112, 0, 18, 94, -112, 0, 18, 94, -112}};
+
+void RGBAToUVRow_SSSE3(const uint8_t* src_rgba,
+ int src_stride_rgba,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ ARGBToUVMatrixRow_SSSE3(src_rgba, src_stride_rgba, dst_u, dst_v, width,
+ &kRGBAI601UVConstants);
+}
+#endif // HAS_ARGBTOUVROW_SSSE3
+
+#ifdef HAS_ARGBTOUVJ444ROW_SSSE3
+// RGB to JPEG coefficients
+// UB 0.500 coefficient = 128
+// UG -0.33126 coefficient = -85
+// UR -0.16874 coefficient = -43
+// VB -0.08131 coefficient = -21
+// VG -0.41869 coefficient = -107
+// VR 0.500 coefficient = 128
+
+static const struct RgbUVConstants kARGBJPEGUVConstants = {
+ {-128, 85, 43, 0, -128, 85, 43, 0, -128, 85, 43, 0, -128, 85, 43, 0},
+ {21, 107, -128, 0, 21, 107, -128, 0, 21, 107, -128, 0, 21, 107, -128, 0}};
+
+void ARGBToUVJ444Row_SSSE3(const uint8_t* src_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ ARGBToUV444MatrixRow_SSSE3(src_argb, dst_u, dst_v, width,
+ &kARGBJPEGUVConstants);
+}
+
+#endif // HAS_ARGBTOUVJ444ROW_SSSE3
+
+#ifdef HAS_ARGBTOUVJ444ROW_AVX2
+void ARGBToUVJ444Row_AVX2(const uint8_t* src_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ ARGBToUV444MatrixRow_AVX2(src_argb, dst_u, dst_v, width,
+ &kARGBJPEGUVConstants);
+}
+#endif // HAS_ARGBTOUVJ444ROW_AVX2
+
+void ARGBToUVJRow_AVX2(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ ARGBToUVMatrixRow_AVX2(src_argb, src_stride_argb, dst_u, dst_v, width,
+ &kARGBJPEGUVConstants);
+}
+
+static const struct RgbUVConstants kABGRJPEGUVConstants = {
+ {43, 85, -128, 0, 43, 85, -128, 0, 43, 85, -128, 0, 43, 85, -128, 0},
+ {-128, 107, 21, 0, -128, 107, 21, 0, -128, 107, 21, 0, -128, 107, 21, 0}};
+
+void ABGRToUVJRow_AVX2(const uint8_t* src_abgr,
+ int src_stride_abgr,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ ARGBToUVMatrixRow_AVX2(src_abgr, src_stride_abgr, dst_u, dst_v, width,
+ &kABGRJPEGUVConstants);
+}
+
+#ifdef HAS_ARGBTOUVJROW_SSSE3
+void ARGBToUVJRow_SSSE3(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ ARGBToUVMatrixRow_SSSE3(src_argb, src_stride_argb, dst_u, dst_v, width,
+ &kARGBJPEGUVConstants);
+}
+
+void ABGRToUVJRow_SSSE3(const uint8_t* src_abgr,
+ int src_stride_abgr,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ ARGBToUVMatrixRow_SSSE3(src_abgr, src_stride_abgr, dst_u, dst_v, width,
+ &kABGRJPEGUVConstants);
+}
+#endif // HAS_ABGRTOUVJROW_SSSE3
+
+void BGRAToYRow_SSSE3(const uint8_t* src_bgra, uint8_t* dst_y, int width) {
+ asm volatile(
+ "movdqa %3,%%xmm4 \n"
+ "movdqa %4,%%xmm5 \n"
+ "movdqa %5,%%xmm7 \n"
+
+ LABELALIGN "" //
+ RGBTOY(xmm7)
+ : "+r"(src_bgra), // %0
+ "+r"(dst_y), // %1
+ "+r"(width) // %2
+ : "m"(kBGRAToY), // %3
+ "m"(kSub128), // %4
+ "m"(kAddY16) // %5
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ "xmm7");
}
void ABGRToYRow_SSSE3(const uint8_t* src_abgr, uint8_t* dst_y, int width) {
@@ -2156,7 +2071,8 @@
"movdqa %4,%%xmm5 \n"
"movdqa %5,%%xmm7 \n"
- LABELALIGN RGBTOY(xmm7)
+ LABELALIGN "" //
+ RGBTOY(xmm7)
: "+r"(src_abgr), // %0
"+r"(dst_y), // %1
"+r"(width) // %2
@@ -2173,7 +2089,8 @@
"movdqa %4,%%xmm5 \n"
"movdqa %5,%%xmm7 \n"
- LABELALIGN RGBTOY(xmm7)
+ LABELALIGN "" //
+ RGBTOY(xmm7)
: "+r"(src_rgba), // %0
"+r"(dst_y), // %1
"+r"(width) // %2
@@ -2184,132 +2101,6 @@
"xmm7");
}
-void ABGRToUVRow_SSSE3(const uint8_t* src_abgr,
- int src_stride_abgr,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- asm volatile(
- "movdqa %5,%%xmm3 \n"
- "movdqa %6,%%xmm4 \n"
- "movdqa %7,%%xmm5 \n"
- "sub %1,%2 \n"
-
- LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x00(%0,%4,1),%%xmm7 \n"
- "pavgb %%xmm7,%%xmm0 \n"
- "movdqu 0x10(%0),%%xmm1 \n"
- "movdqu 0x10(%0,%4,1),%%xmm7 \n"
- "pavgb %%xmm7,%%xmm1 \n"
- "movdqu 0x20(%0),%%xmm2 \n"
- "movdqu 0x20(%0,%4,1),%%xmm7 \n"
- "pavgb %%xmm7,%%xmm2 \n"
- "movdqu 0x30(%0),%%xmm6 \n"
- "movdqu 0x30(%0,%4,1),%%xmm7 \n"
- "pavgb %%xmm7,%%xmm6 \n"
-
- "lea 0x40(%0),%0 \n"
- "movdqa %%xmm0,%%xmm7 \n"
- "shufps $0x88,%%xmm1,%%xmm0 \n"
- "shufps $0xdd,%%xmm1,%%xmm7 \n"
- "pavgb %%xmm7,%%xmm0 \n"
- "movdqa %%xmm2,%%xmm7 \n"
- "shufps $0x88,%%xmm6,%%xmm2 \n"
- "shufps $0xdd,%%xmm6,%%xmm7 \n"
- "pavgb %%xmm7,%%xmm2 \n"
- "movdqa %%xmm0,%%xmm1 \n"
- "movdqa %%xmm2,%%xmm6 \n"
- "pmaddubsw %%xmm4,%%xmm0 \n"
- "pmaddubsw %%xmm4,%%xmm2 \n"
- "pmaddubsw %%xmm3,%%xmm1 \n"
- "pmaddubsw %%xmm3,%%xmm6 \n"
- "phaddw %%xmm2,%%xmm0 \n"
- "phaddw %%xmm6,%%xmm1 \n"
- "psraw $0x8,%%xmm0 \n"
- "psraw $0x8,%%xmm1 \n"
- "packsswb %%xmm1,%%xmm0 \n"
- "paddb %%xmm5,%%xmm0 \n"
- "movlps %%xmm0,(%1) \n"
- "movhps %%xmm0,0x00(%1,%2,1) \n"
- "lea 0x8(%1),%1 \n"
- "sub $0x10,%3 \n"
- "jg 1b \n"
- : "+r"(src_abgr), // %0
- "+r"(dst_u), // %1
- "+r"(dst_v), // %2
- "+rm"(width) // %3
- : "r"((intptr_t)(src_stride_abgr)), // %4
- "m"(kABGRToV), // %5
- "m"(kABGRToU), // %6
- "m"(kAddUV128) // %7
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm6", "xmm7");
-}
-
-void RGBAToUVRow_SSSE3(const uint8_t* src_rgba,
- int src_stride_rgba,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width) {
- asm volatile(
- "movdqa %5,%%xmm3 \n"
- "movdqa %6,%%xmm4 \n"
- "movdqa %7,%%xmm5 \n"
- "sub %1,%2 \n"
-
- LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x00(%0,%4,1),%%xmm7 \n"
- "pavgb %%xmm7,%%xmm0 \n"
- "movdqu 0x10(%0),%%xmm1 \n"
- "movdqu 0x10(%0,%4,1),%%xmm7 \n"
- "pavgb %%xmm7,%%xmm1 \n"
- "movdqu 0x20(%0),%%xmm2 \n"
- "movdqu 0x20(%0,%4,1),%%xmm7 \n"
- "pavgb %%xmm7,%%xmm2 \n"
- "movdqu 0x30(%0),%%xmm6 \n"
- "movdqu 0x30(%0,%4,1),%%xmm7 \n"
- "pavgb %%xmm7,%%xmm6 \n"
-
- "lea 0x40(%0),%0 \n"
- "movdqa %%xmm0,%%xmm7 \n"
- "shufps $0x88,%%xmm1,%%xmm0 \n"
- "shufps $0xdd,%%xmm1,%%xmm7 \n"
- "pavgb %%xmm7,%%xmm0 \n"
- "movdqa %%xmm2,%%xmm7 \n"
- "shufps $0x88,%%xmm6,%%xmm2 \n"
- "shufps $0xdd,%%xmm6,%%xmm7 \n"
- "pavgb %%xmm7,%%xmm2 \n"
- "movdqa %%xmm0,%%xmm1 \n"
- "movdqa %%xmm2,%%xmm6 \n"
- "pmaddubsw %%xmm4,%%xmm0 \n"
- "pmaddubsw %%xmm4,%%xmm2 \n"
- "pmaddubsw %%xmm3,%%xmm1 \n"
- "pmaddubsw %%xmm3,%%xmm6 \n"
- "phaddw %%xmm2,%%xmm0 \n"
- "phaddw %%xmm6,%%xmm1 \n"
- "psraw $0x8,%%xmm0 \n"
- "psraw $0x8,%%xmm1 \n"
- "packsswb %%xmm1,%%xmm0 \n"
- "paddb %%xmm5,%%xmm0 \n"
- "movlps %%xmm0,(%1) \n"
- "movhps %%xmm0,0x00(%1,%2,1) \n"
- "lea 0x8(%1),%1 \n"
- "sub $0x10,%3 \n"
- "jg 1b \n"
- : "+r"(src_rgba), // %0
- "+r"(dst_u), // %1
- "+r"(dst_v), // %2
- "+rm"(width) // %3
- : "r"((intptr_t)(src_stride_rgba)), // %4
- "m"(kRGBAToV), // %5
- "m"(kRGBAToU), // %6
- "m"(kAddUV128) // %7
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm6", "xmm7");
-}
-
#if defined(HAS_I422TOARGBROW_SSSE3) || defined(HAS_I422TOARGBROW_AVX2)
// Read 8 UV from 444
@@ -2467,21 +2258,25 @@
"punpcklbw %%xmm4,%%xmm4 \n" \
"lea 0x8(%[y_buf]),%[y_buf] \n"
-// Read 4 YUY2 with 8 Y and update 4 UV to 8 UV.
+// Read 4 YUY2 with 8 Y and upsample 4 UV to 8 UV.
+// xmm6 kShuffleYUY2Y,
+// xmm7 kShuffleYUY2UV
#define READYUY2 \
"movdqu (%[yuy2_buf]),%%xmm4 \n" \
- "pshufb %[kShuffleYUY2Y], %%xmm4 \n" \
- "movdqu (%[yuy2_buf]),%%xmm3 \n" \
- "pshufb %[kShuffleYUY2UV], %%xmm3 \n" \
- "lea 0x10(%[yuy2_buf]),%[yuy2_buf] \n"
+ "lea 0x10(%[yuy2_buf]),%[yuy2_buf] \n" \
+ "movdqa %%xmm4,%%xmm3 \n" \
+ "pshufb %%xmm6,%%xmm4 \n" \
+ "pshufb %%xmm7,%%xmm3 \n"
-// Read 4 UYVY with 8 Y and update 4 UV to 8 UV.
+// Read 4 UYVY with 8 Y and upsample 4 UV to 8 UV.
+// xmm6 kShuffleUYVYY,
+// xmm7 kShuffleUYVYUV
#define READUYVY \
"movdqu (%[uyvy_buf]),%%xmm4 \n" \
- "pshufb %[kShuffleUYVYY], %%xmm4 \n" \
- "movdqu (%[uyvy_buf]),%%xmm3 \n" \
- "pshufb %[kShuffleUYVYUV], %%xmm3 \n" \
- "lea 0x10(%[uyvy_buf]),%[uyvy_buf] \n"
+ "lea 0x10(%[uyvy_buf]),%[uyvy_buf] \n" \
+ "movdqa %%xmm4,%%xmm3 \n" \
+ "pshufb %%xmm6,%%xmm4 \n" \
+ "pshufb %%xmm7,%%xmm3 \n"
// Read 4 UV from P210, upsample to 8 UV
#define READP210 \
@@ -2536,6 +2331,7 @@
#else
#define YUVTORGB_SETUP(yuvconstants)
+
// Convert 8 pixels: 8 UV and 8 Y
#define YUVTORGB16(yuvconstants) \
"pcmpeqb %%xmm0,%%xmm0 \n" \
@@ -2644,7 +2440,7 @@
"pcmpeqb %%xmm5,%%xmm5 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
READYUV444
YUVTORGB(yuvconstants)
STOREARGB
@@ -2669,33 +2465,26 @@
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width) {
- // clang-format off
- asm volatile (
- YUVTORGB_SETUP(yuvconstants)
- "sub %[u_buf],%[v_buf] \n"
+ asm volatile(YUVTORGB_SETUP(
+ yuvconstants) "sub %[u_buf],%[v_buf] \n"
- LABELALIGN
- "1: \n"
- READYUVA444
- YUVTORGB(yuvconstants)
- STOREARGB
- "subl $0x8,%[width] \n"
- "jg 1b \n"
- : [y_buf]"+r"(y_buf), // %[y_buf]
- [u_buf]"+r"(u_buf), // %[u_buf]
- [v_buf]"+r"(v_buf), // %[v_buf]
- [a_buf]"+r"(a_buf), // %[a_buf]
- [dst_argb]"+r"(dst_argb), // %[dst_argb]
+ LABELALIGN "1: \n" READYUVA444 YUVTORGB(yuvconstants)
+ STOREARGB
+ "subl $0x8,%[width] \n"
+ "jg 1b \n"
+ : [y_buf] "+r"(y_buf), // %[y_buf]
+ [u_buf] "+r"(u_buf), // %[u_buf]
+ [v_buf] "+r"(v_buf), // %[v_buf]
+ [a_buf] "+r"(a_buf), // %[a_buf]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
#if defined(__i386__)
- [width]"+m"(width) // %[width]
+ [width] "+m"(width) // %[width]
#else
- [width]"+rm"(width) // %[width]
+ [width] "+rm"(width) // %[width]
#endif
- : [yuvconstants]"r"(yuvconstants) // %[yuvconstants]
- : "memory", "cc", YUVTORGB_REGS
- "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"
- );
- // clang-format on
+ : [yuvconstants] "r"(yuvconstants) // %[yuvconstants]
+ : "memory", "cc", YUVTORGB_REGS "xmm0", "xmm1", "xmm2", "xmm3",
+ "xmm4", "xmm5");
}
#endif // HAS_I444ALPHATOARGBROW_SSSE3
@@ -2712,7 +2501,7 @@
"sub %[u_buf],%[v_buf] \n"
LABELALIGN
- "1: \n"
+ "1: \n"
READYUV422
YUVTORGB(yuvconstants)
STORERGB24
@@ -2748,7 +2537,7 @@
"sub %[u_buf],%[v_buf] \n"
LABELALIGN
- "1: \n"
+ "1: \n"
READYUV444
YUVTORGB(yuvconstants)
STORERGB24
@@ -2783,7 +2572,7 @@
"pcmpeqb %%xmm5,%%xmm5 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
READYUV422
YUVTORGB(yuvconstants)
STOREARGB
@@ -2817,7 +2606,7 @@
"psrlw $6,%%xmm7 \n" // 1023 for max
LABELALIGN
- "1: \n"
+ "1: \n"
READYUV422
YUVTORGB16(yuvconstants)
STOREAR30
@@ -2847,7 +2636,7 @@
"pcmpeqb %%xmm5,%%xmm5 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
READYUV210
YUVTORGB(yuvconstants)
STOREARGB
@@ -2877,7 +2666,7 @@
"pcmpeqb %%xmm5,%%xmm5 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
READYUV212
YUVTORGB(yuvconstants)
STOREARGB
@@ -2912,7 +2701,7 @@
"psrlw $6,%%xmm7 \n" // 1023 for max
LABELALIGN
- "1: \n"
+ "1: \n"
READYUV210
YUVTORGB16(yuvconstants)
STOREAR30
@@ -2947,7 +2736,7 @@
"psrlw $6,%%xmm7 \n" // 1023 for max
LABELALIGN
- "1: \n"
+ "1: \n"
READYUV212
YUVTORGB16(yuvconstants)
STOREAR30
@@ -2977,7 +2766,7 @@
"pcmpeqb %%xmm5,%%xmm5 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
READYUV410
YUVTORGB(yuvconstants)
STOREARGB
@@ -3002,27 +2791,26 @@
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width) {
- asm volatile(
- YUVTORGB_SETUP(
- yuvconstants) "sub %[u_buf],%[v_buf] \n"
+ asm volatile(YUVTORGB_SETUP(
+ yuvconstants) "sub %[u_buf],%[v_buf] \n"
- LABELALIGN "1: \n" READYUVA210
- YUVTORGB(yuvconstants) STOREARGB
- "subl $0x8,%[width] \n"
- "jg 1b \n"
- : [y_buf] "+r"(y_buf), // %[y_buf]
- [u_buf] "+r"(u_buf), // %[u_buf]
- [v_buf] "+r"(v_buf), // %[v_buf]
- [a_buf] "+r"(a_buf),
- [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ LABELALIGN "1: \n" READYUVA210 YUVTORGB(yuvconstants)
+ STOREARGB
+ "subl $0x8,%[width] \n"
+ "jg 1b \n"
+ : [y_buf] "+r"(y_buf), // %[y_buf]
+ [u_buf] "+r"(u_buf), // %[u_buf]
+ [v_buf] "+r"(v_buf), // %[v_buf]
+ [a_buf] "+r"(a_buf),
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
#if defined(__i386__)
- [width] "+m"(width) // %[width]
+ [width] "+m"(width) // %[width]
#else
- [width] "+rm"(width) // %[width]
+ [width] "+rm"(width) // %[width]
#endif
- : [yuvconstants] "r"(yuvconstants) // %[yuvconstants]
- : "memory", "cc", YUVTORGB_REGS "xmm0", "xmm1", "xmm2", "xmm3", "xmm4",
- "xmm5");
+ : [yuvconstants] "r"(yuvconstants) // %[yuvconstants]
+ : "memory", "cc", YUVTORGB_REGS "xmm0", "xmm1", "xmm2", "xmm3",
+ "xmm4", "xmm5");
}
#endif
@@ -3035,32 +2823,26 @@
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width) {
- // clang-format off
- asm volatile(
- YUVTORGB_SETUP(yuvconstants)
- "sub %[u_buf],%[v_buf] \n"
+ asm volatile(YUVTORGB_SETUP(
+ yuvconstants) "sub %[u_buf],%[v_buf] \n"
- LABELALIGN
- "1: \n"
- READYUVA410
- YUVTORGB(yuvconstants)
- STOREARGB
- "subl $0x8,%[width] \n"
- "jg 1b \n"
- : [y_buf] "+r"(y_buf), // %[y_buf]
- [u_buf] "+r"(u_buf), // %[u_buf]
- [v_buf] "+r"(v_buf), // %[v_buf]
- [a_buf] "+r"(a_buf),
- [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ LABELALIGN "1: \n" READYUVA410 YUVTORGB(yuvconstants)
+ STOREARGB
+ "subl $0x8,%[width] \n"
+ "jg 1b \n"
+ : [y_buf] "+r"(y_buf), // %[y_buf]
+ [u_buf] "+r"(u_buf), // %[u_buf]
+ [v_buf] "+r"(v_buf), // %[v_buf]
+ [a_buf] "+r"(a_buf),
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
#if defined(__i386__)
- [width] "+m"(width) // %[width]
+ [width] "+m"(width) // %[width]
#else
- [width] "+rm"(width) // %[width]
+ [width] "+rm"(width) // %[width]
#endif
- : [yuvconstants] "r"(yuvconstants) // %[yuvconstants]
- : "memory", "cc", YUVTORGB_REGS "xmm0", "xmm1", "xmm2", "xmm3", "xmm4",
- "xmm5");
- // clang-format on
+ : [yuvconstants] "r"(yuvconstants) // %[yuvconstants]
+ : "memory", "cc", YUVTORGB_REGS "xmm0", "xmm1", "xmm2", "xmm3",
+ "xmm4", "xmm5");
}
#endif
@@ -3082,7 +2864,7 @@
"psrlw $6,%%xmm7 \n" // 1023 for max
LABELALIGN
- "1: \n"
+ "1: \n"
READYUV410
YUVTORGB16(yuvconstants)
STOREAR30
@@ -3107,33 +2889,26 @@
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width) {
- // clang-format off
- asm volatile (
- YUVTORGB_SETUP(yuvconstants)
- "sub %[u_buf],%[v_buf] \n"
+ asm volatile(YUVTORGB_SETUP(
+ yuvconstants) "sub %[u_buf],%[v_buf] \n"
- LABELALIGN
- "1: \n"
- READYUVA422
- YUVTORGB(yuvconstants)
- STOREARGB
- "subl $0x8,%[width] \n"
- "jg 1b \n"
- : [y_buf]"+r"(y_buf), // %[y_buf]
- [u_buf]"+r"(u_buf), // %[u_buf]
- [v_buf]"+r"(v_buf), // %[v_buf]
- [a_buf]"+r"(a_buf), // %[a_buf]
- [dst_argb]"+r"(dst_argb), // %[dst_argb]
+ LABELALIGN "1: \n" READYUVA422 YUVTORGB(yuvconstants)
+ STOREARGB
+ "subl $0x8,%[width] \n"
+ "jg 1b \n"
+ : [y_buf] "+r"(y_buf), // %[y_buf]
+ [u_buf] "+r"(u_buf), // %[u_buf]
+ [v_buf] "+r"(v_buf), // %[v_buf]
+ [a_buf] "+r"(a_buf), // %[a_buf]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
#if defined(__i386__)
- [width]"+m"(width) // %[width]
+ [width] "+m"(width) // %[width]
#else
- [width]"+rm"(width) // %[width]
+ [width] "+rm"(width) // %[width]
#endif
- : [yuvconstants]"r"(yuvconstants) // %[yuvconstants]
- : "memory", "cc", YUVTORGB_REGS
- "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"
- );
- // clang-format on
+ : [yuvconstants] "r"(yuvconstants) // %[yuvconstants]
+ : "memory", "cc", YUVTORGB_REGS "xmm0", "xmm1", "xmm2", "xmm3",
+ "xmm4", "xmm5");
}
#endif // HAS_I422ALPHATOARGBROW_SSSE3
@@ -3142,27 +2917,20 @@
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width) {
- // clang-format off
- asm volatile (
- YUVTORGB_SETUP(yuvconstants)
- "pcmpeqb %%xmm5,%%xmm5 \n"
+ asm volatile(YUVTORGB_SETUP(
+ yuvconstants) "pcmpeqb %%xmm5,%%xmm5 \n"
- LABELALIGN
- "1: \n"
- READNV12
- YUVTORGB(yuvconstants)
- STOREARGB
- "sub $0x8,%[width] \n"
- "jg 1b \n"
- : [y_buf]"+r"(y_buf), // %[y_buf]
- [uv_buf]"+r"(uv_buf), // %[uv_buf]
- [dst_argb]"+r"(dst_argb), // %[dst_argb]
- [width]"+rm"(width) // %[width]
- : [yuvconstants]"r"(yuvconstants) // %[yuvconstants]
- : "memory", "cc", YUVTORGB_REGS
- "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"
- );
- // clang-format on
+ LABELALIGN "1: \n" READNV12 YUVTORGB(yuvconstants)
+ STOREARGB
+ "sub $0x8,%[width] \n"
+ "jg 1b \n"
+ : [y_buf] "+r"(y_buf), // %[y_buf]
+ [uv_buf] "+r"(uv_buf), // %[uv_buf]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+rm"(width) // %[width]
+ : [yuvconstants] "r"(yuvconstants) // %[yuvconstants]
+ : "memory", "cc", YUVTORGB_REGS "xmm0", "xmm1", "xmm2", "xmm3",
+ "xmm4", "xmm5");
}
void OMITFP NV21ToARGBRow_SSSE3(const uint8_t* y_buf,
@@ -3170,84 +2938,63 @@
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width) {
- // clang-format off
- asm volatile (
- YUVTORGB_SETUP(yuvconstants)
- "pcmpeqb %%xmm5,%%xmm5 \n"
+ asm volatile(YUVTORGB_SETUP(
+ yuvconstants) "pcmpeqb %%xmm5,%%xmm5 \n"
- LABELALIGN
- "1: \n"
- READNV21
- YUVTORGB(yuvconstants)
- STOREARGB
- "sub $0x8,%[width] \n"
- "jg 1b \n"
- : [y_buf]"+r"(y_buf), // %[y_buf]
- [vu_buf]"+r"(vu_buf), // %[vu_buf]
- [dst_argb]"+r"(dst_argb), // %[dst_argb]
- [width]"+rm"(width) // %[width]
- : [yuvconstants]"r"(yuvconstants), // %[yuvconstants]
- [kShuffleNV21]"m"(kShuffleNV21)
- : "memory", "cc", YUVTORGB_REGS
- "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"
- );
- // clang-format on
+ LABELALIGN "1: \n" READNV21 YUVTORGB(yuvconstants)
+ STOREARGB
+ "sub $0x8,%[width] \n"
+ "jg 1b \n"
+ : [y_buf] "+r"(y_buf), // %[y_buf]
+ [vu_buf] "+r"(vu_buf), // %[vu_buf]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+rm"(width) // %[width]
+ : [yuvconstants] "r"(yuvconstants), // %[yuvconstants]
+ [kShuffleNV21] "m"(kShuffleNV21)
+ : "memory", "cc", YUVTORGB_REGS "xmm0", "xmm1", "xmm2", "xmm3",
+ "xmm4", "xmm5");
}
void OMITFP YUY2ToARGBRow_SSSE3(const uint8_t* yuy2_buf,
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width) {
- // clang-format off
- asm volatile (
- YUVTORGB_SETUP(yuvconstants)
- "pcmpeqb %%xmm5,%%xmm5 \n"
+ asm volatile(
+ "movdqa %[kShuffleYUY2Y],%%xmm6 \n"
+ "movdqa %[kShuffleYUY2UV],%%xmm7 \n" YUVTORGB_SETUP(
+ yuvconstants) "pcmpeqb %%xmm5,%%xmm5 \n"
- LABELALIGN
- "1: \n"
- READYUY2
- YUVTORGB(yuvconstants)
- STOREARGB
+ LABELALIGN "1: \n" READYUY2 YUVTORGB(yuvconstants) STOREARGB
"sub $0x8,%[width] \n"
"jg 1b \n"
- : [yuy2_buf]"+r"(yuy2_buf), // %[yuy2_buf]
- [dst_argb]"+r"(dst_argb), // %[dst_argb]
- [width]"+rm"(width) // %[width]
- : [yuvconstants]"r"(yuvconstants), // %[yuvconstants]
- [kShuffleYUY2Y]"m"(kShuffleYUY2Y),
- [kShuffleYUY2UV]"m"(kShuffleYUY2UV)
- : "memory", "cc", YUVTORGB_REGS
- "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"
- );
- // clang-format on
+ : [yuy2_buf] "+r"(yuy2_buf), // %[yuy2_buf]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+rm"(width) // %[width]
+ : [yuvconstants] "r"(yuvconstants), // %[yuvconstants]
+ [kShuffleYUY2Y] "m"(kShuffleYUY2Y), [kShuffleYUY2UV] "m"(kShuffleYUY2UV)
+ : "memory", "cc", YUVTORGB_REGS "xmm0", "xmm1", "xmm2", "xmm3", "xmm4",
+ "xmm5", "xmm6", "xmm7");
}
void OMITFP UYVYToARGBRow_SSSE3(const uint8_t* uyvy_buf,
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width) {
- // clang-format off
- asm volatile (
- YUVTORGB_SETUP(yuvconstants)
- "pcmpeqb %%xmm5,%%xmm5 \n"
+ asm volatile(
+ "movdqa %[kShuffleUYVYY],%%xmm6 \n"
+ "movdqa %[kShuffleUYVYUV],%%xmm7 \n" YUVTORGB_SETUP(
+ yuvconstants) "pcmpeqb %%xmm5,%%xmm5 \n"
- LABELALIGN
- "1: \n"
- READUYVY
- YUVTORGB(yuvconstants)
- STOREARGB
+ LABELALIGN "1: \n" READUYVY YUVTORGB(yuvconstants) STOREARGB
"sub $0x8,%[width] \n"
"jg 1b \n"
- : [uyvy_buf]"+r"(uyvy_buf), // %[uyvy_buf]
- [dst_argb]"+r"(dst_argb), // %[dst_argb]
- [width]"+rm"(width) // %[width]
- : [yuvconstants]"r"(yuvconstants), // %[yuvconstants]
- [kShuffleUYVYY]"m"(kShuffleUYVYY),
- [kShuffleUYVYUV]"m"(kShuffleUYVYUV)
- : "memory", "cc", YUVTORGB_REGS
- "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"
- );
- // clang-format on
+ : [uyvy_buf] "+r"(uyvy_buf), // %[uyvy_buf]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+rm"(width) // %[width]
+ : [yuvconstants] "r"(yuvconstants), // %[yuvconstants]
+ [kShuffleUYVYY] "m"(kShuffleUYVYY), [kShuffleUYVYUV] "m"(kShuffleUYVYUV)
+ : "memory", "cc", YUVTORGB_REGS "xmm0", "xmm1", "xmm2", "xmm3", "xmm4",
+ "xmm5");
}
void OMITFP P210ToARGBRow_SSSE3(const uint16_t* y_buf,
@@ -3255,21 +3002,20 @@
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width) {
- asm volatile(
- YUVTORGB_SETUP(
- yuvconstants) "pcmpeqb %%xmm5,%%xmm5 \n"
+ asm volatile(YUVTORGB_SETUP(
+ yuvconstants) "pcmpeqb %%xmm5,%%xmm5 \n"
- LABELALIGN "1: \n" READP210
- YUVTORGB(yuvconstants) STOREARGB
- "sub $0x8,%[width] \n"
- "jg 1b \n"
- : [y_buf] "+r"(y_buf), // %[y_buf]
- [uv_buf] "+r"(uv_buf), // %[u_buf]
- [dst_argb] "+r"(dst_argb), // %[dst_argb]
- [width] "+rm"(width) // %[width]
- : [yuvconstants] "r"(yuvconstants) // %[yuvconstants]
- : "memory", "cc", YUVTORGB_REGS "xmm0", "xmm1", "xmm2", "xmm3", "xmm4",
- "xmm5");
+ LABELALIGN "1: \n" READP210 YUVTORGB(yuvconstants)
+ STOREARGB
+ "sub $0x8,%[width] \n"
+ "jg 1b \n"
+ : [y_buf] "+r"(y_buf), // %[y_buf]
+ [uv_buf] "+r"(uv_buf), // %[u_buf]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+rm"(width) // %[width]
+ : [yuvconstants] "r"(yuvconstants) // %[yuvconstants]
+ : "memory", "cc", YUVTORGB_REGS "xmm0", "xmm1", "xmm2", "xmm3",
+ "xmm4", "xmm5");
}
void OMITFP P410ToARGBRow_SSSE3(const uint16_t* y_buf,
@@ -3277,21 +3023,20 @@
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width) {
- asm volatile(
- YUVTORGB_SETUP(
- yuvconstants) "pcmpeqb %%xmm5,%%xmm5 \n"
+ asm volatile(YUVTORGB_SETUP(
+ yuvconstants) "pcmpeqb %%xmm5,%%xmm5 \n"
- LABELALIGN "1: \n" READP410
- YUVTORGB(yuvconstants) STOREARGB
- "sub $0x8,%[width] \n"
- "jg 1b \n"
- : [y_buf] "+r"(y_buf), // %[y_buf]
- [uv_buf] "+r"(uv_buf), // %[u_buf]
- [dst_argb] "+r"(dst_argb), // %[dst_argb]
- [width] "+rm"(width) // %[width]
- : [yuvconstants] "r"(yuvconstants) // %[yuvconstants]
- : "memory", "cc", YUVTORGB_REGS "xmm0", "xmm1", "xmm2", "xmm3", "xmm4",
- "xmm5");
+ LABELALIGN "1: \n" READP410 YUVTORGB(yuvconstants)
+ STOREARGB
+ "sub $0x8,%[width] \n"
+ "jg 1b \n"
+ : [y_buf] "+r"(y_buf), // %[y_buf]
+ [uv_buf] "+r"(uv_buf), // %[u_buf]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+rm"(width) // %[width]
+ : [yuvconstants] "r"(yuvconstants) // %[yuvconstants]
+ : "memory", "cc", YUVTORGB_REGS "xmm0", "xmm1", "xmm2", "xmm3",
+ "xmm4", "xmm5");
}
void OMITFP P210ToAR30Row_SSSE3(const uint16_t* y_buf,
@@ -3309,7 +3054,7 @@
"psrlw $6,%%xmm7 \n" // 1023 for max
LABELALIGN
- "1: \n"
+ "1: \n"
READP210
YUVTORGB16(yuvconstants)
STOREAR30
@@ -3340,7 +3085,7 @@
"psrlw $6,%%xmm7 \n" // 1023 for max
LABELALIGN
- "1: \n"
+ "1: \n"
READP410
YUVTORGB16(yuvconstants)
STOREAR30
@@ -3368,7 +3113,7 @@
"pcmpeqb %%xmm5,%%xmm5 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
READYUV422
YUVTORGB(yuvconstants)
STORERGBA
@@ -3422,7 +3167,7 @@
"vpunpcklbw %%zmm1,%%zmm3,%%zmm3 \n" \
"vpermq $0xd8,%%zmm3,%%zmm3 \n" \
"vpunpcklwd %%zmm3,%%zmm3,%%zmm3 \n" \
- "vmovdqu8 (%[y_buf]),%%ymm4 \n" \
+ "vmovups (%[y_buf]),%%ymm4 \n" \
"vpermq %%zmm4,%%zmm17,%%zmm4 \n" \
"vpermq $0xd8,%%zmm4,%%zmm4 \n" \
"vpunpcklbw %%zmm4,%%zmm4,%%zmm4 \n" \
@@ -3598,19 +3343,21 @@
"lea 0x20(%[y_buf]),%[y_buf] \n"
// Read 8 YUY2 with 16 Y and upsample 8 UV to 16 UV.
+// ymm6 kShuffleYUY2Y,
+// ymm7 kShuffleYUY2UV
#define READYUY2_AVX2 \
- "vmovdqu (%[yuy2_buf]),%%ymm4 \n" \
- "vpshufb %[kShuffleYUY2Y], %%ymm4, %%ymm4 \n" \
- "vmovdqu (%[yuy2_buf]),%%ymm3 \n" \
- "vpshufb %[kShuffleYUY2UV], %%ymm3, %%ymm3 \n" \
+ "vmovdqu (%[yuy2_buf]),%%ymm1 \n" \
+ "vpshufb %%ymm6,%%ymm1,%%ymm4 \n" \
+ "vpshufb %%ymm7,%%ymm1,%%ymm3 \n" \
"lea 0x20(%[yuy2_buf]),%[yuy2_buf] \n"
// Read 8 UYVY with 16 Y and upsample 8 UV to 16 UV.
+// ymm6 kShuffleUYVYY,
+// ymm7 kShuffleUYVYUV
#define READUYVY_AVX2 \
- "vmovdqu (%[uyvy_buf]),%%ymm4 \n" \
- "vpshufb %[kShuffleUYVYY], %%ymm4, %%ymm4 \n" \
- "vmovdqu (%[uyvy_buf]),%%ymm3 \n" \
- "vpshufb %[kShuffleUYVYUV], %%ymm3, %%ymm3 \n" \
+ "vmovdqu (%[uyvy_buf]),%%ymm1 \n" \
+ "vpshufb %%ymm6,%%ymm1,%%ymm4 \n" \
+ "vpshufb %%ymm7,%%ymm1,%%ymm3 \n" \
"lea 0x20(%[uyvy_buf]),%[uyvy_buf] \n"
// TODO(fbarchard): Remove broadcastb
@@ -3631,17 +3378,17 @@
"vpbroadcastq %%xmm8, %%zmm8 \n" \
"vpsllw $7,%%xmm13,%%xmm13 \n" \
"vpbroadcastb %%xmm13,%%zmm13 \n" \
- "movq 32(%[yuvconstants]),%%xmm9 \n" \
+ "movq 32(%[yuvconstants]),%%xmm9 \n" \
"vpbroadcastq %%xmm9,%%zmm9 \n" \
- "movq 64(%[yuvconstants]),%%xmm10 \n" \
+ "movq 64(%[yuvconstants]),%%xmm10 \n" \
"vpbroadcastq %%xmm10,%%zmm10 \n" \
- "movq 96(%[yuvconstants]),%%xmm11 \n" \
+ "movq 96(%[yuvconstants]),%%xmm11 \n" \
"vpbroadcastq %%xmm11,%%zmm11 \n" \
- "movq 128(%[yuvconstants]),%%xmm12 \n" \
+ "movq 128(%[yuvconstants]),%%xmm12 \n" \
"vpbroadcastq %%xmm12,%%zmm12 \n" \
- "vmovdqu8 (%[quadsplitperm]),%%zmm16 \n" \
- "vmovdqu8 (%[dquadsplitperm]),%%zmm17 \n" \
- "vmovdqu8 (%[unperm]),%%zmm18 \n"
+ "vmovups (%[quadsplitperm]),%%zmm16 \n" \
+ "vmovups (%[dquadsplitperm]),%%zmm17 \n" \
+ "vmovups (%[unperm]),%%zmm18 \n"
#define YUVTORGB16_AVX2(yuvconstants) \
"vpsubb %%ymm13,%%ymm3,%%ymm3 \n" \
@@ -3731,8 +3478,8 @@
"vpermq %%zmm2,%%zmm18,%%zmm2 \n" \
"vpunpcklwd %%zmm2,%%zmm0,%%zmm1 \n" \
"vpunpckhwd %%zmm2,%%zmm0,%%zmm0 \n" \
- "vmovdqu8 %%zmm1,(%[dst_argb]) \n" \
- "vmovdqu8 %%zmm0,0x40(%[dst_argb]) \n" \
+ "vmovups %%zmm1,(%[dst_argb]) \n" \
+ "vmovups %%zmm0,0x40(%[dst_argb]) \n" \
"lea 0x80(%[dst_argb]), %[dst_argb] \n"
// Store 16 AR30 values.
@@ -3777,13 +3524,13 @@
"vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
READYUV444_AVX2
YUVTORGB_AVX2(yuvconstants)
STOREARGB_AVX2
"sub $0x10,%[width] \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: [y_buf]"+r"(y_buf), // %[y_buf]
[u_buf]"+r"(u_buf), // %[u_buf]
[v_buf]"+r"(v_buf), // %[v_buf]
@@ -3811,14 +3558,14 @@
"vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
READYUV422_AVX2
YUVTORGB_AVX2(yuvconstants)
STOREARGB_AVX2
"sub $0x10,%[width] \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: [y_buf]"+r"(y_buf), // %[y_buf]
[u_buf]"+r"(u_buf), // %[u_buf]
[v_buf]"+r"(v_buf), // %[v_buf]
@@ -3852,14 +3599,14 @@
"vpbroadcastq %%xmm5,%%zmm5 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
READYUV422_AVX512BW
YUVTORGB_AVX512BW(yuvconstants)
STOREARGB_AVX512BW
"sub $0x20,%[width] \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: [y_buf]"+r"(y_buf), // %[y_buf]
[u_buf]"+r"(u_buf), // %[u_buf]
[v_buf]"+r"(v_buf), // %[v_buf]
@@ -3895,14 +3642,14 @@
"vpsrlw $6,%%ymm7,%%ymm7 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
READYUV422_AVX2
YUVTORGB16_AVX2(yuvconstants)
STOREAR30_AVX2
"sub $0x10,%[width] \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: [y_buf]"+r"(y_buf), // %[y_buf]
[u_buf]"+r"(u_buf), // %[u_buf]
[v_buf]"+r"(v_buf), // %[v_buf]
@@ -3930,14 +3677,14 @@
"vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
READYUV210_AVX2
YUVTORGB_AVX2(yuvconstants)
STOREARGB_AVX2
"sub $0x10,%[width] \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: [y_buf]"+r"(y_buf), // %[y_buf]
[u_buf]"+r"(u_buf), // %[u_buf]
[v_buf]"+r"(v_buf), // %[v_buf]
@@ -3965,14 +3712,14 @@
"vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
READYUV212_AVX2
YUVTORGB_AVX2(yuvconstants)
STOREARGB_AVX2
"sub $0x10,%[width] \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: [y_buf]"+r"(y_buf), // %[y_buf]
[u_buf]"+r"(u_buf), // %[u_buf]
[v_buf]"+r"(v_buf), // %[v_buf]
@@ -4005,14 +3752,14 @@
"vpsrlw $6,%%ymm7,%%ymm7 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
READYUV210_AVX2
YUVTORGB16_AVX2(yuvconstants)
STOREAR30_AVX2
"sub $0x10,%[width] \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: [y_buf]"+r"(y_buf), // %[y_buf]
[u_buf]"+r"(u_buf), // %[u_buf]
[v_buf]"+r"(v_buf), // %[v_buf]
@@ -4045,14 +3792,14 @@
"vpsrlw $6,%%ymm7,%%ymm7 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
READYUV212_AVX2
YUVTORGB16_AVX2(yuvconstants)
STOREAR30_AVX2
"sub $0x10,%[width] \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: [y_buf]"+r"(y_buf), // %[y_buf]
[u_buf]"+r"(u_buf), // %[u_buf]
[v_buf]"+r"(v_buf), // %[v_buf]
@@ -4080,13 +3827,13 @@
"vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
READYUV410_AVX2
YUVTORGB_AVX2(yuvconstants)
STOREARGB_AVX2
"sub $0x10,%[width] \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: [y_buf]"+r"(y_buf), // %[y_buf]
[u_buf]"+r"(u_buf), // %[u_buf]
@@ -4110,29 +3857,28 @@
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width) {
- asm volatile(
- YUVTORGB_SETUP_AVX2(
- yuvconstants) "sub %[u_buf],%[v_buf] \n"
+ asm volatile(YUVTORGB_SETUP_AVX2(
+ yuvconstants) "sub %[u_buf],%[v_buf] \n"
- LABELALIGN "1: \n" READYUVA210_AVX2
- YUVTORGB_AVX2(yuvconstants) STOREARGB_AVX2
- "subl $0x10,%[width] \n"
- "jg 1b \n"
- "vzeroupper \n"
+ LABELALIGN "1: \n" READYUVA210_AVX2 YUVTORGB_AVX2(
+ yuvconstants) STOREARGB_AVX2
+ "subl $0x10,%[width] \n"
+ "jg 1b \n"
+ "vzeroupper \n"
- : [y_buf] "+r"(y_buf), // %[y_buf]
- [u_buf] "+r"(u_buf), // %[u_buf]
- [v_buf] "+r"(v_buf), // %[v_buf]
- [a_buf] "+r"(a_buf), // %[a_buf]
- [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ : [y_buf] "+r"(y_buf), // %[y_buf]
+ [u_buf] "+r"(u_buf), // %[u_buf]
+ [v_buf] "+r"(v_buf), // %[v_buf]
+ [a_buf] "+r"(a_buf), // %[a_buf]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
#if defined(__i386__)
- [width] "+m"(width) // %[width]
+ [width] "+m"(width) // %[width]
#else
- [width] "+rm"(width) // %[width]
+ [width] "+rm"(width) // %[width]
#endif
- : [yuvconstants] "r"(yuvconstants) // %[yuvconstants]
- : "memory", "cc", YUVTORGB_REGS_AVX2 "xmm0", "xmm1", "xmm2", "xmm3",
- "xmm4", "xmm5");
+ : [yuvconstants] "r"(yuvconstants) // %[yuvconstants]
+ : "memory", "cc", YUVTORGB_REGS_AVX2 "xmm0", "xmm1", "xmm2",
+ "xmm3", "xmm4", "xmm5");
}
#endif // HAS_I210TOARGBROW_AVX2
@@ -4146,29 +3892,28 @@
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width) {
- asm volatile(
- YUVTORGB_SETUP_AVX2(
- yuvconstants) "sub %[u_buf],%[v_buf] \n"
+ asm volatile(YUVTORGB_SETUP_AVX2(
+ yuvconstants) "sub %[u_buf],%[v_buf] \n"
- LABELALIGN "1: \n" READYUVA410_AVX2
- YUVTORGB_AVX2(yuvconstants) STOREARGB_AVX2
- "subl $0x10,%[width] \n"
- "jg 1b \n"
- "vzeroupper \n"
+ LABELALIGN "1: \n" READYUVA410_AVX2 YUVTORGB_AVX2(
+ yuvconstants) STOREARGB_AVX2
+ "subl $0x10,%[width] \n"
+ "jg 1b \n"
+ "vzeroupper \n"
- : [y_buf] "+r"(y_buf), // %[y_buf]
- [u_buf] "+r"(u_buf), // %[u_buf]
- [v_buf] "+r"(v_buf), // %[v_buf]
- [a_buf] "+r"(a_buf), // %[a_buf]
- [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ : [y_buf] "+r"(y_buf), // %[y_buf]
+ [u_buf] "+r"(u_buf), // %[u_buf]
+ [v_buf] "+r"(v_buf), // %[v_buf]
+ [a_buf] "+r"(a_buf), // %[a_buf]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
#if defined(__i386__)
- [width] "+m"(width) // %[width]
+ [width] "+m"(width) // %[width]
#else
- [width] "+rm"(width) // %[width]
+ [width] "+rm"(width) // %[width]
#endif
- : [yuvconstants] "r"(yuvconstants) // %[yuvconstants]
- : "memory", "cc", YUVTORGB_REGS_AVX2 "xmm0", "xmm1", "xmm2", "xmm3",
- "xmm4", "xmm5");
+ : [yuvconstants] "r"(yuvconstants) // %[yuvconstants]
+ : "memory", "cc", YUVTORGB_REGS_AVX2 "xmm0", "xmm1", "xmm2",
+ "xmm3", "xmm4", "xmm5");
}
#endif // HAS_I410TOARGBROW_AVX2
@@ -4192,14 +3937,14 @@
"vpsrlw $6,%%ymm7,%%ymm7 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
READYUV410_AVX2
YUVTORGB16_AVX2(yuvconstants)
STOREAR30_AVX2
"sub $0x10,%[width] \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: [y_buf]"+r"(y_buf), // %[y_buf]
[u_buf]"+r"(u_buf), // %[u_buf]
[v_buf]"+r"(v_buf), // %[v_buf]
@@ -4222,34 +3967,27 @@
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width) {
- // clang-format off
- asm volatile (
- YUVTORGB_SETUP_AVX2(yuvconstants)
- "sub %[u_buf],%[v_buf] \n"
+ asm volatile(YUVTORGB_SETUP_AVX2(
+ yuvconstants) "sub %[u_buf],%[v_buf] \n"
- LABELALIGN
- "1: \n"
- READYUVA444_AVX2
- YUVTORGB_AVX2(yuvconstants)
- STOREARGB_AVX2
- "subl $0x10,%[width] \n"
- "jg 1b \n"
- "vzeroupper \n"
- : [y_buf]"+r"(y_buf), // %[y_buf]
- [u_buf]"+r"(u_buf), // %[u_buf]
- [v_buf]"+r"(v_buf), // %[v_buf]
- [a_buf]"+r"(a_buf), // %[a_buf]
- [dst_argb]"+r"(dst_argb), // %[dst_argb]
+ LABELALIGN "1: \n" READYUVA444_AVX2 YUVTORGB_AVX2(
+ yuvconstants) STOREARGB_AVX2
+ "subl $0x10,%[width] \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : [y_buf] "+r"(y_buf), // %[y_buf]
+ [u_buf] "+r"(u_buf), // %[u_buf]
+ [v_buf] "+r"(v_buf), // %[v_buf]
+ [a_buf] "+r"(a_buf), // %[a_buf]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
#if defined(__i386__)
- [width]"+m"(width) // %[width]
+ [width] "+m"(width) // %[width]
#else
- [width]"+rm"(width) // %[width]
+ [width] "+rm"(width) // %[width]
#endif
- : [yuvconstants]"r"(yuvconstants) // %[yuvconstants]
- : "memory", "cc", YUVTORGB_REGS_AVX2
- "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"
- );
- // clang-format on
+ : [yuvconstants] "r"(yuvconstants) // %[yuvconstants]
+ : "memory", "cc", YUVTORGB_REGS_AVX2 "xmm0", "xmm1", "xmm2",
+ "xmm3", "xmm4", "xmm5");
}
#endif // HAS_I444ALPHATOARGBROW_AVX2
@@ -4263,34 +4001,27 @@
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width) {
- // clang-format off
- asm volatile (
- YUVTORGB_SETUP_AVX2(yuvconstants)
- "sub %[u_buf],%[v_buf] \n"
+ asm volatile(YUVTORGB_SETUP_AVX2(
+ yuvconstants) "sub %[u_buf],%[v_buf] \n"
- LABELALIGN
- "1: \n"
- READYUVA422_AVX2
- YUVTORGB_AVX2(yuvconstants)
- STOREARGB_AVX2
- "subl $0x10,%[width] \n"
- "jg 1b \n"
- "vzeroupper \n"
- : [y_buf]"+r"(y_buf), // %[y_buf]
- [u_buf]"+r"(u_buf), // %[u_buf]
- [v_buf]"+r"(v_buf), // %[v_buf]
- [a_buf]"+r"(a_buf), // %[a_buf]
- [dst_argb]"+r"(dst_argb), // %[dst_argb]
+ LABELALIGN "1: \n" READYUVA422_AVX2 YUVTORGB_AVX2(
+ yuvconstants) STOREARGB_AVX2
+ "subl $0x10,%[width] \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : [y_buf] "+r"(y_buf), // %[y_buf]
+ [u_buf] "+r"(u_buf), // %[u_buf]
+ [v_buf] "+r"(v_buf), // %[v_buf]
+ [a_buf] "+r"(a_buf), // %[a_buf]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
#if defined(__i386__)
- [width]"+m"(width) // %[width]
+ [width] "+m"(width) // %[width]
#else
- [width]"+rm"(width) // %[width]
+ [width] "+rm"(width) // %[width]
#endif
- : [yuvconstants]"r"(yuvconstants) // %[yuvconstants]
- : "memory", "cc", YUVTORGB_REGS_AVX2
- "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"
- );
- // clang-format on
+ : [yuvconstants] "r"(yuvconstants) // %[yuvconstants]
+ : "memory", "cc", YUVTORGB_REGS_AVX2 "xmm0", "xmm1", "xmm2",
+ "xmm3", "xmm4", "xmm5");
}
#endif // HAS_I422ALPHATOARGBROW_AVX2
@@ -4309,7 +4040,7 @@
"vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
READYUV422_AVX2
YUVTORGB_AVX2(yuvconstants)
@@ -4346,28 +4077,21 @@
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width) {
- // clang-format off
- asm volatile (
- YUVTORGB_SETUP_AVX2(yuvconstants)
- "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
+ asm volatile(YUVTORGB_SETUP_AVX2(
+ yuvconstants) "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
- LABELALIGN
- "1: \n"
- READNV12_AVX2
- YUVTORGB_AVX2(yuvconstants)
- STOREARGB_AVX2
- "sub $0x10,%[width] \n"
- "jg 1b \n"
- "vzeroupper \n"
- : [y_buf]"+r"(y_buf), // %[y_buf]
- [uv_buf]"+r"(uv_buf), // %[uv_buf]
- [dst_argb]"+r"(dst_argb), // %[dst_argb]
- [width]"+rm"(width) // %[width]
- : [yuvconstants]"r"(yuvconstants) // %[yuvconstants]
- : "memory", "cc", YUVTORGB_REGS_AVX2
- "xmm0", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"
- );
- // clang-format on
+ LABELALIGN "1: \n" READNV12_AVX2 YUVTORGB_AVX2(
+ yuvconstants) STOREARGB_AVX2
+ "sub $0x10,%[width] \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : [y_buf] "+r"(y_buf), // %[y_buf]
+ [uv_buf] "+r"(uv_buf), // %[uv_buf]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+rm"(width) // %[width]
+ : [yuvconstants] "r"(yuvconstants) // %[yuvconstants]
+ : "memory", "cc", YUVTORGB_REGS_AVX2 "xmm0", "xmm0", "xmm1",
+ "xmm2", "xmm3", "xmm4", "xmm5");
}
#endif // HAS_NV12TOARGBROW_AVX2
@@ -4379,29 +4103,22 @@
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width) {
- // clang-format off
- asm volatile (
- YUVTORGB_SETUP_AVX2(yuvconstants)
- "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
+ asm volatile(YUVTORGB_SETUP_AVX2(
+ yuvconstants) "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
- LABELALIGN
- "1: \n"
- READNV21_AVX2
- YUVTORGB_AVX2(yuvconstants)
- STOREARGB_AVX2
- "sub $0x10,%[width] \n"
- "jg 1b \n"
- "vzeroupper \n"
- : [y_buf]"+r"(y_buf), // %[y_buf]
- [vu_buf]"+r"(vu_buf), // %[vu_buf]
- [dst_argb]"+r"(dst_argb), // %[dst_argb]
- [width]"+rm"(width) // %[width]
- : [yuvconstants]"r"(yuvconstants), // %[yuvconstants]
- [kShuffleNV21]"m"(kShuffleNV21)
- : "memory", "cc", YUVTORGB_REGS_AVX2
- "xmm0", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"
- );
- // clang-format on
+ LABELALIGN "1: \n" READNV21_AVX2 YUVTORGB_AVX2(
+ yuvconstants) STOREARGB_AVX2
+ "sub $0x10,%[width] \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : [y_buf] "+r"(y_buf), // %[y_buf]
+ [vu_buf] "+r"(vu_buf), // %[vu_buf]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+rm"(width) // %[width]
+ : [yuvconstants] "r"(yuvconstants), // %[yuvconstants]
+ [kShuffleNV21] "m"(kShuffleNV21)
+ : "memory", "cc", YUVTORGB_REGS_AVX2 "xmm0", "xmm0", "xmm1",
+ "xmm2", "xmm3", "xmm4", "xmm5");
}
#endif // HAS_NV21TOARGBROW_AVX2
@@ -4412,29 +4129,23 @@
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width) {
- // clang-format off
- asm volatile (
- YUVTORGB_SETUP_AVX2(yuvconstants)
- "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
+ asm volatile(
+ "vbroadcastf128 %[kShuffleYUY2Y],%%ymm6 \n"
+ "vbroadcastf128 %[kShuffleYUY2UV],%%ymm7 \n" YUVTORGB_SETUP_AVX2(
+ yuvconstants) "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
- LABELALIGN
- "1: \n"
- READYUY2_AVX2
- YUVTORGB_AVX2(yuvconstants)
- STOREARGB_AVX2
+ LABELALIGN "1: \n" READYUY2_AVX2 YUVTORGB_AVX2(yuvconstants)
+ STOREARGB_AVX2
"sub $0x10,%[width] \n"
"jg 1b \n"
- "vzeroupper \n"
- : [yuy2_buf]"+r"(yuy2_buf), // %[yuy2_buf]
- [dst_argb]"+r"(dst_argb), // %[dst_argb]
- [width]"+rm"(width) // %[width]
- : [yuvconstants]"r"(yuvconstants), // %[yuvconstants]
- [kShuffleYUY2Y]"m"(kShuffleYUY2Y),
- [kShuffleYUY2UV]"m"(kShuffleYUY2UV)
- : "memory", "cc", YUVTORGB_REGS_AVX2
- "xmm0", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"
- );
- // clang-format on
+ "vzeroupper \n"
+ : [yuy2_buf] "+r"(yuy2_buf), // %[yuy2_buf]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+rm"(width) // %[width]
+ : [yuvconstants] "r"(yuvconstants), // %[yuvconstants]
+ [kShuffleYUY2Y] "m"(kShuffleYUY2Y), [kShuffleYUY2UV] "m"(kShuffleYUY2UV)
+ : "memory", "cc", YUVTORGB_REGS_AVX2 "xmm0", "xmm0", "xmm1", "xmm2",
+ "xmm3", "xmm4", "xmm5", "xmm6", "xmm7");
}
#endif // HAS_YUY2TOARGBROW_AVX2
@@ -4445,29 +4156,23 @@
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width) {
- // clang-format off
- asm volatile (
- YUVTORGB_SETUP_AVX2(yuvconstants)
- "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
+ asm volatile(
+ "vbroadcastf128 %[kShuffleUYVYY],%%ymm6 \n"
+ "vbroadcastf128 %[kShuffleUYVYUV],%%ymm7 \n" YUVTORGB_SETUP_AVX2(
+ yuvconstants) "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
- LABELALIGN
- "1: \n"
- READUYVY_AVX2
- YUVTORGB_AVX2(yuvconstants)
- STOREARGB_AVX2
+ LABELALIGN "1: \n" READUYVY_AVX2 YUVTORGB_AVX2(yuvconstants)
+ STOREARGB_AVX2
"sub $0x10,%[width] \n"
"jg 1b \n"
- "vzeroupper \n"
- : [uyvy_buf]"+r"(uyvy_buf), // %[uyvy_buf]
- [dst_argb]"+r"(dst_argb), // %[dst_argb]
- [width]"+rm"(width) // %[width]
- : [yuvconstants]"r"(yuvconstants), // %[yuvconstants]
- [kShuffleUYVYY]"m"(kShuffleUYVYY),
- [kShuffleUYVYUV]"m"(kShuffleUYVYUV)
- : "memory", "cc", YUVTORGB_REGS_AVX2
- "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"
- );
- // clang-format on
+ "vzeroupper \n"
+ : [uyvy_buf] "+r"(uyvy_buf), // %[uyvy_buf]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+rm"(width) // %[width]
+ : [yuvconstants] "r"(yuvconstants), // %[yuvconstants]
+ [kShuffleUYVYY] "m"(kShuffleUYVYY), [kShuffleUYVYUV] "m"(kShuffleUYVYUV)
+ : "memory", "cc", YUVTORGB_REGS_AVX2 "xmm0", "xmm1", "xmm2", "xmm3",
+ "xmm4", "xmm5", "xmm6", "xmm7");
}
#endif // HAS_UYVYTOARGBROW_AVX2
@@ -4479,28 +4184,21 @@
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width) {
- // clang-format off
- asm volatile (
- YUVTORGB_SETUP_AVX2(yuvconstants)
- "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
+ asm volatile(YUVTORGB_SETUP_AVX2(
+ yuvconstants) "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
- LABELALIGN
- "1: \n"
- READP210_AVX2
- YUVTORGB_AVX2(yuvconstants)
- STOREARGB_AVX2
- "sub $0x10,%[width] \n"
- "jg 1b \n"
- "vzeroupper \n"
- : [y_buf]"+r"(y_buf), // %[y_buf]
- [uv_buf]"+r"(uv_buf), // %[uv_buf]
- [dst_argb]"+r"(dst_argb), // %[dst_argb]
- [width]"+rm"(width) // %[width]
- : [yuvconstants]"r"(yuvconstants) // %[yuvconstants]
- : "memory", "cc", YUVTORGB_REGS_AVX2
- "xmm0", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"
- );
- // clang-format on
+ LABELALIGN "1: \n" READP210_AVX2 YUVTORGB_AVX2(
+ yuvconstants) STOREARGB_AVX2
+ "sub $0x10,%[width] \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : [y_buf] "+r"(y_buf), // %[y_buf]
+ [uv_buf] "+r"(uv_buf), // %[uv_buf]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+rm"(width) // %[width]
+ : [yuvconstants] "r"(yuvconstants) // %[yuvconstants]
+ : "memory", "cc", YUVTORGB_REGS_AVX2 "xmm0", "xmm0", "xmm1",
+ "xmm2", "xmm3", "xmm4", "xmm5");
}
#endif // HAS_P210TOARGBROW_AVX2
@@ -4512,28 +4210,21 @@
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width) {
- // clang-format off
- asm volatile (
- YUVTORGB_SETUP_AVX2(yuvconstants)
- "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
+ asm volatile(YUVTORGB_SETUP_AVX2(
+ yuvconstants) "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n"
- LABELALIGN
- "1: \n"
- READP410_AVX2
- YUVTORGB_AVX2(yuvconstants)
- STOREARGB_AVX2
- "sub $0x10,%[width] \n"
- "jg 1b \n"
- "vzeroupper \n"
- : [y_buf]"+r"(y_buf), // %[y_buf]
- [uv_buf]"+r"(uv_buf), // %[uv_buf]
- [dst_argb]"+r"(dst_argb), // %[dst_argb]
- [width]"+rm"(width) // %[width]
- : [yuvconstants]"r"(yuvconstants) // %[yuvconstants]
- : "memory", "cc", YUVTORGB_REGS_AVX2
- "xmm0", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"
- );
- // clang-format on
+ LABELALIGN "1: \n" READP410_AVX2 YUVTORGB_AVX2(
+ yuvconstants) STOREARGB_AVX2
+ "sub $0x10,%[width] \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : [y_buf] "+r"(y_buf), // %[y_buf]
+ [uv_buf] "+r"(uv_buf), // %[uv_buf]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+rm"(width) // %[width]
+ : [yuvconstants] "r"(yuvconstants) // %[yuvconstants]
+ : "memory", "cc", YUVTORGB_REGS_AVX2 "xmm0", "xmm0", "xmm1",
+ "xmm2", "xmm3", "xmm4", "xmm5");
}
#endif // HAS_P410TOARGBROW_AVX2
@@ -4555,14 +4246,14 @@
"vpsrlw $6,%%ymm7,%%ymm7 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
READP210_AVX2
YUVTORGB16_AVX2(yuvconstants)
STOREAR30_AVX2
"sub $0x10,%[width] \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: [y_buf]"+r"(y_buf), // %[y_buf]
[uv_buf]"+r"(uv_buf), // %[uv_buf]
[dst_ar30]"+r"(dst_ar30), // %[dst_ar30]
@@ -4592,14 +4283,14 @@
"vpsrlw $6,%%ymm7,%%ymm7 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
READP410_AVX2
YUVTORGB16_AVX2(yuvconstants)
STOREAR30_AVX2
"sub $0x10,%[width] \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: [y_buf]"+r"(y_buf), // %[y_buf]
[uv_buf]"+r"(uv_buf), // %[uv_buf]
[dst_ar30]"+r"(dst_ar30), // %[dst_ar30]
@@ -4623,7 +4314,7 @@
"pslld $0x18,%%xmm4 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
// Step 1: Scale Y contribution to 8 G values. G = (y - 16) * 1.164
"movq (%0),%%xmm0 \n"
"lea 0x8(%0),%0 \n"
@@ -4668,7 +4359,7 @@
"vpslld $0x18,%%ymm4,%%ymm4 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
// Step 1: Scale Y contribution to 16 G values. G = (y - 16) * 1.164
"vmovdqu (%0),%%xmm0 \n"
"lea 0x10(%0),%0 \n"
@@ -4705,48 +4396,44 @@
void MirrorRow_SSSE3(const uint8_t* src, uint8_t* dst, int width) {
intptr_t temp_width = (intptr_t)(width);
- asm volatile(
+ asm volatile("movdqa %3,%%xmm5 \n"
- "movdqa %3,%%xmm5 \n"
-
- LABELALIGN
- "1: \n"
- "movdqu -0x10(%0,%2,1),%%xmm0 \n"
- "pshufb %%xmm5,%%xmm0 \n"
- "movdqu %%xmm0,(%1) \n"
- "lea 0x10(%1),%1 \n"
- "sub $0x10,%2 \n"
- "jg 1b \n"
- : "+r"(src), // %0
- "+r"(dst), // %1
- "+r"(temp_width) // %2
- : "m"(kShuffleMirror) // %3
- : "memory", "cc", "xmm0", "xmm5");
+ LABELALIGN
+ "1: \n"
+ "movdqu -0x10(%0,%2,1),%%xmm0 \n"
+ "pshufb %%xmm5,%%xmm0 \n"
+ "movdqu %%xmm0,(%1) \n"
+ "lea 0x10(%1),%1 \n"
+ "sub $0x10,%2 \n"
+ "jg 1b \n"
+ : "+r"(src), // %0
+ "+r"(dst), // %1
+ "+r"(temp_width) // %2
+ : "m"(kShuffleMirror) // %3
+ : "memory", "cc", "xmm0", "xmm5");
}
#endif // HAS_MIRRORROW_SSSE3
#ifdef HAS_MIRRORROW_AVX2
void MirrorRow_AVX2(const uint8_t* src, uint8_t* dst, int width) {
intptr_t temp_width = (intptr_t)(width);
- asm volatile(
+ asm volatile("vbroadcastf128 %3,%%ymm5 \n"
- "vbroadcastf128 %3,%%ymm5 \n"
-
- LABELALIGN
- "1: \n"
- "vmovdqu -0x20(%0,%2,1),%%ymm0 \n"
- "vpshufb %%ymm5,%%ymm0,%%ymm0 \n"
- "vpermq $0x4e,%%ymm0,%%ymm0 \n"
- "vmovdqu %%ymm0,(%1) \n"
- "lea 0x20(%1),%1 \n"
- "sub $0x20,%2 \n"
- "jg 1b \n"
- "vzeroupper \n"
- : "+r"(src), // %0
- "+r"(dst), // %1
- "+r"(temp_width) // %2
- : "m"(kShuffleMirror) // %3
- : "memory", "cc", "xmm0", "xmm5");
+ LABELALIGN
+ "1: \n"
+ "vmovdqu -0x20(%0,%2,1),%%ymm0 \n"
+ "vpshufb %%ymm5,%%ymm0,%%ymm0 \n"
+ "vpermq $0x4e,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm0,(%1) \n"
+ "lea 0x20(%1),%1 \n"
+ "sub $0x20,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src), // %0
+ "+r"(dst), // %1
+ "+r"(temp_width) // %2
+ : "m"(kShuffleMirror) // %3
+ : "memory", "cc", "xmm0", "xmm5");
}
#endif // HAS_MIRRORROW_AVX2
@@ -4757,48 +4444,44 @@
void MirrorUVRow_SSSE3(const uint8_t* src_uv, uint8_t* dst_uv, int width) {
intptr_t temp_width = (intptr_t)(width);
- asm volatile(
+ asm volatile("movdqa %3,%%xmm5 \n"
- "movdqa %3,%%xmm5 \n"
-
- LABELALIGN
- "1: \n"
- "movdqu -0x10(%0,%2,2),%%xmm0 \n"
- "pshufb %%xmm5,%%xmm0 \n"
- "movdqu %%xmm0,(%1) \n"
- "lea 0x10(%1),%1 \n"
- "sub $0x8,%2 \n"
- "jg 1b \n"
- : "+r"(src_uv), // %0
- "+r"(dst_uv), // %1
- "+r"(temp_width) // %2
- : "m"(kShuffleMirrorUV) // %3
- : "memory", "cc", "xmm0", "xmm5");
+ LABELALIGN
+ "1: \n"
+ "movdqu -0x10(%0,%2,2),%%xmm0 \n"
+ "pshufb %%xmm5,%%xmm0 \n"
+ "movdqu %%xmm0,(%1) \n"
+ "lea 0x10(%1),%1 \n"
+ "sub $0x8,%2 \n"
+ "jg 1b \n"
+ : "+r"(src_uv), // %0
+ "+r"(dst_uv), // %1
+ "+r"(temp_width) // %2
+ : "m"(kShuffleMirrorUV) // %3
+ : "memory", "cc", "xmm0", "xmm5");
}
#endif // HAS_MIRRORUVROW_SSSE3
#ifdef HAS_MIRRORUVROW_AVX2
void MirrorUVRow_AVX2(const uint8_t* src_uv, uint8_t* dst_uv, int width) {
intptr_t temp_width = (intptr_t)(width);
- asm volatile(
+ asm volatile("vbroadcastf128 %3,%%ymm5 \n"
- "vbroadcastf128 %3,%%ymm5 \n"
-
- LABELALIGN
- "1: \n"
- "vmovdqu -0x20(%0,%2,2),%%ymm0 \n"
- "vpshufb %%ymm5,%%ymm0,%%ymm0 \n"
- "vpermq $0x4e,%%ymm0,%%ymm0 \n"
- "vmovdqu %%ymm0,(%1) \n"
- "lea 0x20(%1),%1 \n"
- "sub $0x10,%2 \n"
- "jg 1b \n"
- "vzeroupper \n"
- : "+r"(src_uv), // %0
- "+r"(dst_uv), // %1
- "+r"(temp_width) // %2
- : "m"(kShuffleMirrorUV) // %3
- : "memory", "cc", "xmm0", "xmm5");
+ LABELALIGN
+ "1: \n"
+ "vmovdqu -0x20(%0,%2,2),%%ymm0 \n"
+ "vpshufb %%ymm5,%%ymm0,%%ymm0 \n"
+ "vpermq $0x4e,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm0,(%1) \n"
+ "lea 0x20(%1),%1 \n"
+ "sub $0x10,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_uv), // %0
+ "+r"(dst_uv), // %1
+ "+r"(temp_width) // %2
+ : "m"(kShuffleMirrorUV) // %3
+ : "memory", "cc", "xmm0", "xmm5");
}
#endif // HAS_MIRRORUVROW_AVX2
@@ -4817,7 +4500,7 @@
"sub %1,%2 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n"
"lea -0x10(%0),%0 \n"
"pshufb %%xmm1,%%xmm0 \n"
@@ -4856,7 +4539,7 @@
"movdqa %4,%%xmm5 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n" // first 5
"movdqu 15(%0),%%xmm1 \n" // next 5
"movdqu 30(%0),%%xmm2 \n" // next 5
@@ -4886,24 +4569,22 @@
void ARGBMirrorRow_SSE2(const uint8_t* src, uint8_t* dst, int width) {
intptr_t temp_width = (intptr_t)(width);
- asm volatile(
+ asm volatile("lea -0x10(%0,%2,4),%0 \n"
- "lea -0x10(%0,%2,4),%0 \n"
-
- LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "pshufd $0x1b,%%xmm0,%%xmm0 \n"
- "lea -0x10(%0),%0 \n"
- "movdqu %%xmm0,(%1) \n"
- "lea 0x10(%1),%1 \n"
- "sub $0x4,%2 \n"
- "jg 1b \n"
- : "+r"(src), // %0
- "+r"(dst), // %1
- "+r"(temp_width) // %2
- :
- : "memory", "cc", "xmm0");
+ LABELALIGN
+ "1: \n"
+ "movdqu (%0),%%xmm0 \n"
+ "pshufd $0x1b,%%xmm0,%%xmm0 \n"
+ "lea -0x10(%0),%0 \n"
+ "movdqu %%xmm0,(%1) \n"
+ "lea 0x10(%1),%1 \n"
+ "sub $0x4,%2 \n"
+ "jg 1b \n"
+ : "+r"(src), // %0
+ "+r"(dst), // %1
+ "+r"(temp_width) // %2
+ :
+ : "memory", "cc", "xmm0");
}
#endif // HAS_ARGBMIRRORROW_SSE2
@@ -4912,23 +4593,21 @@
static const ulvec32 kARGBShuffleMirror_AVX2 = {7u, 6u, 5u, 4u, 3u, 2u, 1u, 0u};
void ARGBMirrorRow_AVX2(const uint8_t* src, uint8_t* dst, int width) {
intptr_t temp_width = (intptr_t)(width);
- asm volatile(
+ asm volatile("vmovdqu %3,%%ymm5 \n"
- "vmovdqu %3,%%ymm5 \n"
-
- LABELALIGN
- "1: \n"
- "vpermd -0x20(%0,%2,4),%%ymm5,%%ymm0 \n"
- "vmovdqu %%ymm0,(%1) \n"
- "lea 0x20(%1),%1 \n"
- "sub $0x8,%2 \n"
- "jg 1b \n"
- "vzeroupper \n"
- : "+r"(src), // %0
- "+r"(dst), // %1
- "+r"(temp_width) // %2
- : "m"(kARGBShuffleMirror_AVX2) // %3
- : "memory", "cc", "xmm0", "xmm5");
+ LABELALIGN
+ "1: \n"
+ "vpermd -0x20(%0,%2,4),%%ymm5,%%ymm0 \n"
+ "vmovdqu %%ymm0,(%1) \n"
+ "lea 0x20(%1),%1 \n"
+ "sub $0x8,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src), // %0
+ "+r"(dst), // %1
+ "+r"(temp_width) // %2
+ : "m"(kARGBShuffleMirror_AVX2) // %3
+ : "memory", "cc", "xmm0", "xmm5");
}
#endif // HAS_ARGBMIRRORROW_AVX2
@@ -4943,7 +4622,7 @@
"sub %1,%2 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"vmovdqu (%0),%%ymm0 \n"
"vmovdqu 0x20(%0),%%ymm1 \n"
"lea 0x40(%0),%0 \n"
@@ -4960,7 +4639,7 @@
"lea 0x20(%1),%1 \n"
"sub $0x20,%3 \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: "+r"(src_uv), // %0
"+r"(dst_u), // %1
"+r"(dst_v), // %2
@@ -4981,7 +4660,7 @@
"sub %1,%2 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n"
"movdqu 0x10(%0),%%xmm1 \n"
"lea 0x20(%0),%0 \n"
@@ -5013,7 +4692,7 @@
uint8_t* dst,
int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n"
"sub $0x10,%2 \n"
"lea (%0,%3),%0 \n"
@@ -5034,7 +4713,7 @@
uint16_t* dst,
int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n"
"movdqu 0x10(%0),%%xmm1 \n"
"lea (%0,%3,2),%0 \n"
@@ -5057,14 +4736,14 @@
uint16_t* dst,
int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"vmovdqu (%0),%%ymm0 \n"
"lea (%0,%3,2),%0 \n"
"vmovdqu %%ymm0,(%1) \n"
"lea 0x20(%1),%1 \n"
"sub $0x10,%2 \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: "+r"(src), // %0
"+r"(dst), // %1
"+r"(width) // %2
@@ -5082,7 +4761,7 @@
uint8_t* dst_yuy2,
int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n" // Load 16 Y
"sub $0x10,%3 \n"
"lea (%0,%4),%0 \n"
@@ -5122,7 +4801,7 @@
int width) {
asm volatile(
"movdqu %4,%%xmm1 \n"
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n"
"lea (%0, %5),%0 \n"
"pshufb %%xmm1,%%xmm0 \n"
@@ -5147,20 +4826,20 @@
const uint8_t* src_v,
uint8_t* dst_uv,
int width) {
- asm volatile("sub %0,%1 \n"
+ asm volatile("sub %0,%1 \n"
LABELALIGN
- "1: \n"
- "vpmovzxbw (%0),%%zmm0 \n"
- "vpmovzxbw 0x00(%0,%1,1),%%zmm1 \n"
- "lea 0x20(%0),%0 \n"
- "vpsllw $0x8,%%zmm1,%%zmm1 \n"
- "vporq %%zmm0,%%zmm1,%%zmm2 \n"
- "vmovdqu64 %%zmm2,(%2) \n"
- "lea 0x40(%2),%2 \n"
- "sub $0x20,%3 \n"
- "jg 1b \n"
- "vzeroupper \n"
+ "1: \n"
+ "vpmovzxbw (%0),%%zmm0 \n"
+ "vpmovzxbw 0x00(%0,%1,1),%%zmm1 \n"
+ "lea 0x20(%0),%0 \n"
+ "vpsllw $0x8,%%zmm1,%%zmm1 \n"
+ "vporq %%zmm0,%%zmm1,%%zmm2 \n"
+ "vmovdqu64 %%zmm2,(%2) \n"
+ "lea 0x40(%2),%2 \n"
+ "sub $0x20,%3 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
: "+r"(src_u), // %0
"+r"(src_v), // %1
"+r"(dst_uv), // %2
@@ -5175,20 +4854,20 @@
const uint8_t* src_v,
uint8_t* dst_uv,
int width) {
- asm volatile("sub %0,%1 \n"
+ asm volatile("sub %0,%1 \n"
LABELALIGN
- "1: \n"
- "vpmovzxbw (%0),%%ymm0 \n"
- "vpmovzxbw 0x00(%0,%1,1),%%ymm1 \n"
- "lea 0x10(%0),%0 \n"
- "vpsllw $0x8,%%ymm1,%%ymm1 \n"
- "vpor %%ymm0,%%ymm1,%%ymm2 \n"
- "vmovdqu %%ymm2,(%2) \n"
- "lea 0x20(%2),%2 \n"
- "sub $0x10,%3 \n"
- "jg 1b \n"
- "vzeroupper \n"
+ "1: \n"
+ "vpmovzxbw (%0),%%ymm0 \n"
+ "vpmovzxbw 0x00(%0,%1,1),%%ymm1 \n"
+ "lea 0x10(%0),%0 \n"
+ "vpsllw $0x8,%%ymm1,%%ymm1 \n"
+ "vpor %%ymm0,%%ymm1,%%ymm2 \n"
+ "vmovdqu %%ymm2,(%2) \n"
+ "lea 0x20(%2),%2 \n"
+ "sub $0x10,%3 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
: "+r"(src_u), // %0
"+r"(src_v), // %1
"+r"(dst_uv), // %2
@@ -5203,21 +4882,21 @@
const uint8_t* src_v,
uint8_t* dst_uv,
int width) {
- asm volatile("sub %0,%1 \n"
+ asm volatile("sub %0,%1 \n"
LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x00(%0,%1,1),%%xmm1 \n"
- "lea 0x10(%0),%0 \n"
- "movdqa %%xmm0,%%xmm2 \n"
- "punpcklbw %%xmm1,%%xmm0 \n"
- "punpckhbw %%xmm1,%%xmm2 \n"
- "movdqu %%xmm0,(%2) \n"
- "movdqu %%xmm2,0x10(%2) \n"
- "lea 0x20(%2),%2 \n"
- "sub $0x10,%3 \n"
- "jg 1b \n"
+ "1: \n"
+ "movdqu (%0),%%xmm0 \n"
+ "movdqu 0x00(%0,%1,1),%%xmm1 \n"
+ "lea 0x10(%0),%0 \n"
+ "movdqa %%xmm0,%%xmm2 \n"
+ "punpcklbw %%xmm1,%%xmm0 \n"
+ "punpckhbw %%xmm1,%%xmm2 \n"
+ "movdqu %%xmm0,(%2) \n"
+ "movdqu %%xmm2,0x10(%2) \n"
+ "lea 0x20(%2),%2 \n"
+ "sub $0x10,%3 \n"
+ "jg 1b \n"
: "+r"(src_u), // %0
"+r"(src_v), // %1
"+r"(dst_uv), // %2
@@ -5233,17 +4912,15 @@
uint16_t* dst_uv,
int depth,
int width) {
- // clang-format off
- asm volatile (
+ asm volatile(
"vmovd %4,%%xmm3 \n"
"vmovd %5,%%xmm4 \n"
-
"sub %0,%1 \n"
// 8 pixels per loop.
LABELALIGN
- "1: \n"
+ "1: \n"
"vpmovzxwd (%0),%%ymm0 \n"
"vpmovzxwd 0x00(%0,%1,1),%%ymm1 \n"
"lea 0x10(%0),%0 \n"
@@ -5254,15 +4931,14 @@
"lea 0x20(%2),%2 \n"
"sub $0x8,%3 \n"
"jg 1b \n"
- "vzeroupper \n"
- : "+r"(src_u), // %0
- "+r"(src_v), // %1
- "+r"(dst_uv), // %2
- "+r"(width) // %3
- : "r"(16 - depth), // %4
- "r"(32 - depth) // %5
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4");
- // clang-format on
+ "vzeroupper \n"
+ : "+r"(src_u), // %0
+ "+r"(src_v), // %1
+ "+r"(dst_uv), // %2
+ "+r"(width) // %3
+ : "r"(16 - depth), // %4
+ "r"(32 - depth) // %5
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4");
}
#endif // HAS_MERGEUVROW_AVX2
@@ -5275,15 +4951,14 @@
int depth,
int width) {
depth = 16 - depth;
- // clang-format off
- asm volatile (
+ asm volatile(
"vmovd %4,%%xmm3 \n"
"vbroadcastf128 %5,%%ymm4 \n"
"sub %1,%2 \n"
- // 16 pixels per loop.
- LABELALIGN
- "1: \n"
+ // 16 pixels per loop.
+ LABELALIGN
+ "1: \n"
"vmovdqu (%0),%%ymm0 \n"
"vmovdqu 0x20(%0),%%ymm1 \n"
"add $0x40,%0 \n"
@@ -5301,15 +4976,14 @@
"add $0x20,%1 \n"
"sub $0x10,%3 \n"
"jg 1b \n"
- "vzeroupper \n"
- : "+r"(src_uv), // %0
- "+r"(dst_u), // %1
- "+r"(dst_v), // %2
- "+r"(width) // %3
- : "r"(depth), // %4
- "m"(kSplitUVShuffle16) // %5
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4");
- // clang-format on
+ "vzeroupper \n"
+ : "+r"(src_uv), // %0
+ "+r"(dst_u), // %1
+ "+r"(dst_v), // %2
+ "+r"(width) // %3
+ : "r"(depth), // %4
+ "m"(kSplitUVShuffle16) // %5
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4");
}
#endif // HAS_SPLITUVROW_16_AVX2
@@ -5323,16 +4997,14 @@
uint16_t* dst_y,
int scale,
int width) {
- // clang-format off
- asm volatile (
+ asm volatile(
"vmovd %3,%%xmm3 \n"
- "vpunpcklwd %%xmm3,%%xmm3,%%xmm3 \n"
- "vbroadcastss %%xmm3,%%ymm3 \n"
+ "vpbroadcastw %%xmm3,%%ymm3 \n"
"sub %0,%1 \n"
- // 32 pixels per loop.
- LABELALIGN
- "1: \n"
+ // 32 pixels per loop.
+ LABELALIGN
+ "1: \n"
"vmovdqu (%0),%%ymm0 \n"
"vmovdqu 0x20(%0),%%ymm1 \n"
"vpmullw %%ymm3,%%ymm0,%%ymm0 \n"
@@ -5342,13 +5014,12 @@
"add $0x40,%0 \n"
"sub $0x20,%2 \n"
"jg 1b \n"
- "vzeroupper \n"
- : "+r"(src_y), // %0
- "+r"(dst_y), // %1
- "+r"(width) // %2
- : "r"(scale) // %3
- : "memory", "cc", "xmm0", "xmm1", "xmm3");
- // clang-format on
+ "vzeroupper \n"
+ : "+r"(src_y), // %0
+ "+r"(dst_y), // %1
+ "+r"(width) // %2
+ : "r"(scale) // %3
+ : "memory", "cc", "xmm0", "xmm1", "xmm3");
}
#endif // HAS_MULTIPLYROW_16_AVX2
@@ -5362,16 +5033,14 @@
uint16_t* dst_y,
int scale,
int width) {
- // clang-format off
- asm volatile (
+ asm volatile(
"vmovd %3,%%xmm3 \n"
- "vpunpcklwd %%xmm3,%%xmm3,%%xmm3 \n"
- "vbroadcastss %%xmm3,%%ymm3 \n"
+ "vpbroadcastw %%xmm3,%%ymm3 \n"
"sub %0,%1 \n"
- // 32 pixels per loop.
- LABELALIGN
- "1: \n"
+ // 32 pixels per loop.
+ LABELALIGN
+ "1: \n"
"vmovdqu (%0),%%ymm0 \n"
"vmovdqu 0x20(%0),%%ymm1 \n"
"vpmulhuw %%ymm3,%%ymm0,%%ymm0 \n"
@@ -5381,14 +5050,13 @@
"add $0x40,%0 \n"
"sub $0x20,%2 \n"
"jg 1b \n"
- "vzeroupper \n"
- : "+r"(src_y), // %0
- "+r"(dst_y), // %1
- "+r"(width), // %2
- "+r"(scale) // %3
- :
- : "memory", "cc", "xmm0", "xmm1", "xmm3");
- // clang-format on
+ "vzeroupper \n"
+ : "+r"(src_y), // %0
+ "+r"(dst_y), // %1
+ "+r"(width), // %2
+ "+r"(scale) // %3
+ :
+ : "memory", "cc", "xmm0", "xmm1", "xmm3");
}
#endif // HAS_MULTIPLYROW_16_AVX2
@@ -5401,15 +5069,14 @@
uint8_t* dst_y,
int scale,
int width) {
- // clang-format off
- asm volatile (
+ asm volatile(
"movd %3,%%xmm2 \n"
"punpcklwd %%xmm2,%%xmm2 \n"
"pshufd $0x0,%%xmm2,%%xmm2 \n"
- // 32 pixels per loop.
- LABELALIGN
- "1: \n"
+ // 32 pixels per loop.
+ LABELALIGN
+ "1: \n"
"movdqu (%0),%%xmm0 \n"
"movdqu 0x10(%0),%%xmm1 \n"
"add $0x20,%0 \n"
@@ -5420,12 +5087,11 @@
"add $0x10,%1 \n"
"sub $0x10,%2 \n"
"jg 1b \n"
- : "+r"(src_y), // %0
- "+r"(dst_y), // %1
- "+r"(width) // %2
- : "r"(scale) // %3
- : "memory", "cc", "xmm0", "xmm1", "xmm2");
- // clang-format on
+ : "+r"(src_y), // %0
+ "+r"(dst_y), // %1
+ "+r"(width) // %2
+ : "r"(scale) // %3
+ : "memory", "cc", "xmm0", "xmm1", "xmm2");
}
#ifdef HAS_CONVERT16TO8ROW_AVX2
@@ -5433,15 +5099,13 @@
uint8_t* dst_y,
int scale,
int width) {
- // clang-format off
- asm volatile (
+ asm volatile(
"vmovd %3,%%xmm2 \n"
- "vpunpcklwd %%xmm2,%%xmm2,%%xmm2 \n"
- "vbroadcastss %%xmm2,%%ymm2 \n"
+ "vpbroadcastw %%xmm2,%%ymm2 \n"
- // 32 pixels per loop.
- LABELALIGN
- "1: \n"
+ // 32 pixels per loop.
+ LABELALIGN
+ "1: \n"
"vmovdqu (%0),%%ymm0 \n"
"vmovdqu 0x20(%0),%%ymm1 \n"
"add $0x40,%0 \n"
@@ -5453,13 +5117,43 @@
"add $0x20,%1 \n"
"sub $0x20,%2 \n"
"jg 1b \n"
- "vzeroupper \n"
- : "+r"(src_y), // %0
- "+r"(dst_y), // %1
- "+r"(width) // %2
- : "r"(scale) // %3
- : "memory", "cc", "xmm0", "xmm1", "xmm2");
- // clang-format on
+ "vzeroupper \n"
+ : "+r"(src_y), // %0
+ "+r"(dst_y), // %1
+ "+r"(width) // %2
+ : "r"(scale) // %3
+ : "memory", "cc", "xmm0", "xmm1", "xmm2");
+}
+#endif // HAS_CONVERT16TO8ROW_AVX2
+
+#ifdef HAS_CONVERT16TO8ROW_AVX512BW
+void Convert16To8Row_AVX512BW(const uint16_t* src_y,
+ uint8_t* dst_y,
+ int scale,
+ int width) {
+ asm volatile("vpbroadcastw %3,%%zmm2 \n"
+
+ // 64 pixels per loop.
+ LABELALIGN
+ "1: \n"
+ "vmovups (%0),%%zmm0 \n"
+ "vmovups 0x40(%0),%%zmm1 \n"
+ "add $0x80,%0 \n"
+ "vpmulhuw %%zmm2,%%zmm0,%%zmm0 \n"
+ "vpmulhuw %%zmm2,%%zmm1,%%zmm1 \n"
+ "vpmovuswb %%zmm0,%%ymm0 \n"
+ "vpmovuswb %%zmm1,%%ymm1 \n"
+ "vmovups %%ymm0,(%1) \n"
+ "vmovups %%ymm1,0x20(%1) \n"
+ "add $0x40,%1 \n"
+ "sub $0x40,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_y), // %0
+ "+r"(dst_y), // %1
+ "+r"(width) // %2
+ : "r"(scale) // %3
+ : "memory", "cc", "xmm0", "xmm1", "xmm2");
}
#endif // HAS_CONVERT16TO8ROW_AVX2
@@ -5471,15 +5165,14 @@
uint16_t* dst_y,
int scale,
int width) {
- // clang-format off
- asm volatile (
+ asm volatile(
"movd %3,%%xmm2 \n"
"punpcklwd %%xmm2,%%xmm2 \n"
"pshufd $0x0,%%xmm2,%%xmm2 \n"
- // 32 pixels per loop.
- LABELALIGN
- "1: \n"
+ // 32 pixels per loop.
+ LABELALIGN
+ "1: \n"
"movdqu (%0),%%xmm0 \n"
"movdqa %%xmm0,%%xmm1 \n"
"punpcklbw %%xmm0,%%xmm0 \n"
@@ -5492,12 +5185,11 @@
"add $0x20,%1 \n"
"sub $0x10,%2 \n"
"jg 1b \n"
- : "+r"(src_y), // %0
- "+r"(dst_y), // %1
- "+r"(width) // %2
- : "r"(scale) // %3
- : "memory", "cc", "xmm0", "xmm1", "xmm2");
- // clang-format on
+ : "+r"(src_y), // %0
+ "+r"(dst_y), // %1
+ "+r"(width) // %2
+ : "r"(scale) // %3
+ : "memory", "cc", "xmm0", "xmm1", "xmm2");
}
#ifdef HAS_CONVERT8TO16ROW_AVX2
@@ -5505,15 +5197,13 @@
uint16_t* dst_y,
int scale,
int width) {
- // clang-format off
- asm volatile (
+ asm volatile(
"vmovd %3,%%xmm2 \n"
- "vpunpcklwd %%xmm2,%%xmm2,%%xmm2 \n"
- "vbroadcastss %%xmm2,%%ymm2 \n"
+ "vpbroadcastw %%xmm2,%%ymm2 \n"
- // 32 pixels per loop.
- LABELALIGN
- "1: \n"
+ // 32 pixels per loop.
+ LABELALIGN
+ "1: \n"
"vmovdqu (%0),%%ymm0 \n"
"vpermq $0xd8,%%ymm0,%%ymm0 \n"
"add $0x20,%0 \n"
@@ -5526,13 +5216,12 @@
"add $0x40,%1 \n"
"sub $0x20,%2 \n"
"jg 1b \n"
- "vzeroupper \n"
- : "+r"(src_y), // %0
- "+r"(dst_y), // %1
- "+r"(width) // %2
- : "r"(scale) // %3
- : "memory", "cc", "xmm0", "xmm1", "xmm2");
- // clang-format on
+ "vzeroupper \n"
+ : "+r"(src_y), // %0
+ "+r"(dst_y), // %1
+ "+r"(width) // %2
+ : "r"(scale) // %3
+ : "memory", "cc", "xmm0", "xmm1", "xmm2");
}
#endif // HAS_CONVERT8TO16ROW_AVX2
@@ -5564,9 +5253,7 @@
uint8_t* dst_b,
int width) {
asm volatile(
-
- LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n"
"movdqu 0x10(%0),%%xmm1 \n"
"movdqu 0x20(%0),%%xmm2 \n"
@@ -5612,6 +5299,123 @@
}
#endif // HAS_SPLITRGBROW_SSSE3
+#ifdef HAS_SPLITRGBROW_SSE41
+// Shuffle table for converting RGB to Planar, SSE4.1. Note: these are used for
+// the AVX2 implementation as well.
+static const uvec8 kSplitRGBShuffleSSE41[5] = {
+ {0u, 3u, 6u, 9u, 12u, 15u, 2u, 5u, 8u, 11u, 14u, 1u, 4u, 7u, 10u, 13u},
+ {1u, 4u, 7u, 10u, 13u, 0u, 3u, 6u, 9u, 12u, 15u, 2u, 5u, 8u, 11u, 14u},
+ {2u, 5u, 8u, 11u, 14u, 1u, 4u, 7u, 10u, 13u, 0u, 3u, 6u, 9u, 12u, 15u},
+ {0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u, 0u, 0u},
+ {0u, 0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u, 0u},
+};
+
+void SplitRGBRow_SSE41(const uint8_t* src_rgb,
+ uint8_t* dst_r,
+ uint8_t* dst_g,
+ uint8_t* dst_b,
+ int width) {
+ asm volatile(
+ "movdqa 48(%5), %%xmm0 \n"
+ "1: \n"
+ "movdqu (%0),%%xmm1 \n"
+ "movdqu 0x10(%0),%%xmm2 \n"
+ "movdqu 0x20(%0),%%xmm3 \n"
+ "lea 0x30(%0),%0 \n"
+ "movdqa %%xmm1, %%xmm4 \n"
+ "pblendvb %%xmm3, %%xmm1 \n"
+ "pblendvb %%xmm2, %%xmm3 \n"
+ "pblendvb %%xmm4, %%xmm2 \n"
+ "palignr $0xF, %%xmm0, %%xmm0 \n"
+ "pblendvb %%xmm2, %%xmm1 \n"
+ "pblendvb %%xmm3, %%xmm2 \n"
+ "pblendvb %%xmm4, %%xmm3 \n"
+ "palignr $0x1, %%xmm0, %%xmm0 \n"
+ "pshufb 0(%5), %%xmm1 \n"
+ "pshufb 16(%5), %%xmm2 \n"
+ "pshufb 32(%5), %%xmm3 \n"
+ "movdqu %%xmm1,(%1) \n"
+ "lea 0x10(%1),%1 \n"
+ "movdqu %%xmm2,(%2) \n"
+ "lea 0x10(%2),%2 \n"
+ "movdqu %%xmm3,(%3) \n"
+ "lea 0x10(%3),%3 \n"
+ "sub $0x10,%4 \n"
+ "jg 1b \n"
+ : "+r"(src_rgb), // %0
+ "+r"(dst_r), // %1
+ "+r"(dst_g), // %2
+ "+r"(dst_b), // %3
+ "+r"(width) // %4
+ : "r"(&kSplitRGBShuffleSSE41[0]) // %5
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4");
+}
+#endif // HAS_SPLITRGBROW_SSE41
+
+#ifdef HAS_SPLITRGBROW_AVX2
+void SplitRGBRow_AVX2(const uint8_t* src_rgb,
+ uint8_t* dst_r,
+ uint8_t* dst_g,
+ uint8_t* dst_b,
+ int width) {
+ asm volatile(
+ "vbroadcasti128 48(%5), %%ymm0 \n"
+ "vbroadcasti128 64(%5), %%ymm7 \n"
+#if defined(__x86_64__)
+ "vbroadcasti128 0(%5), %%ymm8 \n"
+ "vbroadcasti128 16(%5), %%ymm9 \n"
+ "vbroadcasti128 32(%5), %%ymm10 \n"
+#endif
+ "1: \n"
+ "vmovdqu (%0),%%ymm4 \n"
+ "vmovdqu 0x20(%0),%%ymm5 \n"
+ "vmovdqu 0x40(%0),%%ymm6 \n"
+ "lea 0x60(%0),%0 \n"
+ "vpblendd $240, %%ymm5, %%ymm4, %%ymm1 \n"
+ "vperm2i128 $33, %%ymm6, %%ymm4, %%ymm2 \n"
+ "vpblendd $240, %%ymm6, %%ymm5, %%ymm3 \n"
+ "vpblendvb %%ymm0, %%ymm3, %%ymm1, %%ymm4 \n"
+ "vpblendvb %%ymm0, %%ymm1, %%ymm2, %%ymm5 \n"
+ "vpblendvb %%ymm0, %%ymm2, %%ymm3, %%ymm6 \n"
+ "vpblendvb %%ymm7, %%ymm5, %%ymm4, %%ymm1 \n"
+ "vpblendvb %%ymm7, %%ymm6, %%ymm5, %%ymm2 \n"
+ "vpblendvb %%ymm7, %%ymm4, %%ymm6, %%ymm3 \n"
+#if defined(__x86_64__)
+ "vpshufb %%ymm8, %%ymm1, %%ymm1 \n"
+ "vpshufb %%ymm9, %%ymm2, %%ymm2 \n"
+ "vpshufb %%ymm10, %%ymm3, %%ymm3 \n"
+#else
+ "vbroadcasti128 0(%5), %%ymm4 \n"
+ "vbroadcasti128 16(%5), %%ymm5 \n"
+ "vbroadcasti128 32(%5), %%ymm6 \n"
+ "vpshufb %%ymm4, %%ymm1, %%ymm1 \n"
+ "vpshufb %%ymm5, %%ymm2, %%ymm2 \n"
+ "vpshufb %%ymm6, %%ymm3, %%ymm3 \n"
+#endif
+ "vmovdqu %%ymm1,(%1) \n"
+ "lea 0x20(%1),%1 \n"
+ "vmovdqu %%ymm2,(%2) \n"
+ "lea 0x20(%2),%2 \n"
+ "vmovdqu %%ymm3,(%3) \n"
+ "lea 0x20(%3),%3 \n"
+ "sub $0x20,%4 \n"
+ "jg 1b \n"
+ : "+r"(src_rgb), // %0
+ "+r"(dst_r), // %1
+ "+r"(dst_g), // %2
+ "+r"(dst_b), // %3
+ "+r"(width) // %4
+ : "r"(&kSplitRGBShuffleSSE41[0]) // %5
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ "xmm7"
+#if defined(__x86_64__)
+ ,
+ "xmm8", "xmm9", "xmm10"
+#endif
+ );
+}
+#endif // HAS_SPLITRGBROW_AVX2
+
#ifdef HAS_MERGERGBROW_SSSE3
// Shuffle table for converting Planar to RGB.
static const uvec8 kMergeRGBShuffle[9] = {
@@ -5640,9 +5444,7 @@
uint8_t* dst_rgb,
int width) {
asm volatile(
-
- LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n"
"movdqu (%1),%%xmm1 \n"
"movdqu (%2),%%xmm2 \n"
@@ -5697,13 +5499,12 @@
uint8_t* dst_argb,
int width) {
asm volatile(
-
"sub %0,%1 \n"
"sub %0,%2 \n"
"sub %0,%3 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"movq (%0,%2),%%xmm0 \n" // B
"movq (%0),%%xmm1 \n" // R
@@ -5739,9 +5540,7 @@
uint8_t* dst_argb,
int width) {
asm volatile(
-
- LABELALIGN
- "1: \n"
+ "1: \n"
"movq (%2),%%xmm0 \n" // B
"movq (%0),%%xmm1 \n" // R
@@ -5779,13 +5578,12 @@
uint8_t* dst_argb,
int width) {
asm volatile(
-
"sub %0,%1 \n"
"sub %0,%2 \n"
"sub %0,%3 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"vmovdqu (%0,%2),%%xmm0 \n" // B
"vmovdqu (%0,%1),%%xmm1 \n" // R
@@ -5806,7 +5604,7 @@
"lea 64(%4),%4 \n"
"sub $0x10,%5 \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: "+r"(src_r), // %0
"+r"(src_g), // %1
"+r"(src_b), // %2
@@ -5825,12 +5623,10 @@
uint8_t* dst_argb,
int width) {
asm volatile(
-
- LABELALIGN
- "1: \n"
+ "1: \n"
"vmovdqu (%2),%%xmm0 \n" // B
- "vpcmpeqd %%ymm1,%%ymm1,%%ymm1 \n" // A(255)
+ "vpcmpeqb %%ymm1,%%ymm1,%%ymm1 \n" // A(255)
"vinserti128 $0,(%1),%%ymm1,%%ymm1 \n" // R
"vinserti128 $1,(%0),%%ymm0,%%ymm0 \n" // G
"vpunpckhbw %%ymm1,%%ymm0,%%ymm2 \n"
@@ -5850,14 +5646,14 @@
"lea 64(%3),%3 \n"
"sub $0x10,%4 \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: "+r"(src_r), // %0
"+r"(src_g), // %1
"+r"(src_b), // %2
"+r"(dst_argb), // %3
"+rm"(width) // %4
- :
- : "memory", "cc", "xmm0", "xmm1", "xmm2");
+ ::"memory",
+ "cc", "xmm0", "xmm1", "xmm2");
}
#endif // HAS_MERGEARGBROW_AVX2
@@ -5869,13 +5665,12 @@
uint8_t* dst_a,
int width) {
asm volatile(
-
"sub %1,%2 \n"
"sub %1,%3 \n"
"sub %1,%4 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n" // 00-0F
"movdqu 16(%0),%%xmm1 \n" // 10-1F
@@ -5921,9 +5716,7 @@
uint8_t* dst_b,
int width) {
asm volatile(
-
- LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n" // 00-0F
"movdqu 16(%0),%%xmm1 \n" // 10-1F
@@ -5972,14 +5765,13 @@
uint8_t* dst_a,
int width) {
asm volatile(
-
"movdqa %6,%%xmm3 \n"
"sub %1,%2 \n"
"sub %1,%3 \n"
"sub %1,%4 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n" // 00-0F
"movdqu 16(%0),%%xmm1 \n" // 10-1F
@@ -6005,7 +5797,7 @@
#if defined(__i386__)
"+m"(width) // %5
#else
- "+rm"(width) // %5
+ "+rm"(width) // %5
#endif
: "m"(kShuffleMaskARGBSplit) // %6
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3");
@@ -6019,11 +5811,10 @@
uint8_t* dst_b,
int width) {
asm volatile(
-
"movdqa %5,%%xmm3 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n" // 00-0F
"movdqu 16(%0),%%xmm1 \n" // 10-1F
@@ -6061,7 +5852,6 @@
uint8_t* dst_a,
int width) {
asm volatile(
-
"sub %1,%2 \n"
"sub %1,%3 \n"
"sub %1,%4 \n"
@@ -6069,7 +5859,7 @@
"vbroadcastf128 %6,%%ymm4 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"vmovdqu (%0),%%xmm0 \n" // 00-0F
"vmovdqu 16(%0),%%xmm1 \n" // 10-1F
@@ -6089,7 +5879,7 @@
"lea 16(%1),%1 \n"
"subl $0x10,%5 \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: "+r"(src_argb), // %0
"+r"(dst_r), // %1
"+r"(dst_g), // %2
@@ -6098,7 +5888,7 @@
#if defined(__i386__)
"+m"(width) // %5
#else
- "+rm"(width) // %5
+ "+rm"(width) // %5
#endif
: "m"(kShuffleMaskARGBSplit), // %6
"m"(kShuffleMaskARGBPermute) // %7
@@ -6113,12 +5903,11 @@
uint8_t* dst_b,
int width) {
asm volatile(
-
"vmovdqa %6,%%ymm3 \n"
"vbroadcastf128 %5,%%ymm4 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"vmovdqu (%0),%%xmm0 \n" // 00-0F
"vmovdqu 16(%0),%%xmm1 \n" // 10-1F
@@ -6140,7 +5929,7 @@
"lea 16(%3),%3 \n"
"sub $0x10,%4 \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: "+r"(src_argb), // %0
"+r"(dst_r), // %1
"+r"(dst_g), // %2
@@ -6161,7 +5950,6 @@
int width) {
int shift = depth - 10;
asm volatile(
-
"sub %0,%1 \n"
"sub %0,%2 \n"
"vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n" // AR30 constants
@@ -6172,7 +5960,7 @@
"vmovd %5,%%xmm4 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"vmovdqu (%0),%%ymm0 \n"
"vmovdqu (%0,%1),%%ymm1 \n"
"vmovdqu (%0,%2),%%ymm2 \n"
@@ -6200,7 +5988,7 @@
"lea 0x40(%3),%3 \n"
"sub $0x10,%4 \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: "+r"(src_r), // %0
"+r"(src_g), // %1
"+r"(src_b), // %2
@@ -6209,7 +5997,7 @@
#if defined(__i386__)
: "m"(shift) // %5
#else
- : "rm"(shift) // %5
+ : "rm"(shift) // %5
#endif
: "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
}
@@ -6228,7 +6016,6 @@
int mask = (1 << depth) - 1;
mask = (mask << 16) + mask;
asm volatile(
-
"sub %0,%1 \n"
"sub %0,%2 \n"
"sub %0,%3 \n"
@@ -6237,7 +6024,7 @@
"vbroadcastss %7,%%ymm7 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"vmovdqu (%0),%%ymm0 \n" // R
"vmovdqu (%0,%1),%%ymm1 \n" // G
"vmovdqu (%0,%2),%%ymm2 \n" // B
@@ -6270,7 +6057,7 @@
"lea 0x80(%4),%4 \n"
"subl $0x10,%5 \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: "+r"(src_r), // %0
"+r"(src_g), // %1
"+r"(src_b), // %2
@@ -6279,7 +6066,7 @@
#if defined(__i386__)
"+m"(width) // %5
#else
- "+rm"(width) // %5
+ "+rm"(width) // %5
#endif
: "m"(shift), // %6
"m"(mask), // %7
@@ -6300,7 +6087,6 @@
int mask = (1 << depth) - 1;
mask = (mask << 16) + mask;
asm volatile(
-
"sub %0,%1 \n"
"sub %0,%2 \n"
"vmovdqa %7,%%ymm5 \n"
@@ -6308,7 +6094,7 @@
"vbroadcastss %6,%%ymm7 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"vmovdqu (%0),%%ymm0 \n" // R
"vmovdqu (%0,%1),%%ymm1 \n" // G
"vmovdqu (%0,%2),%%ymm2 \n" // B
@@ -6338,7 +6124,7 @@
"lea 0x80(%3),%3 \n"
"subl $0x10,%4 \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: "+r"(src_r), // %0
"+r"(src_g), // %1
"+r"(src_b), // %2
@@ -6364,7 +6150,6 @@
int width) {
int shift = depth - 8;
asm volatile(
-
"sub %0,%1 \n"
"sub %0,%2 \n"
"sub %0,%3 \n"
@@ -6372,7 +6157,7 @@
"vmovd %6,%%xmm6 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"vmovdqu (%0),%%ymm0 \n" // R
"vmovdqu (%0,%1),%%ymm1 \n" // G
"vmovdqu (%0,%2),%%ymm2 \n" // B
@@ -6395,7 +6180,7 @@
"lea 0x40(%4),%4 \n"
"subl $0x10,%5 \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: "+r"(src_r), // %0
"+r"(src_g), // %1
"+r"(src_b), // %2
@@ -6404,7 +6189,7 @@
#if defined(__i386__)
"+m"(width) // %5
#else
- "+rm"(width) // %5
+ "+rm"(width) // %5
#endif
: "m"(shift), // %6
"m"(MergeARGB16To8Shuffle) // %7
@@ -6421,7 +6206,6 @@
int width) {
int shift = depth - 8;
asm volatile(
-
"sub %0,%1 \n"
"sub %0,%2 \n"
"vbroadcastf128 %6,%%ymm5 \n"
@@ -6430,7 +6214,7 @@
"vpsrlw $8,%%ymm3,%%ymm3 \n" // A (0xff)
LABELALIGN
- "1: \n"
+ "1: \n"
"vmovdqu (%0),%%ymm0 \n" // R
"vmovdqu (%0,%1),%%ymm1 \n" // G
"vmovdqu (%0,%2),%%ymm2 \n" // B
@@ -6451,7 +6235,7 @@
"lea 0x40(%3),%3 \n"
"subl $0x10,%4 \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: "+r"(src_r), // %0
"+r"(src_g), // %1
"+r"(src_b), // %2
@@ -6472,7 +6256,7 @@
"jne 2f \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"movdqa (%0),%%xmm0 \n"
"movdqa 0x10(%0),%%xmm1 \n"
"lea 0x20(%0),%0 \n"
@@ -6484,7 +6268,7 @@
"jmp 9f \n"
LABELALIGN
- "2: \n"
+ "2: \n"
"movdqu (%0),%%xmm0 \n"
"movdqu 0x10(%0),%%xmm1 \n"
"lea 0x20(%0),%0 \n"
@@ -6494,7 +6278,7 @@
"sub $0x20,%2 \n"
"jg 2b \n"
- LABELALIGN "9: \n"
+ LABELALIGN "9: \n"
: "+r"(src), // %0
"+r"(dst), // %1
"+r"(width) // %2
@@ -6506,9 +6290,7 @@
#ifdef HAS_COPYROW_AVX
void CopyRow_AVX(const uint8_t* src, uint8_t* dst, int width) {
asm volatile(
-
- LABELALIGN
- "1: \n"
+ "1: \n"
"vmovdqu (%0),%%ymm0 \n"
"vmovdqu 0x20(%0),%%ymm1 \n"
"lea 0x40(%0),%0 \n"
@@ -6517,7 +6299,7 @@
"lea 0x40(%1),%1 \n"
"sub $0x40,%2 \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: "+r"(src), // %0
"+r"(dst), // %1
"+r"(width) // %2
@@ -6526,18 +6308,37 @@
}
#endif // HAS_COPYROW_AVX
+#ifdef HAS_COPYROW_AVX512BW
+void CopyRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) {
+ asm volatile(
+ "1: \n"
+ "vmovups (%0),%%zmm0 \n"
+ "vmovups 0x40(%0),%%zmm1 \n"
+ "lea 0x80(%0),%0 \n"
+ "vmovups %%zmm0,(%1) \n"
+ "vmovups %%zmm1,0x40(%1) \n"
+ "lea 0x80(%1),%1 \n"
+ "sub $0x80,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src), // %0
+ "+r"(dst), // %1
+ "+r"(width) // %2
+ :
+ : "memory", "cc", "xmm0", "xmm1");
+}
+#endif // HAS_COPYROW_AVX512
+
#ifdef HAS_COPYROW_ERMS
// Multiple of 1.
void CopyRow_ERMS(const uint8_t* src, uint8_t* dst, int width) {
size_t width_tmp = (size_t)(width);
- asm volatile(
-
- "rep movsb \n"
- : "+S"(src), // %0
- "+D"(dst), // %1
- "+c"(width_tmp) // %2
- :
- : "memory", "cc");
+ asm volatile("rep movsb \n"
+ : "+S"(src), // %0
+ "+D"(dst), // %1
+ "+c"(width_tmp) // %2
+ :
+ : "memory", "cc");
}
#endif // HAS_COPYROW_ERMS
@@ -6551,7 +6352,7 @@
"psrld $0x8,%%xmm1 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm2 \n"
"movdqu 0x10(%0),%%xmm3 \n"
"lea 0x20(%0),%0 \n"
@@ -6584,7 +6385,7 @@
"vpsrld $0x8,%%ymm0,%%ymm0 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"vmovdqu (%0),%%ymm1 \n"
"vmovdqu 0x20(%0),%%ymm2 \n"
"lea 0x40(%0),%0 \n"
@@ -6595,7 +6396,7 @@
"lea 0x40(%1),%1 \n"
"sub $0x10,%2 \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: "+r"(src), // %0
"+r"(dst), // %1
"+r"(width) // %2
@@ -6610,9 +6411,7 @@
uint8_t* dst_a,
int width) {
asm volatile(
-
- LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%0), %%xmm0 \n"
"movdqu 0x10(%0), %%xmm1 \n"
"lea 0x20(%0), %0 \n"
@@ -6645,7 +6444,7 @@
"vbroadcastf128 %4,%%ymm5 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"vmovdqu (%0), %%ymm0 \n"
"vmovdqu 0x20(%0), %%ymm1 \n"
"vpshufb %%ymm5,%%ymm0,%%ymm0 \n" // vpsrld $0x18, %%ymm0
@@ -6663,7 +6462,7 @@
"lea 0x20(%1),%1 \n"
"sub $0x20, %2 \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: "+r"(src_argb), // %0
"+r"(dst_a), // %1
"+rm"(width) // %2
@@ -6683,7 +6482,7 @@
"psrld $0x8,%%xmm1 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"movq (%0),%%xmm2 \n"
"lea 0x8(%0),%0 \n"
"punpcklbw %%xmm2,%%xmm2 \n"
@@ -6718,7 +6517,7 @@
"vpsrld $0x8,%%ymm0,%%ymm0 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"vpmovzxbd (%0),%%ymm1 \n"
"vpmovzxbd 0x8(%0),%%ymm2 \n"
"lea 0x10(%0),%0 \n"
@@ -6731,7 +6530,7 @@
"lea 0x40(%1),%1 \n"
"sub $0x10,%2 \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: "+r"(src), // %0
"+r"(dst), // %1
"+r"(width) // %2
@@ -6744,35 +6543,29 @@
void SetRow_X86(uint8_t* dst, uint8_t v8, int width) {
size_t width_tmp = (size_t)(width >> 2);
const uint32_t v32 = v8 * 0x01010101u; // Duplicate byte to all bytes.
- asm volatile(
-
- "rep stosl \n"
- : "+D"(dst), // %0
- "+c"(width_tmp) // %1
- : "a"(v32) // %2
- : "memory", "cc");
+ asm volatile("rep stosl \n"
+ : "+D"(dst), // %0
+ "+c"(width_tmp) // %1
+ : "a"(v32) // %2
+ : "memory", "cc");
}
void SetRow_ERMS(uint8_t* dst, uint8_t v8, int width) {
size_t width_tmp = (size_t)(width);
- asm volatile(
-
- "rep stosb \n"
- : "+D"(dst), // %0
- "+c"(width_tmp) // %1
- : "a"(v8) // %2
- : "memory", "cc");
+ asm volatile("rep stosb \n"
+ : "+D"(dst), // %0
+ "+c"(width_tmp) // %1
+ : "a"(v8) // %2
+ : "memory", "cc");
}
void ARGBSetRow_X86(uint8_t* dst_argb, uint32_t v32, int width) {
size_t width_tmp = (size_t)(width);
- asm volatile(
-
- "rep stosl \n"
- : "+D"(dst_argb), // %0
- "+c"(width_tmp) // %1
- : "a"(v32) // %2
- : "memory", "cc");
+ asm volatile("rep stosl \n"
+ : "+D"(dst_argb), // %0
+ "+c"(width_tmp) // %1
+ : "a"(v32) // %2
+ : "memory", "cc");
}
#endif // HAS_SETROW_X86
@@ -6783,7 +6576,7 @@
"psrlw $0x8,%%xmm5 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n"
"movdqu 0x10(%0),%%xmm1 \n"
"lea 0x20(%0),%0 \n"
@@ -6805,8 +6598,8 @@
int stride_yuy2,
uint8_t* dst_uv,
int width) {
- asm volatile(LABELALIGN
- "1: \n"
+ asm volatile(
+ "1: \n"
"movdqu (%0),%%xmm0 \n"
"movdqu 0x10(%0),%%xmm1 \n"
"movdqu 0x00(%0,%3,1),%%xmm2 \n"
@@ -6821,11 +6614,11 @@
"lea 0x10(%1),%1 \n"
"sub $0x10,%2 \n"
"jg 1b \n"
- : "+r"(src_yuy2), // %0
- "+r"(dst_uv), // %1
- "+r"(width) // %2
- : "r"((intptr_t)(stride_yuy2)) // %3
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3");
+ : "+r"(src_yuy2), // %0
+ "+r"(dst_uv), // %1
+ "+r"(width) // %2
+ : "r"((intptr_t)(stride_yuy2)) // %3
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3");
}
void YUY2ToUVRow_SSE2(const uint8_t* src_yuy2,
@@ -6839,7 +6632,7 @@
"sub %1,%2 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n"
"movdqu 0x10(%0),%%xmm1 \n"
"movdqu 0x00(%0,%4,1),%%xmm2 \n"
@@ -6878,7 +6671,7 @@
"sub %1,%2 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n"
"movdqu 0x10(%0),%%xmm1 \n"
"lea 0x20(%0),%0 \n"
@@ -6905,9 +6698,7 @@
void UYVYToYRow_SSE2(const uint8_t* src_uyvy, uint8_t* dst_y, int width) {
asm volatile(
-
- LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n"
"movdqu 0x10(%0),%%xmm1 \n"
"lea 0x20(%0),%0 \n"
@@ -6936,7 +6727,7 @@
"sub %1,%2 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n"
"movdqu 0x10(%0),%%xmm1 \n"
"movdqu 0x00(%0,%4,1),%%xmm2 \n"
@@ -6975,7 +6766,7 @@
"sub %1,%2 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n"
"movdqu 0x10(%0),%%xmm1 \n"
"lea 0x20(%0),%0 \n"
@@ -7008,7 +6799,7 @@
"vpsrlw $0x8,%%ymm5,%%ymm5 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"vmovdqu (%0),%%ymm0 \n"
"vmovdqu 0x20(%0),%%ymm1 \n"
"lea 0x40(%0),%0 \n"
@@ -7020,7 +6811,7 @@
"lea 0x20(%1),%1 \n"
"sub $0x20,%2 \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: "+r"(src_yuy2), // %0
"+r"(dst_y), // %1
"+r"(width) // %2
@@ -7033,9 +6824,7 @@
uint8_t* dst_uv,
int width) {
asm volatile(
-
- LABELALIGN
- "1: \n"
+ "1: \n"
"vmovdqu (%0),%%ymm0 \n"
"vmovdqu 0x20(%0),%%ymm1 \n"
"vpavgb 0x00(%0,%3,1),%%ymm0,%%ymm0 \n"
@@ -7049,7 +6838,7 @@
"lea 0x20(%1),%1 \n"
"sub $0x20,%2 \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: "+r"(src_yuy2), // %0
"+r"(dst_uv), // %1
"+r"(width) // %2
@@ -7068,7 +6857,7 @@
"sub %1,%2 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"vmovdqu (%0),%%ymm0 \n"
"vmovdqu 0x20(%0),%%ymm1 \n"
"vpavgb 0x00(%0,%4,1),%%ymm0,%%ymm0 \n"
@@ -7089,7 +6878,7 @@
"lea 0x10(%1),%1 \n"
"sub $0x20,%3 \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: "+r"(src_yuy2), // %0
"+r"(dst_u), // %1
"+r"(dst_v), // %2
@@ -7108,7 +6897,7 @@
"sub %1,%2 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"vmovdqu (%0),%%ymm0 \n"
"vmovdqu 0x20(%0),%%ymm1 \n"
"lea 0x40(%0),%0 \n"
@@ -7127,7 +6916,7 @@
"lea 0x10(%1),%1 \n"
"sub $0x20,%3 \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: "+r"(src_yuy2), // %0
"+r"(dst_u), // %1
"+r"(dst_v), // %2
@@ -7138,9 +6927,7 @@
void UYVYToYRow_AVX2(const uint8_t* src_uyvy, uint8_t* dst_y, int width) {
asm volatile(
-
- LABELALIGN
- "1: \n"
+ "1: \n"
"vmovdqu (%0),%%ymm0 \n"
"vmovdqu 0x20(%0),%%ymm1 \n"
"lea 0x40(%0),%0 \n"
@@ -7152,7 +6939,7 @@
"lea 0x20(%1),%1 \n"
"sub $0x20,%2 \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: "+r"(src_uyvy), // %0
"+r"(dst_y), // %1
"+r"(width) // %2
@@ -7170,7 +6957,7 @@
"sub %1,%2 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"vmovdqu (%0),%%ymm0 \n"
"vmovdqu 0x20(%0),%%ymm1 \n"
"vpavgb 0x00(%0,%4,1),%%ymm0,%%ymm0 \n"
@@ -7191,7 +6978,7 @@
"lea 0x10(%1),%1 \n"
"sub $0x20,%3 \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: "+r"(src_uyvy), // %0
"+r"(dst_u), // %1
"+r"(dst_v), // %2
@@ -7210,7 +6997,7 @@
"sub %1,%2 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"vmovdqu (%0),%%ymm0 \n"
"vmovdqu 0x20(%0),%%ymm1 \n"
"lea 0x40(%0),%0 \n"
@@ -7229,7 +7016,7 @@
"lea 0x10(%1),%1 \n"
"sub $0x20,%3 \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: "+r"(src_uyvy), // %0
"+r"(dst_u), // %1
"+r"(dst_v), // %2
@@ -7263,7 +7050,7 @@
// 4 pixel loop.
LABELALIGN
- "40: \n"
+ "40: \n"
"movdqu (%0),%%xmm3 \n"
"lea 0x10(%0),%0 \n"
"movdqa %%xmm3,%%xmm0 \n"
@@ -7287,12 +7074,12 @@
"sub $0x4,%3 \n"
"jge 40b \n"
- "49: \n"
+ "49: \n"
"add $0x3,%3 \n"
"jl 99f \n"
// 1 pixel loop.
- "91: \n"
+ "91: \n"
"movd (%0),%%xmm3 \n"
"lea 0x4(%0),%0 \n"
"movdqa %%xmm3,%%xmm0 \n"
@@ -7315,7 +7102,7 @@
"lea 0x4(%2),%2 \n"
"sub $0x1,%3 \n"
"jge 91b \n"
- "99: \n"
+ "99: \n"
: "+r"(src_argb), // %0
"+r"(src_argb1), // %1
"+r"(dst_argb), // %2
@@ -7352,7 +7139,7 @@
// 8 pixel loop.
LABELALIGN
- "1: \n"
+ "1: \n"
"movq (%2),%%xmm0 \n"
"punpcklbw %%xmm0,%%xmm0 \n"
"pxor %%xmm5,%%xmm0 \n"
@@ -7404,7 +7191,7 @@
// 32 pixel loop.
LABELALIGN
- "1: \n"
+ "1: \n"
"vmovdqu (%2),%%ymm0 \n"
"vpunpckhbw %%ymm0,%%ymm0,%%ymm3 \n"
"vpunpcklbw %%ymm0,%%ymm0,%%ymm0 \n"
@@ -7427,7 +7214,7 @@
"lea 0x20(%2),%2 \n"
"sub $0x20,%4 \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: "+r"(src0), // %0
"+r"(src1), // %1
"+r"(alpha), // %2
@@ -7460,7 +7247,7 @@
// 4 pixel loop.
LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm6 \n"
"movdqa %%xmm6,%%xmm0 \n"
"movdqa %%xmm6,%%xmm1 \n"
@@ -7515,7 +7302,7 @@
// 8 pixel loop.
LABELALIGN
- "1: \n"
+ "1: \n"
"vmovdqu (%0),%%ymm6 \n"
"vpunpcklbw %%ymm5,%%ymm6,%%ymm0 \n"
"vpunpckhbw %%ymm5,%%ymm6,%%ymm1 \n"
@@ -7534,7 +7321,7 @@
"lea 0x20(%0),%0 \n"
"sub $0x8,%2 \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: "+r"(src_argb), // %0
"+r"(dst_argb), // %1
"+r"(width) // %2
@@ -7553,7 +7340,7 @@
asm volatile(
// 4 pixel loop.
LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n"
"movzb 0x03(%0),%3 \n"
"punpcklbw %%xmm0,%%xmm0 \n"
@@ -7604,7 +7391,7 @@
// 8 pixel loop.
LABELALIGN
- "1: \n"
+ "1: \n"
// replace VPGATHER
"movzb 0x03(%0),%3 \n"
"vmovd 0x00(%4,%3,4),%%xmm0 \n"
@@ -7645,7 +7432,7 @@
"lea 0x20(%0),%0 \n"
"sub $0x8,%2 \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: "+r"(src_argb), // %0
"+r"(dst_argb), // %1
"+r"(width), // %2
@@ -7666,7 +7453,7 @@
// 8 pixel loop.
LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n"
"movdqu 0x10(%0),%%xmm1 \n"
"psubb %%xmm5,%%xmm0 \n"
@@ -7729,7 +7516,7 @@
// 8 pixel loop.
LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n"
"movdqu 0x10(%0),%%xmm6 \n"
"pmaddubsw %%xmm2,%%xmm0 \n"
@@ -7792,7 +7579,7 @@
// 8 pixel loop.
LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n"
"movdqu 0x10(%0),%%xmm7 \n"
"pmaddubsw %%xmm2,%%xmm0 \n"
@@ -7864,7 +7651,7 @@
// 4 pixel loop.
LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n"
"punpcklbw %%xmm5,%%xmm0 \n"
"pmulhuw %%xmm2,%%xmm0 \n"
@@ -7906,7 +7693,7 @@
// 4 pixel loop.
LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n"
"lea 0x10(%0),%0 \n"
"movdqa %%xmm0,%%xmm1 \n"
@@ -7935,36 +7722,34 @@
const uint8_t* src_argb1,
uint8_t* dst_argb,
int width) {
- asm volatile(
+ asm volatile("pxor %%xmm5,%%xmm5 \n"
- "pxor %%xmm5,%%xmm5 \n"
-
- // 4 pixel loop.
- LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "lea 0x10(%0),%0 \n"
- "movdqu (%1),%%xmm2 \n"
- "lea 0x10(%1),%1 \n"
- "movdqu %%xmm0,%%xmm1 \n"
- "movdqu %%xmm2,%%xmm3 \n"
- "punpcklbw %%xmm0,%%xmm0 \n"
- "punpckhbw %%xmm1,%%xmm1 \n"
- "punpcklbw %%xmm5,%%xmm2 \n"
- "punpckhbw %%xmm5,%%xmm3 \n"
- "pmulhuw %%xmm2,%%xmm0 \n"
- "pmulhuw %%xmm3,%%xmm1 \n"
- "packuswb %%xmm1,%%xmm0 \n"
- "movdqu %%xmm0,(%2) \n"
- "lea 0x10(%2),%2 \n"
- "sub $0x4,%3 \n"
- "jg 1b \n"
- : "+r"(src_argb), // %0
- "+r"(src_argb1), // %1
- "+r"(dst_argb), // %2
- "+r"(width) // %3
- :
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm5");
+ // 4 pixel loop.
+ LABELALIGN
+ "1: \n"
+ "movdqu (%0),%%xmm0 \n"
+ "lea 0x10(%0),%0 \n"
+ "movdqu (%1),%%xmm2 \n"
+ "lea 0x10(%1),%1 \n"
+ "movdqu %%xmm0,%%xmm1 \n"
+ "movdqu %%xmm2,%%xmm3 \n"
+ "punpcklbw %%xmm0,%%xmm0 \n"
+ "punpckhbw %%xmm1,%%xmm1 \n"
+ "punpcklbw %%xmm5,%%xmm2 \n"
+ "punpckhbw %%xmm5,%%xmm3 \n"
+ "pmulhuw %%xmm2,%%xmm0 \n"
+ "pmulhuw %%xmm3,%%xmm1 \n"
+ "packuswb %%xmm1,%%xmm0 \n"
+ "movdqu %%xmm0,(%2) \n"
+ "lea 0x10(%2),%2 \n"
+ "sub $0x4,%3 \n"
+ "jg 1b \n"
+ : "+r"(src_argb), // %0
+ "+r"(src_argb1), // %1
+ "+r"(dst_argb), // %2
+ "+r"(width) // %3
+ :
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm5");
}
#endif // HAS_ARGBMULTIPLYROW_SSE2
@@ -7974,35 +7759,33 @@
const uint8_t* src_argb1,
uint8_t* dst_argb,
int width) {
- asm volatile(
+ asm volatile("vpxor %%ymm5,%%ymm5,%%ymm5 \n"
- "vpxor %%ymm5,%%ymm5,%%ymm5 \n"
-
- // 4 pixel loop.
- LABELALIGN
- "1: \n"
- "vmovdqu (%0),%%ymm1 \n"
- "lea 0x20(%0),%0 \n"
- "vmovdqu (%1),%%ymm3 \n"
- "lea 0x20(%1),%1 \n"
- "vpunpcklbw %%ymm1,%%ymm1,%%ymm0 \n"
- "vpunpckhbw %%ymm1,%%ymm1,%%ymm1 \n"
- "vpunpcklbw %%ymm5,%%ymm3,%%ymm2 \n"
- "vpunpckhbw %%ymm5,%%ymm3,%%ymm3 \n"
- "vpmulhuw %%ymm2,%%ymm0,%%ymm0 \n"
- "vpmulhuw %%ymm3,%%ymm1,%%ymm1 \n"
- "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n"
- "vmovdqu %%ymm0,(%2) \n"
- "lea 0x20(%2),%2 \n"
- "sub $0x8,%3 \n"
- "jg 1b \n"
- "vzeroupper \n"
- : "+r"(src_argb), // %0
- "+r"(src_argb1), // %1
- "+r"(dst_argb), // %2
- "+r"(width) // %3
- :
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm5");
+ // 4 pixel loop.
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%ymm1 \n"
+ "lea 0x20(%0),%0 \n"
+ "vmovdqu (%1),%%ymm3 \n"
+ "lea 0x20(%1),%1 \n"
+ "vpunpcklbw %%ymm1,%%ymm1,%%ymm0 \n"
+ "vpunpckhbw %%ymm1,%%ymm1,%%ymm1 \n"
+ "vpunpcklbw %%ymm5,%%ymm3,%%ymm2 \n"
+ "vpunpckhbw %%ymm5,%%ymm3,%%ymm3 \n"
+ "vpmulhuw %%ymm2,%%ymm0,%%ymm0 \n"
+ "vpmulhuw %%ymm3,%%ymm1,%%ymm1 \n"
+ "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm0,(%2) \n"
+ "lea 0x20(%2),%2 \n"
+ "sub $0x8,%3 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_argb), // %0
+ "+r"(src_argb1), // %1
+ "+r"(dst_argb), // %2
+ "+r"(width) // %3
+ :
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm5");
}
#endif // HAS_ARGBMULTIPLYROW_AVX2
@@ -8015,7 +7798,7 @@
asm volatile(
// 4 pixel loop.
LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n"
"lea 0x10(%0),%0 \n"
"movdqu (%1),%%xmm1 \n"
@@ -8043,7 +7826,7 @@
asm volatile(
// 4 pixel loop.
LABELALIGN
- "1: \n"
+ "1: \n"
"vmovdqu (%0),%%ymm0 \n"
"lea 0x20(%0),%0 \n"
"vpaddusb (%1),%%ymm0,%%ymm0 \n"
@@ -8052,7 +7835,7 @@
"lea 0x20(%2),%2 \n"
"sub $0x8,%3 \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: "+r"(src_argb), // %0
"+r"(src_argb1), // %1
"+r"(dst_argb), // %2
@@ -8071,7 +7854,7 @@
asm volatile(
// 4 pixel loop.
LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n"
"lea 0x10(%0),%0 \n"
"movdqu (%1),%%xmm1 \n"
@@ -8099,7 +7882,7 @@
asm volatile(
// 4 pixel loop.
LABELALIGN
- "1: \n"
+ "1: \n"
"vmovdqu (%0),%%ymm0 \n"
"lea 0x20(%0),%0 \n"
"vpsubusb (%1),%%ymm0,%%ymm0 \n"
@@ -8108,7 +7891,7 @@
"lea 0x20(%2),%2 \n"
"sub $0x8,%3 \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: "+r"(src_argb), // %0
"+r"(src_argb1), // %1
"+r"(dst_argb), // %2
@@ -8136,7 +7919,7 @@
// 8 pixel loop.
LABELALIGN
- "1: \n"
+ "1: \n"
"movq (%0),%%xmm0 \n"
"movq 0x2(%0),%%xmm1 \n"
"punpcklbw %%xmm5,%%xmm0 \n"
@@ -8189,7 +7972,7 @@
// 8 pixel loop.
LABELALIGN
- "1: \n"
+ "1: \n"
"movq (%0),%%xmm0 \n"
"movq 0x00(%0,%1,1),%%xmm1 \n"
"punpcklbw %%xmm5,%%xmm0 \n"
@@ -8242,7 +8025,7 @@
// 8 pixel loop.
LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n"
"movdqu 0x00(%0,%1,1),%%xmm1 \n"
"lea 0x10(%0),%0 \n"
@@ -8289,7 +8072,7 @@
// 8 pixel loop.
LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n"
"movdqu 0x00(%0,%1,1),%%xmm1 \n"
"lea 0x10(%0),%0 \n"
@@ -8323,7 +8106,7 @@
// 8 pixel loop.
LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n"
"movdqu 0x00(%0,%1,1),%%xmm1 \n"
"lea 0x10(%0),%0 \n"
@@ -8375,7 +8158,7 @@
// 4 pixel loop.
LABELALIGN
- "40: \n"
+ "40: \n"
"movdqu (%0),%%xmm2 \n"
"lea 0x10(%0),%0 \n"
"movdqa %%xmm2,%%xmm4 \n"
@@ -8408,13 +8191,13 @@
"sub $0x4,%3 \n"
"jge 40b \n"
- "49: \n"
+ "49: \n"
"add $0x3,%3 \n"
"jl 19f \n"
// 1 pixel loop.
LABELALIGN
- "10: \n"
+ "10: \n"
"movd (%0),%%xmm2 \n"
"lea 0x4(%0),%0 \n"
"punpcklbw %%xmm1,%%xmm2 \n"
@@ -8428,7 +8211,7 @@
"sub $0x1,%3 \n"
"jge 10b \n"
- "19: \n"
+ "19: \n"
: "+r"(row), // %0
"+r"(cumsum), // %1
"+r"(previous_cumsum), // %2
@@ -8466,7 +8249,7 @@
// 4 pixel small loop.
LABELALIGN
- "4: \n"
+ "4: \n"
"movdqu (%0),%%xmm0 \n"
"movdqu 0x10(%0),%%xmm1 \n"
"movdqu 0x20(%0),%%xmm2 \n"
@@ -8498,7 +8281,7 @@
// 4 pixel loop
LABELALIGN
- "40: \n"
+ "40: \n"
"movdqu (%0),%%xmm0 \n"
"movdqu 0x10(%0),%%xmm1 \n"
"movdqu 0x20(%0),%%xmm2 \n"
@@ -8537,13 +8320,13 @@
"sub $0x4,%3 \n"
"jge 40b \n"
- "49: \n"
+ "49: \n"
"add $0x3,%3 \n"
"jl 19f \n"
// 1 pixel loop
LABELALIGN
- "10: \n"
+ "10: \n"
"movdqu (%0),%%xmm0 \n"
"psubd 0x00(%0,%4,4),%%xmm0 \n"
"lea 0x10(%0),%0 \n"
@@ -8559,7 +8342,7 @@
"lea 0x4(%2),%2 \n"
"sub $0x1,%3 \n"
"jge 10b \n"
- "19: \n"
+ "19: \n"
: "+r"(topleft), // %0
"+r"(botleft), // %1
"+r"(dst), // %2
@@ -8602,7 +8385,7 @@
// 4 pixel loop
LABELALIGN
- "40: \n"
+ "40: \n"
"cvttps2dq %%xmm2,%%xmm0 \n" // x,y float->int first 2
"cvttps2dq %%xmm3,%%xmm1 \n" // x,y float->int next 2
"packssdw %%xmm1,%%xmm0 \n" // x, y as 8 shorts
@@ -8628,13 +8411,13 @@
"sub $0x4,%4 \n"
"jge 40b \n"
- "49: \n"
+ "49: \n"
"add $0x3,%4 \n"
"jl 19f \n"
// 1 pixel loop
LABELALIGN
- "10: \n"
+ "10: \n"
"cvttps2dq %%xmm2,%%xmm0 \n"
"packssdw %%xmm0,%%xmm0 \n"
"pmaddwd %%xmm5,%%xmm0 \n"
@@ -8645,7 +8428,7 @@
"lea 0x04(%2),%2 \n"
"sub $0x1,%4 \n"
"jge 10b \n"
- "19: \n"
+ "19: \n"
: "+r"(src_argb), // %0
"+r"(src_argb_stride_temp), // %1
"+r"(dst_argb), // %2
@@ -8685,7 +8468,7 @@
// General purpose row blend.
LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%1),%%xmm0 \n"
"movdqu 0x00(%1,%4,1),%%xmm2 \n"
"movdqa %%xmm0,%%xmm1 \n"
@@ -8710,7 +8493,7 @@
// Blend 50 / 50.
LABELALIGN
- "50: \n"
+ "50: \n"
"movdqu (%1),%%xmm0 \n"
"movdqu 0x00(%1,%4,1),%%xmm1 \n"
"pavgb %%xmm1,%%xmm0 \n"
@@ -8722,14 +8505,14 @@
// Blend 100 / 0 - Copy row unchanged.
LABELALIGN
- "100: \n"
+ "100: \n"
"movdqu (%1),%%xmm0 \n"
"movdqu %%xmm0,0x00(%1,%0,1) \n"
"lea 0x10(%1),%1 \n"
"sub $0x10,%2 \n"
"jg 100b \n"
- "99: \n"
+ "99: \n"
: "+r"(dst_ptr), // %0
"+r"(src_ptr), // %1
"+rm"(width), // %2
@@ -8766,7 +8549,7 @@
// General purpose row blend.
LABELALIGN
- "1: \n"
+ "1: \n"
"vmovdqu (%1),%%ymm0 \n"
"vmovdqu 0x00(%1,%4,1),%%ymm2 \n"
"vpunpckhbw %%ymm2,%%ymm0,%%ymm1 \n"
@@ -8788,7 +8571,7 @@
// Blend 50 / 50.
LABELALIGN
- "50: \n"
+ "50: \n"
"vmovdqu (%1),%%ymm0 \n"
"vpavgb 0x00(%1,%4,1),%%ymm0,%%ymm0 \n"
"vmovdqu %%ymm0,0x00(%1,%0,1) \n"
@@ -8799,15 +8582,15 @@
// Blend 100 / 0 - Copy row unchanged.
LABELALIGN
- "100: \n"
+ "100: \n"
"vmovdqu (%1),%%ymm0 \n"
"vmovdqu %%ymm0,0x00(%1,%0,1) \n"
"lea 0x20(%1),%1 \n"
"sub $0x20,%2 \n"
"jg 100b \n"
- "99: \n"
- "vzeroupper \n"
+ "99: \n"
+ "vzeroupper \n"
: "+r"(dst_ptr), // %0
"+r"(src_ptr), // %1
"+r"(width), // %2
@@ -8823,27 +8606,25 @@
uint8_t* dst_argb,
const uint8_t* shuffler,
int width) {
- asm volatile(
+ asm volatile("movdqu (%3),%%xmm5 \n"
- "movdqu (%3),%%xmm5 \n"
-
- LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x10(%0),%%xmm1 \n"
- "lea 0x20(%0),%0 \n"
- "pshufb %%xmm5,%%xmm0 \n"
- "pshufb %%xmm5,%%xmm1 \n"
- "movdqu %%xmm0,(%1) \n"
- "movdqu %%xmm1,0x10(%1) \n"
- "lea 0x20(%1),%1 \n"
- "sub $0x8,%2 \n"
- "jg 1b \n"
- : "+r"(src_argb), // %0
- "+r"(dst_argb), // %1
- "+r"(width) // %2
- : "r"(shuffler) // %3
- : "memory", "cc", "xmm0", "xmm1", "xmm5");
+ LABELALIGN
+ "1: \n"
+ "movdqu (%0),%%xmm0 \n"
+ "movdqu 0x10(%0),%%xmm1 \n"
+ "lea 0x20(%0),%0 \n"
+ "pshufb %%xmm5,%%xmm0 \n"
+ "pshufb %%xmm5,%%xmm1 \n"
+ "movdqu %%xmm0,(%1) \n"
+ "movdqu %%xmm1,0x10(%1) \n"
+ "lea 0x20(%1),%1 \n"
+ "sub $0x8,%2 \n"
+ "jg 1b \n"
+ : "+r"(src_argb), // %0
+ "+r"(dst_argb), // %1
+ "+r"(width) // %2
+ : "r"(shuffler) // %3
+ : "memory", "cc", "xmm0", "xmm1", "xmm5");
}
#endif // HAS_ARGBSHUFFLEROW_SSSE3
@@ -8853,28 +8634,26 @@
uint8_t* dst_argb,
const uint8_t* shuffler,
int width) {
- asm volatile(
+ asm volatile("vbroadcastf128 (%3),%%ymm5 \n"
- "vbroadcastf128 (%3),%%ymm5 \n"
-
- LABELALIGN
- "1: \n"
- "vmovdqu (%0),%%ymm0 \n"
- "vmovdqu 0x20(%0),%%ymm1 \n"
- "lea 0x40(%0),%0 \n"
- "vpshufb %%ymm5,%%ymm0,%%ymm0 \n"
- "vpshufb %%ymm5,%%ymm1,%%ymm1 \n"
- "vmovdqu %%ymm0,(%1) \n"
- "vmovdqu %%ymm1,0x20(%1) \n"
- "lea 0x40(%1),%1 \n"
- "sub $0x10,%2 \n"
- "jg 1b \n"
- "vzeroupper \n"
- : "+r"(src_argb), // %0
- "+r"(dst_argb), // %1
- "+r"(width) // %2
- : "r"(shuffler) // %3
- : "memory", "cc", "xmm0", "xmm1", "xmm5");
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vmovdqu 0x20(%0),%%ymm1 \n"
+ "lea 0x40(%0),%0 \n"
+ "vpshufb %%ymm5,%%ymm0,%%ymm0 \n"
+ "vpshufb %%ymm5,%%ymm1,%%ymm1 \n"
+ "vmovdqu %%ymm0,(%1) \n"
+ "vmovdqu %%ymm1,0x20(%1) \n"
+ "lea 0x40(%1),%1 \n"
+ "sub $0x10,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_argb), // %0
+ "+r"(dst_argb), // %1
+ "+r"(width) // %2
+ : "r"(shuffler) // %3
+ : "memory", "cc", "xmm0", "xmm1", "xmm5");
}
#endif // HAS_ARGBSHUFFLEROW_AVX2
@@ -8884,33 +8663,31 @@
const uint8_t* src_v,
uint8_t* dst_yuy2,
int width) {
- asm volatile(
+ asm volatile("sub %1,%2 \n"
- "sub %1,%2 \n"
-
- LABELALIGN
- "1: \n"
- "movq (%1),%%xmm2 \n"
- "movq 0x00(%1,%2,1),%%xmm1 \n"
- "add $0x8,%1 \n"
- "punpcklbw %%xmm1,%%xmm2 \n"
- "movdqu (%0),%%xmm0 \n"
- "add $0x10,%0 \n"
- "movdqa %%xmm0,%%xmm1 \n"
- "punpcklbw %%xmm2,%%xmm0 \n"
- "punpckhbw %%xmm2,%%xmm1 \n"
- "movdqu %%xmm0,(%3) \n"
- "movdqu %%xmm1,0x10(%3) \n"
- "lea 0x20(%3),%3 \n"
- "sub $0x10,%4 \n"
- "jg 1b \n"
- : "+r"(src_y), // %0
- "+r"(src_u), // %1
- "+r"(src_v), // %2
- "+r"(dst_yuy2), // %3
- "+rm"(width) // %4
- :
- : "memory", "cc", "xmm0", "xmm1", "xmm2");
+ LABELALIGN
+ "1: \n"
+ "movq (%1),%%xmm2 \n"
+ "movq 0x00(%1,%2,1),%%xmm1 \n"
+ "add $0x8,%1 \n"
+ "punpcklbw %%xmm1,%%xmm2 \n"
+ "movdqu (%0),%%xmm0 \n"
+ "add $0x10,%0 \n"
+ "movdqa %%xmm0,%%xmm1 \n"
+ "punpcklbw %%xmm2,%%xmm0 \n"
+ "punpckhbw %%xmm2,%%xmm1 \n"
+ "movdqu %%xmm0,(%3) \n"
+ "movdqu %%xmm1,0x10(%3) \n"
+ "lea 0x20(%3),%3 \n"
+ "sub $0x10,%4 \n"
+ "jg 1b \n"
+ : "+r"(src_y), // %0
+ "+r"(src_u), // %1
+ "+r"(src_v), // %2
+ "+r"(dst_yuy2), // %3
+ "+rm"(width) // %4
+ :
+ : "memory", "cc", "xmm0", "xmm1", "xmm2");
}
#endif // HAS_I422TOYUY2ROW_SSE2
@@ -8920,33 +8697,31 @@
const uint8_t* src_v,
uint8_t* dst_uyvy,
int width) {
- asm volatile(
+ asm volatile("sub %1,%2 \n"
- "sub %1,%2 \n"
-
- LABELALIGN
- "1: \n"
- "movq (%1),%%xmm2 \n"
- "movq 0x00(%1,%2,1),%%xmm1 \n"
- "add $0x8,%1 \n"
- "punpcklbw %%xmm1,%%xmm2 \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqa %%xmm2,%%xmm1 \n"
- "add $0x10,%0 \n"
- "punpcklbw %%xmm0,%%xmm1 \n"
- "punpckhbw %%xmm0,%%xmm2 \n"
- "movdqu %%xmm1,(%3) \n"
- "movdqu %%xmm2,0x10(%3) \n"
- "lea 0x20(%3),%3 \n"
- "sub $0x10,%4 \n"
- "jg 1b \n"
- : "+r"(src_y), // %0
- "+r"(src_u), // %1
- "+r"(src_v), // %2
- "+r"(dst_uyvy), // %3
- "+rm"(width) // %4
- :
- : "memory", "cc", "xmm0", "xmm1", "xmm2");
+ LABELALIGN
+ "1: \n"
+ "movq (%1),%%xmm2 \n"
+ "movq 0x00(%1,%2,1),%%xmm1 \n"
+ "add $0x8,%1 \n"
+ "punpcklbw %%xmm1,%%xmm2 \n"
+ "movdqu (%0),%%xmm0 \n"
+ "movdqa %%xmm2,%%xmm1 \n"
+ "add $0x10,%0 \n"
+ "punpcklbw %%xmm0,%%xmm1 \n"
+ "punpckhbw %%xmm0,%%xmm2 \n"
+ "movdqu %%xmm1,(%3) \n"
+ "movdqu %%xmm2,0x10(%3) \n"
+ "lea 0x20(%3),%3 \n"
+ "sub $0x10,%4 \n"
+ "jg 1b \n"
+ : "+r"(src_y), // %0
+ "+r"(src_u), // %1
+ "+r"(src_v), // %2
+ "+r"(dst_uyvy), // %3
+ "+rm"(width) // %4
+ :
+ : "memory", "cc", "xmm0", "xmm1", "xmm2");
}
#endif // HAS_I422TOUYVYROW_SSE2
@@ -8956,36 +8731,34 @@
const uint8_t* src_v,
uint8_t* dst_yuy2,
int width) {
- asm volatile(
+ asm volatile("sub %1,%2 \n"
- "sub %1,%2 \n"
-
- LABELALIGN
- "1: \n"
- "vpmovzxbw (%1),%%ymm1 \n"
- "vpmovzxbw 0x00(%1,%2,1),%%ymm2 \n"
- "add $0x10,%1 \n"
- "vpsllw $0x8,%%ymm2,%%ymm2 \n"
- "vpor %%ymm1,%%ymm2,%%ymm2 \n"
- "vmovdqu (%0),%%ymm0 \n"
- "add $0x20,%0 \n"
- "vpunpcklbw %%ymm2,%%ymm0,%%ymm1 \n"
- "vpunpckhbw %%ymm2,%%ymm0,%%ymm2 \n"
- "vextractf128 $0x0,%%ymm1,(%3) \n"
- "vextractf128 $0x0,%%ymm2,0x10(%3) \n"
- "vextractf128 $0x1,%%ymm1,0x20(%3) \n"
- "vextractf128 $0x1,%%ymm2,0x30(%3) \n"
- "lea 0x40(%3),%3 \n"
- "sub $0x20,%4 \n"
- "jg 1b \n"
- "vzeroupper \n"
- : "+r"(src_y), // %0
- "+r"(src_u), // %1
- "+r"(src_v), // %2
- "+r"(dst_yuy2), // %3
- "+rm"(width) // %4
- :
- : "memory", "cc", "xmm0", "xmm1", "xmm2");
+ LABELALIGN
+ "1: \n"
+ "vpmovzxbw (%1),%%ymm1 \n"
+ "vpmovzxbw 0x00(%1,%2,1),%%ymm2 \n"
+ "add $0x10,%1 \n"
+ "vpsllw $0x8,%%ymm2,%%ymm2 \n"
+ "vpor %%ymm1,%%ymm2,%%ymm2 \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "add $0x20,%0 \n"
+ "vpunpcklbw %%ymm2,%%ymm0,%%ymm1 \n"
+ "vpunpckhbw %%ymm2,%%ymm0,%%ymm2 \n"
+ "vextractf128 $0x0,%%ymm1,(%3) \n"
+ "vextractf128 $0x0,%%ymm2,0x10(%3) \n"
+ "vextractf128 $0x1,%%ymm1,0x20(%3) \n"
+ "vextractf128 $0x1,%%ymm2,0x30(%3) \n"
+ "lea 0x40(%3),%3 \n"
+ "sub $0x20,%4 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_y), // %0
+ "+r"(src_u), // %1
+ "+r"(src_v), // %2
+ "+r"(dst_yuy2), // %3
+ "+rm"(width) // %4
+ :
+ : "memory", "cc", "xmm0", "xmm1", "xmm2");
}
#endif // HAS_I422TOYUY2ROW_AVX2
@@ -8995,36 +8768,34 @@
const uint8_t* src_v,
uint8_t* dst_uyvy,
int width) {
- asm volatile(
+ asm volatile("sub %1,%2 \n"
- "sub %1,%2 \n"
-
- LABELALIGN
- "1: \n"
- "vpmovzxbw (%1),%%ymm1 \n"
- "vpmovzxbw 0x00(%1,%2,1),%%ymm2 \n"
- "add $0x10,%1 \n"
- "vpsllw $0x8,%%ymm2,%%ymm2 \n"
- "vpor %%ymm1,%%ymm2,%%ymm2 \n"
- "vmovdqu (%0),%%ymm0 \n"
- "add $0x20,%0 \n"
- "vpunpcklbw %%ymm0,%%ymm2,%%ymm1 \n"
- "vpunpckhbw %%ymm0,%%ymm2,%%ymm2 \n"
- "vextractf128 $0x0,%%ymm1,(%3) \n"
- "vextractf128 $0x0,%%ymm2,0x10(%3) \n"
- "vextractf128 $0x1,%%ymm1,0x20(%3) \n"
- "vextractf128 $0x1,%%ymm2,0x30(%3) \n"
- "lea 0x40(%3),%3 \n"
- "sub $0x20,%4 \n"
- "jg 1b \n"
- "vzeroupper \n"
- : "+r"(src_y), // %0
- "+r"(src_u), // %1
- "+r"(src_v), // %2
- "+r"(dst_uyvy), // %3
- "+rm"(width) // %4
- :
- : "memory", "cc", "xmm0", "xmm1", "xmm2");
+ LABELALIGN
+ "1: \n"
+ "vpmovzxbw (%1),%%ymm1 \n"
+ "vpmovzxbw 0x00(%1,%2,1),%%ymm2 \n"
+ "add $0x10,%1 \n"
+ "vpsllw $0x8,%%ymm2,%%ymm2 \n"
+ "vpor %%ymm1,%%ymm2,%%ymm2 \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "add $0x20,%0 \n"
+ "vpunpcklbw %%ymm0,%%ymm2,%%ymm1 \n"
+ "vpunpckhbw %%ymm0,%%ymm2,%%ymm2 \n"
+ "vextractf128 $0x0,%%ymm1,(%3) \n"
+ "vextractf128 $0x0,%%ymm2,0x10(%3) \n"
+ "vextractf128 $0x1,%%ymm1,0x20(%3) \n"
+ "vextractf128 $0x1,%%ymm2,0x30(%3) \n"
+ "lea 0x40(%3),%3 \n"
+ "sub $0x20,%4 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_y), // %0
+ "+r"(src_u), // %1
+ "+r"(src_v), // %2
+ "+r"(dst_uyvy), // %3
+ "+rm"(width) // %4
+ :
+ : "memory", "cc", "xmm0", "xmm1", "xmm2");
}
#endif // HAS_I422TOUYVYROW_AVX2
@@ -9033,54 +8804,53 @@
uint8_t* dst_argb,
const float* poly,
int width) {
- asm volatile(
+ asm volatile("pxor %%xmm3,%%xmm3 \n"
- "pxor %%xmm3,%%xmm3 \n"
-
- // 2 pixel loop.
- LABELALIGN
- "1: \n"
- "movq (%0),%%xmm0 \n"
- "lea 0x8(%0),%0 \n"
- "punpcklbw %%xmm3,%%xmm0 \n"
- "movdqa %%xmm0,%%xmm4 \n"
- "punpcklwd %%xmm3,%%xmm0 \n"
- "punpckhwd %%xmm3,%%xmm4 \n"
- "cvtdq2ps %%xmm0,%%xmm0 \n"
- "cvtdq2ps %%xmm4,%%xmm4 \n"
- "movdqa %%xmm0,%%xmm1 \n"
- "movdqa %%xmm4,%%xmm5 \n"
- "mulps 0x10(%3),%%xmm0 \n"
- "mulps 0x10(%3),%%xmm4 \n"
- "addps (%3),%%xmm0 \n"
- "addps (%3),%%xmm4 \n"
- "movdqa %%xmm1,%%xmm2 \n"
- "movdqa %%xmm5,%%xmm6 \n"
- "mulps %%xmm1,%%xmm2 \n"
- "mulps %%xmm5,%%xmm6 \n"
- "mulps %%xmm2,%%xmm1 \n"
- "mulps %%xmm6,%%xmm5 \n"
- "mulps 0x20(%3),%%xmm2 \n"
- "mulps 0x20(%3),%%xmm6 \n"
- "mulps 0x30(%3),%%xmm1 \n"
- "mulps 0x30(%3),%%xmm5 \n"
- "addps %%xmm2,%%xmm0 \n"
- "addps %%xmm6,%%xmm4 \n"
- "addps %%xmm1,%%xmm0 \n"
- "addps %%xmm5,%%xmm4 \n"
- "cvttps2dq %%xmm0,%%xmm0 \n"
- "cvttps2dq %%xmm4,%%xmm4 \n"
- "packuswb %%xmm4,%%xmm0 \n"
- "packuswb %%xmm0,%%xmm0 \n"
- "movq %%xmm0,(%1) \n"
- "lea 0x8(%1),%1 \n"
- "sub $0x2,%2 \n"
- "jg 1b \n"
- : "+r"(src_argb), // %0
- "+r"(dst_argb), // %1
- "+r"(width) // %2
- : "r"(poly) // %3
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6");
+ // 2 pixel loop.
+ LABELALIGN
+ "1: \n"
+ "movq (%0),%%xmm0 \n"
+ "lea 0x8(%0),%0 \n"
+ "punpcklbw %%xmm3,%%xmm0 \n"
+ "movdqa %%xmm0,%%xmm4 \n"
+ "punpcklwd %%xmm3,%%xmm0 \n"
+ "punpckhwd %%xmm3,%%xmm4 \n"
+ "cvtdq2ps %%xmm0,%%xmm0 \n"
+ "cvtdq2ps %%xmm4,%%xmm4 \n"
+ "movdqa %%xmm0,%%xmm1 \n"
+ "movdqa %%xmm4,%%xmm5 \n"
+ "mulps 0x10(%3),%%xmm0 \n"
+ "mulps 0x10(%3),%%xmm4 \n"
+ "addps (%3),%%xmm0 \n"
+ "addps (%3),%%xmm4 \n"
+ "movdqa %%xmm1,%%xmm2 \n"
+ "movdqa %%xmm5,%%xmm6 \n"
+ "mulps %%xmm1,%%xmm2 \n"
+ "mulps %%xmm5,%%xmm6 \n"
+ "mulps %%xmm2,%%xmm1 \n"
+ "mulps %%xmm6,%%xmm5 \n"
+ "mulps 0x20(%3),%%xmm2 \n"
+ "mulps 0x20(%3),%%xmm6 \n"
+ "mulps 0x30(%3),%%xmm1 \n"
+ "mulps 0x30(%3),%%xmm5 \n"
+ "addps %%xmm2,%%xmm0 \n"
+ "addps %%xmm6,%%xmm4 \n"
+ "addps %%xmm1,%%xmm0 \n"
+ "addps %%xmm5,%%xmm4 \n"
+ "cvttps2dq %%xmm0,%%xmm0 \n"
+ "cvttps2dq %%xmm4,%%xmm4 \n"
+ "packuswb %%xmm4,%%xmm0 \n"
+ "packuswb %%xmm0,%%xmm0 \n"
+ "movq %%xmm0,(%1) \n"
+ "lea 0x8(%1),%1 \n"
+ "sub $0x2,%2 \n"
+ "jg 1b \n"
+ : "+r"(src_argb), // %0
+ "+r"(dst_argb), // %1
+ "+r"(width) // %2
+ : "r"(poly) // %3
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
+ "xmm6");
}
#endif // HAS_ARGBPOLYNOMIALROW_SSE2
@@ -9097,7 +8867,7 @@
// 2 pixel loop.
LABELALIGN
- "1: \n"
+ "1: \n"
"vpmovzxbd (%0),%%ymm0 \n" // 2 ARGB pixels
"lea 0x8(%0),%0 \n"
"vcvtdq2ps %%ymm0,%%ymm0 \n" // X 8 floats
@@ -9115,7 +8885,7 @@
"lea 0x8(%1),%1 \n"
"sub $0x2,%2 \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: "+r"(src_argb), // %0
"+r"(dst_argb), // %1
"+r"(width) // %2
@@ -9140,7 +8910,7 @@
// 16 pixel loop.
LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm2 \n" // 8 shorts
"add $0x10,%0 \n"
"movdqa %%xmm2,%%xmm3 \n"
@@ -9177,7 +8947,7 @@
// 16 pixel loop.
LABELALIGN
- "1: \n"
+ "1: \n"
"vmovdqu (%0),%%ymm2 \n" // 16 shorts
"add $0x20,%0 \n"
"vpunpckhwd %%ymm5,%%ymm2,%%ymm3 \n" // mutates
@@ -9193,14 +8963,14 @@
"sub $0x10,%2 \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: "+r"(src), // %0
"+r"(dst), // %1
"+r"(width) // %2
#if defined(__x86_64__)
: "x"(scale) // %3
#else
- : "m"(scale) // %3
+ : "m"(scale) // %3
#endif
: "memory", "cc", "xmm2", "xmm3", "xmm4", "xmm5");
}
@@ -9217,7 +8987,7 @@
// 16 pixel loop.
LABELALIGN
- "1: \n"
+ "1: \n"
"vpmovzxwd (%0),%%ymm2 \n" // 16 shorts -> 16 ints
"vpmovzxwd 0x10(%0),%%ymm3 \n"
"vcvtdq2ps %%ymm2,%%ymm2 \n"
@@ -9231,14 +9001,14 @@
"add $0x20,%0 \n"
"sub $0x10,%2 \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: "+r"(src), // %0
"+r"(dst), // %1
"+r"(width) // %2
#if defined(__x86_64__)
: "x"(scale) // %3
#else
- : "m"(scale) // %3
+ : "m"(scale) // %3
#endif
: "memory", "cc", "xmm2", "xmm3", "xmm4");
}
@@ -9250,7 +9020,7 @@
"sub %0,%1 \n"
// 16 pixel loop.
LABELALIGN
- "1: \n"
+ "1: \n"
"vpmovzxwd (%0),%%ymm2 \n" // 16 shorts -> 16 ints
"vpmovzxwd 0x10(%0),%%ymm3 \n"
"vcvtdq2ps %%ymm2,%%ymm2 \n"
@@ -9262,7 +9032,7 @@
"add $0x20,%0 \n"
"sub $0x10,%2 \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: "+r"(src), // %0
"+r"(dst), // %1
"+r"(width) // %2
@@ -9280,7 +9050,7 @@
asm volatile(
// 1 pixel loop.
LABELALIGN
- "1: \n"
+ "1: \n"
"movzb (%0),%1 \n"
"lea 0x4(%0),%0 \n"
"movzb 0x00(%3,%1,4),%1 \n"
@@ -9313,7 +9083,7 @@
asm volatile(
// 1 pixel loop.
LABELALIGN
- "1: \n"
+ "1: \n"
"movzb (%0),%1 \n"
"lea 0x4(%0),%0 \n"
"movzb 0x00(%3,%1,4),%1 \n"
@@ -9352,7 +9122,7 @@
// 4 pixel loop.
LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%2),%%xmm0 \n"
"pmaddubsw %%xmm3,%%xmm0 \n"
"phaddw %%xmm0,%%xmm0 \n"
@@ -9452,7 +9222,7 @@
"movdqa (%4),%%xmm4 \n" // 3 shuffler constants
"movdqa 16(%4),%%xmm5 \n"
"movdqa 32(%4),%%xmm6 \n"
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm2 \n" // load 16 Y values
"movdqu (%0,%1),%%xmm3 \n" // load 8 VU values
"lea 16(%0),%0 \n"
@@ -9491,7 +9261,7 @@
"vbroadcastf128 16(%4),%%ymm5 \n"
"vbroadcastf128 32(%4),%%ymm6 \n"
- "1: \n"
+ "1: \n"
"vmovdqu (%0),%%ymm2 \n" // load 32 Y values
"vmovdqu (%0,%1),%%ymm3 \n" // load 16 VU values
"lea 32(%0),%0 \n"
@@ -9510,7 +9280,7 @@
"lea 96(%2),%2 \n"
"sub $32,%3 \n" // 32 pixels per loop
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: "+r"(src_y), // %0
"+r"(src_vu), // %1
"+r"(dst_yuv24), // %2
@@ -9538,7 +9308,7 @@
"vmovdqa (%4),%%ymm4 \n" // 3 shuffler constants
"vmovdqa 32(%4),%%ymm5 \n"
"vmovdqa 64(%4),%%ymm6 \n" LABELALIGN
- "1: \n"
+ "1: \n"
"vmovdqu (%0),%%ymm2 \n" // load 32 Y values
"vmovdqu (%0,%1),%%ymm3 \n" // load 16 VU values
"lea 32(%0),%0 \n"
@@ -9553,7 +9323,7 @@
"lea 96(%2),%2 \n"
"sub $32,%3 \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: "+r"(src_y), // %0
"+r"(src_vu), // %1
"+r"(dst_yuv24), // %2
@@ -9572,54 +9342,50 @@
// Convert UV plane of NV12 to VU of NV21.
void SwapUVRow_SSSE3(const uint8_t* src_uv, uint8_t* dst_vu, int width) {
- asm volatile(
+ asm volatile("movdqu %3,%%xmm5 \n"
- "movdqu %3,%%xmm5 \n"
-
- LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm0 \n"
- "movdqu 0x10(%0),%%xmm1 \n"
- "lea 0x20(%0),%0 \n"
- "pshufb %%xmm5,%%xmm0 \n"
- "pshufb %%xmm5,%%xmm1 \n"
- "movdqu %%xmm0,(%1) \n"
- "movdqu %%xmm1,0x10(%1) \n"
- "lea 0x20(%1),%1 \n"
- "sub $0x10,%2 \n"
- "jg 1b \n"
- : "+r"(src_uv), // %0
- "+r"(dst_vu), // %1
- "+r"(width) // %2
- : "m"(kShuffleUVToVU) // %3
- : "memory", "cc", "xmm0", "xmm1", "xmm5");
+ LABELALIGN
+ "1: \n"
+ "movdqu (%0),%%xmm0 \n"
+ "movdqu 0x10(%0),%%xmm1 \n"
+ "lea 0x20(%0),%0 \n"
+ "pshufb %%xmm5,%%xmm0 \n"
+ "pshufb %%xmm5,%%xmm1 \n"
+ "movdqu %%xmm0,(%1) \n"
+ "movdqu %%xmm1,0x10(%1) \n"
+ "lea 0x20(%1),%1 \n"
+ "sub $0x10,%2 \n"
+ "jg 1b \n"
+ : "+r"(src_uv), // %0
+ "+r"(dst_vu), // %1
+ "+r"(width) // %2
+ : "m"(kShuffleUVToVU) // %3
+ : "memory", "cc", "xmm0", "xmm1", "xmm5");
}
#endif // HAS_SWAPUVROW_SSSE3
#ifdef HAS_SWAPUVROW_AVX2
void SwapUVRow_AVX2(const uint8_t* src_uv, uint8_t* dst_vu, int width) {
- asm volatile(
+ asm volatile("vbroadcastf128 %3,%%ymm5 \n"
- "vbroadcastf128 %3,%%ymm5 \n"
-
- LABELALIGN
- "1: \n"
- "vmovdqu (%0),%%ymm0 \n"
- "vmovdqu 0x20(%0),%%ymm1 \n"
- "lea 0x40(%0),%0 \n"
- "vpshufb %%ymm5,%%ymm0,%%ymm0 \n"
- "vpshufb %%ymm5,%%ymm1,%%ymm1 \n"
- "vmovdqu %%ymm0,(%1) \n"
- "vmovdqu %%ymm1,0x20(%1) \n"
- "lea 0x40(%1),%1 \n"
- "sub $0x20,%2 \n"
- "jg 1b \n"
- "vzeroupper \n"
- : "+r"(src_uv), // %0
- "+r"(dst_vu), // %1
- "+r"(width) // %2
- : "m"(kShuffleUVToVU) // %3
- : "memory", "cc", "xmm0", "xmm1", "xmm5");
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vmovdqu 0x20(%0),%%ymm1 \n"
+ "lea 0x40(%0),%0 \n"
+ "vpshufb %%ymm5,%%ymm0,%%ymm0 \n"
+ "vpshufb %%ymm5,%%ymm1,%%ymm1 \n"
+ "vmovdqu %%ymm0,(%1) \n"
+ "vmovdqu %%ymm1,0x20(%1) \n"
+ "lea 0x40(%1),%1 \n"
+ "sub $0x20,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_uv), // %0
+ "+r"(dst_vu), // %1
+ "+r"(width) // %2
+ : "m"(kShuffleUVToVU) // %3
+ : "memory", "cc", "xmm0", "xmm1", "xmm5");
}
#endif // HAS_SWAPUVROW_AVX2
@@ -9630,13 +9396,12 @@
uint8_t* dst_uv,
int width) {
asm volatile(
- "pcmpeqb %%xmm4,%%xmm4 \n"
- "psrlw $0xf,%%xmm4 \n"
- "packuswb %%xmm4,%%xmm4 \n"
+ "pcmpeqb %%xmm4,%%xmm4 \n" // 0x0101
+ "pabsb %%xmm4,%%xmm4 \n"
"pxor %%xmm5,%%xmm5 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n" // load 16 U values
"movdqu (%1),%%xmm1 \n" // load 16 V values
"movdqu 0(%0,%4,1),%%xmm2 \n" // 16 from next row
@@ -9677,12 +9442,11 @@
int width) {
asm volatile(
"vpcmpeqb %%ymm4,%%ymm4,%%ymm4 \n"
- "vpsrlw $0xf,%%ymm4,%%ymm4 \n"
- "vpackuswb %%ymm4,%%ymm4,%%ymm4 \n"
+ "vpabsb %%ymm4,%%ymm4 \n"
"vpxor %%ymm5,%%ymm5,%%ymm5 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"vmovdqu (%0),%%ymm0 \n" // load 32 U values
"vmovdqu (%1),%%ymm1 \n" // load 32 V values
"vmovdqu 0(%0,%4,1),%%ymm2 \n" // 32 from next row
@@ -9706,7 +9470,7 @@
"lea 0x20(%2),%2 \n"
"sub $0x20,%3 \n" // 32 src pixels per loop
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: "+r"(src_u), // %0
"+r"(src_v), // %1
"+r"(dst_uv), // %2
@@ -9721,7 +9485,7 @@
"pxor %%xmm1,%%xmm1 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"movd (%0),%%xmm0 \n" // load float
"maxss %%xmm1, %%xmm0 \n" // clamp to zero
"add 4, %0 \n"
@@ -9736,6 +9500,45 @@
: "memory", "cc", "xmm0", "xmm1");
}
+#ifdef HAS_CONVERT16TO8ROW_AVX2
+void Convert8To8Row_AVX2(const uint8_t* src_y,
+ uint8_t* dst_y,
+ int scale,
+ int bias,
+ int width) {
+ asm volatile(
+ "sub %0,%1 \n"
+ "vmovd %3,%%xmm2 \n"
+ "vmovd %4,%%xmm3 \n"
+ "vpbroadcastw %%xmm2,%%ymm2 \n"
+ "vpbroadcastb %%xmm3,%%ymm3 \n"
+ "vpxor %%ymm4,%%ymm4,%%ymm4 \n"
+ "vpsllw $8,%%ymm2,%%ymm2 \n"
+
+ // 32 pixels per loop.
+ LABELALIGN
+ "1: \n"
+ "vmovdqu (%0),%%ymm0 \n"
+ "vpunpckhbw %%ymm4,%%ymm0,%%ymm1 \n" // mutates
+ "vpunpcklbw %%ymm4,%%ymm0,%%ymm0 \n"
+ "vpmulhuw %%ymm2,%%ymm0,%%ymm0 \n"
+ "vpmulhuw %%ymm2,%%ymm1,%%ymm1 \n"
+ "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n" // unmutates
+ "vpaddb %%ymm3,%%ymm0,%%ymm0 \n"
+ "vmovdqu %%ymm0,(%0,%1) \n"
+ "add $0x20,%0 \n"
+ "sub $0x20,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
+ : "+r"(src_y), // %0
+ "+r"(dst_y), // %1
+ "+r"(width) // %2
+ : "r"(scale), // %3
+ "r"(bias) // %4
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4");
+}
+#endif // HAS_CONVERT16TO8ROW_AVX2
+
#endif // defined(__x86_64__) || defined(__i386__)
#ifdef __cplusplus
diff --git a/source/row_lasx.cc b/source/row_lasx.cc
index be85022..3613b0a 100644
--- a/source/row_lasx.cc
+++ b/source/row_lasx.cc
@@ -1148,24 +1148,26 @@
__m256i b, g, r, a, dst0, dst1;
__m256i control = {0x0005000100040000, 0x0007000300060002, 0x0005000100040000,
0x0007000300060002};
+ __m256i zero = __lasx_xvldi(0);
+ __m256i const_add = __lasx_xvldi(0x8ff);
for (x = 0; x < len; x++) {
DUP2_ARG2(__lasx_xvld, src_argb, 0, src_argb, 32, src0, src1);
tmp0 = __lasx_xvpickev_b(src1, src0);
tmp1 = __lasx_xvpickod_b(src1, src0);
- b = __lasx_xvpackev_b(tmp0, tmp0);
- r = __lasx_xvpackod_b(tmp0, tmp0);
- g = __lasx_xvpackev_b(tmp1, tmp1);
- a = __lasx_xvpackod_b(tmp1, tmp1);
- reg0 = __lasx_xvmulwev_w_hu(b, a);
- reg1 = __lasx_xvmulwod_w_hu(b, a);
- reg2 = __lasx_xvmulwev_w_hu(r, a);
- reg3 = __lasx_xvmulwod_w_hu(r, a);
- reg4 = __lasx_xvmulwev_w_hu(g, a);
- reg5 = __lasx_xvmulwod_w_hu(g, a);
- reg0 = __lasx_xvssrani_h_w(reg1, reg0, 24);
- reg2 = __lasx_xvssrani_h_w(reg3, reg2, 24);
- reg4 = __lasx_xvssrani_h_w(reg5, reg4, 24);
+ b = __lasx_xvpackev_b(zero, tmp0);
+ r = __lasx_xvpackod_b(zero, tmp0);
+ g = __lasx_xvpackev_b(zero, tmp1);
+ a = __lasx_xvpackod_b(zero, tmp1);
+ reg0 = __lasx_xvmaddwev_w_hu(const_add, b, a);
+ reg1 = __lasx_xvmaddwod_w_hu(const_add, b, a);
+ reg2 = __lasx_xvmaddwev_w_hu(const_add, r, a);
+ reg3 = __lasx_xvmaddwod_w_hu(const_add, r, a);
+ reg4 = __lasx_xvmaddwev_w_hu(const_add, g, a);
+ reg5 = __lasx_xvmaddwod_w_hu(const_add, g, a);
+ reg0 = __lasx_xvssrani_h_w(reg1, reg0, 8);
+ reg2 = __lasx_xvssrani_h_w(reg3, reg2, 8);
+ reg4 = __lasx_xvssrani_h_w(reg5, reg4, 8);
reg0 = __lasx_xvshuf_h(control, reg0, reg0);
reg2 = __lasx_xvshuf_h(control, reg2, reg2);
reg4 = __lasx_xvshuf_h(control, reg4, reg4);
diff --git a/source/row_lsx.cc b/source/row_lsx.cc
index fa088c9..10546a9 100644
--- a/source/row_lsx.cc
+++ b/source/row_lsx.cc
@@ -1102,24 +1102,26 @@
__m128i reg0, reg1, reg2, reg3, reg4, reg5;
__m128i b, g, r, a, dst0, dst1;
__m128i control = {0x0005000100040000, 0x0007000300060002};
+ __m128i zero = __lsx_vldi(0);
+ __m128i const_add = __lsx_vldi(0x8ff);
for (x = 0; x < len; x++) {
DUP2_ARG2(__lsx_vld, src_argb, 0, src_argb, 16, src0, src1);
tmp0 = __lsx_vpickev_b(src1, src0);
tmp1 = __lsx_vpickod_b(src1, src0);
- b = __lsx_vpackev_b(tmp0, tmp0);
- r = __lsx_vpackod_b(tmp0, tmp0);
- g = __lsx_vpackev_b(tmp1, tmp1);
- a = __lsx_vpackod_b(tmp1, tmp1);
- reg0 = __lsx_vmulwev_w_hu(b, a);
- reg1 = __lsx_vmulwod_w_hu(b, a);
- reg2 = __lsx_vmulwev_w_hu(r, a);
- reg3 = __lsx_vmulwod_w_hu(r, a);
- reg4 = __lsx_vmulwev_w_hu(g, a);
- reg5 = __lsx_vmulwod_w_hu(g, a);
- reg0 = __lsx_vssrani_h_w(reg1, reg0, 24);
- reg2 = __lsx_vssrani_h_w(reg3, reg2, 24);
- reg4 = __lsx_vssrani_h_w(reg5, reg4, 24);
+ b = __lsx_vpackev_b(zero, tmp0);
+ r = __lsx_vpackod_b(zero, tmp0);
+ g = __lsx_vpackev_b(zero, tmp1);
+ a = __lsx_vpackod_b(zero, tmp1);
+ reg0 = __lsx_vmaddwev_w_hu(const_add, b, a);
+ reg1 = __lsx_vmaddwod_w_hu(const_add, b, a);
+ reg2 = __lsx_vmaddwev_w_hu(const_add, r, a);
+ reg3 = __lsx_vmaddwod_w_hu(const_add, r, a);
+ reg4 = __lsx_vmaddwev_w_hu(const_add, g, a);
+ reg5 = __lsx_vmaddwod_w_hu(const_add, g, a);
+ reg0 = __lsx_vssrani_h_w(reg1, reg0, 8);
+ reg2 = __lsx_vssrani_h_w(reg3, reg2, 8);
+ reg4 = __lsx_vssrani_h_w(reg5, reg4, 8);
reg0 = __lsx_vshuf_h(control, reg0, reg0);
reg2 = __lsx_vshuf_h(control, reg2, reg2);
reg4 = __lsx_vshuf_h(control, reg4, reg4);
diff --git a/source/row_neon.cc b/source/row_neon.cc
index 31142a9..9dff94e 100644
--- a/source/row_neon.cc
+++ b/source/row_neon.cc
@@ -143,9 +143,9 @@
asm volatile(
YUVTORGB_SETUP
"vmov.u8 d6, #255 \n"
- "1: \n" READYUV444 YUVTORGB
- RGBTORGB8
- "subs %[width], %[width], #8 \n"
+ "1: \n" //
+ READYUV444
+ "subs %[width], %[width], #8 \n" YUVTORGB RGBTORGB8
"vst4.8 {d0, d2, d4, d6}, [%[dst_argb]]! \n"
"bgt 1b \n"
: [src_y] "+r"(src_y), // %[src_y]
@@ -166,9 +166,9 @@
int width) {
asm volatile(
YUVTORGB_SETUP
- "1: \n" READYUV444 YUVTORGB
- RGBTORGB8
- "subs %[width], %[width], #8 \n"
+ "1: \n" //
+ READYUV444
+ "subs %[width], %[width], #8 \n" YUVTORGB RGBTORGB8
"vst3.8 {d0, d2, d4}, [%[dst_rgb24]]! \n"
"bgt 1b \n"
: [src_y] "+r"(src_y), // %[src_y]
@@ -190,9 +190,9 @@
asm volatile(
YUVTORGB_SETUP
"vmov.u8 d6, #255 \n"
- "1: \n" READYUV422 YUVTORGB
- RGBTORGB8
- "subs %[width], %[width], #8 \n"
+ "1: \n" //
+ READYUV422
+ "subs %[width], %[width], #8 \n" YUVTORGB RGBTORGB8
"vst4.8 {d0, d2, d4, d6}, [%[dst_argb]]! \n"
"bgt 1b \n"
: [src_y] "+r"(src_y), // %[src_y]
@@ -214,10 +214,10 @@
int width) {
asm volatile(
YUVTORGB_SETUP
- "1: \n" READYUV444 YUVTORGB
- RGBTORGB8
+ "1: \n" //
+ READYUV444
+ "subs %[width], %[width], #8 \n" YUVTORGB RGBTORGB8
"vld1.8 {d6}, [%[src_a]]! \n"
- "subs %[width], %[width], #8 \n"
"vst4.8 {d0, d2, d4, d6}, [%[dst_argb]]! \n"
"bgt 1b \n"
: [src_y] "+r"(src_y), // %[src_y]
@@ -240,10 +240,10 @@
int width) {
asm volatile(
YUVTORGB_SETUP
- "1: \n" READYUV422 YUVTORGB
- RGBTORGB8
+ "1: \n" //
+ READYUV422
+ "subs %[width], %[width], #8 \n" YUVTORGB RGBTORGB8
"vld1.8 {d6}, [%[src_a]]! \n"
- "subs %[width], %[width], #8 \n"
"vst4.8 {d0, d2, d4, d6}, [%[dst_argb]]! \n"
"bgt 1b \n"
: [src_y] "+r"(src_y), // %[src_y]
@@ -266,9 +266,10 @@
asm volatile(
YUVTORGB_SETUP
"vmov.u8 d6, #255 \n"
- "1: \n" READYUV422 YUVTORGB
- RGBTORGB8 "subs %[width], %[width], #8 \n" STORERGBA
- "bgt 1b \n"
+ "1: \n" //
+ READYUV422
+ "subs %[width], %[width], #8 \n" YUVTORGB RGBTORGB8
+ STORERGBA "bgt 1b \n"
: [src_y] "+r"(src_y), // %[src_y]
[src_u] "+r"(src_u), // %[src_u]
[src_v] "+r"(src_v), // %[src_v]
@@ -288,9 +289,9 @@
asm volatile(
YUVTORGB_SETUP
"vmov.u8 d6, #255 \n"
- "1: \n" READYUV422 YUVTORGB
- RGBTORGB8
- "subs %[width], %[width], #8 \n"
+ "1: \n" //
+ READYUV422
+ "subs %[width], %[width], #8 \n" YUVTORGB RGBTORGB8
"vst3.8 {d0, d2, d4}, [%[dst_rgb24]]! \n"
"bgt 1b \n"
: [src_y] "+r"(src_y), // %[src_y]
@@ -319,8 +320,10 @@
asm volatile(
YUVTORGB_SETUP
"vmov.u8 d6, #255 \n"
- "1: \n" READYUV422 YUVTORGB
- RGBTORGB8 "subs %[width], %[width], #8 \n" ARGBTORGB565
+ "1: \n" //
+ READYUV422
+ "subs %[width], %[width], #8 \n" YUVTORGB RGBTORGB8
+ ARGBTORGB565
"vst1.8 {q2}, [%[dst_rgb565]]! \n" // store 8 pixels RGB565.
"bgt 1b \n"
: [src_y] "+r"(src_y), // %[src_y]
@@ -350,9 +353,9 @@
int width) {
asm volatile(
YUVTORGB_SETUP
- "1: \n" READYUV422 YUVTORGB
- RGBTORGB8
- "subs %[width], %[width], #8 \n"
+ "1: \n" //
+ READYUV422
+ "subs %[width], %[width], #8 \n" YUVTORGB RGBTORGB8
"vmov.u8 d6, #0xff \n" ARGBTOARGB1555
"vst1.8 {q3}, [%[dst_argb1555]]! \n" // store 8 pixels RGB1555.
"bgt 1b \n"
@@ -385,8 +388,8 @@
YUVTORGB_SETUP
"vmov.u8 d6, #255 \n"
"vmov.u8 d7, #0x0f \n" // vbic bits to clear
- "1: \n" READYUV422 YUVTORGB
- RGBTORGB8
+ "1: \n" //
+ READYUV422 YUVTORGB RGBTORGB8
"subs %[width], %[width], #8 \n" ARGBTOARGB4444
"vst1.8 {q0}, [%[dst_argb4444]]! \n" // store 8 pixels
"bgt 1b \n"
@@ -407,8 +410,8 @@
asm volatile(
YUVTORGB_SETUP
"vmov.u8 d6, #255 \n"
- "1: \n" READYUV400 YUVTORGB
- RGBTORGB8
+ "1: \n" //
+ READYUV400 YUVTORGB RGBTORGB8
"subs %[width], %[width], #8 \n"
"vst4.8 {d0, d2, d4, d6}, [%[dst_argb]]! \n"
"bgt 1b \n"
@@ -423,11 +426,11 @@
void J400ToARGBRow_NEON(const uint8_t* src_y, uint8_t* dst_argb, int width) {
asm volatile(
"vmov.u8 d23, #255 \n"
- "1: \n"
+ "1: \n"
"vld1.8 {d20}, [%0]! \n"
+ "subs %2, %2, #8 \n"
"vmov d21, d20 \n"
"vmov d22, d20 \n"
- "subs %2, %2, #8 \n"
"vst4.8 {d20, d21, d22, d23}, [%1]! \n"
"bgt 1b \n"
: "+r"(src_y), // %0
@@ -445,7 +448,8 @@
asm volatile(
YUVTORGB_SETUP
"vmov.u8 d6, #255 \n"
- "1: \n" READNV12 YUVTORGB RGBTORGB8
+ "1: \n" //
+ READNV12 YUVTORGB RGBTORGB8
"subs %[width], %[width], #8 \n"
"vst4.8 {d0, d2, d4, d6}, [%[dst_argb]]! \n"
"bgt 1b \n"
@@ -466,7 +470,8 @@
asm volatile(
YUVTORGB_SETUP
"vmov.u8 d6, #255 \n"
- "1: \n" READNV21 YUVTORGB RGBTORGB8
+ "1: \n" //
+ READNV21 YUVTORGB RGBTORGB8
"subs %[width], %[width], #8 \n"
"vst4.8 {d0, d2, d4, d6}, [%[dst_argb]]! \n"
"bgt 1b \n"
@@ -487,7 +492,8 @@
asm volatile(
YUVTORGB_SETUP
"vmov.u8 d6, #255 \n"
- "1: \n" READNV12 YUVTORGB RGBTORGB8
+ "1: \n" //
+ READNV12 YUVTORGB RGBTORGB8
"subs %[width], %[width], #8 \n"
"vst3.8 {d0, d2, d4}, [%[dst_rgb24]]! \n"
"bgt 1b \n"
@@ -508,7 +514,8 @@
asm volatile(
YUVTORGB_SETUP
"vmov.u8 d6, #255 \n"
- "1: \n" READNV21 YUVTORGB RGBTORGB8
+ "1: \n" //
+ READNV21 YUVTORGB RGBTORGB8
"subs %[width], %[width], #8 \n"
"vst3.8 {d0, d2, d4}, [%[dst_rgb24]]! \n"
"bgt 1b \n"
@@ -529,7 +536,8 @@
asm volatile(
YUVTORGB_SETUP
"vmov.u8 d6, #255 \n"
- "1: \n" READNV12 YUVTORGB RGBTORGB8
+ "1: \n" //
+ READNV12 YUVTORGB RGBTORGB8
"subs %[width], %[width], #8 \n" ARGBTORGB565
"vst1.8 {q2}, [%[dst_rgb565]]! \n" // store 8 pixels RGB565.
"bgt 1b \n"
@@ -549,7 +557,8 @@
asm volatile(
YUVTORGB_SETUP
"vmov.u8 d6, #255 \n"
- "1: \n" READYUY2 YUVTORGB RGBTORGB8
+ "1: \n" //
+ READYUY2 YUVTORGB RGBTORGB8
"subs %[width], %[width], #8 \n"
"vst4.8 {d0, d2, d4, d6}, [%[dst_argb]]! \n"
"bgt 1b \n"
@@ -568,7 +577,8 @@
asm volatile(
YUVTORGB_SETUP
"vmov.u8 d6, #255 \n"
- "1: \n" READUYVY YUVTORGB RGBTORGB8
+ "1: \n" //
+ READUYVY YUVTORGB RGBTORGB8
"subs %[width], %[width], #8 \n"
"vst4.8 {d0, d2, d4, d6}, [%[dst_argb]]! \n"
"bgt 1b \n"
@@ -586,7 +596,7 @@
uint8_t* dst_v,
int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"vld2.8 {q0, q1}, [%0]! \n" // load 16 pairs of UV
"subs %3, %3, #16 \n" // 16 processed per loop
"vst1.8 {q0}, [%1]! \n" // store U
@@ -610,7 +620,7 @@
uint8_t* dst,
int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"vld1.8 {q0}, [%0], %3 \n" // load 16 bytes
"subs %2, %2, #16 \n" // 16 processed per loop
"pld [%0, #1792] \n"
@@ -630,7 +640,7 @@
uint16_t* dst,
int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"vld1.16 {q0, q1}, [%0], %3 \n" // load 16 pixels
"subs %2, %2, #16 \n" // 16 processed per loop
"pld [%0, #3584] \n"
@@ -651,7 +661,7 @@
uint8_t* dst_v,
int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"vld2.8 {d0, d1}, [%0], %4 \n"
"subs %3, %3, #16 \n"
"pld [%0, #1792] \n"
@@ -667,7 +677,7 @@
);
}
-#if LIBYUV_USE_ST2
+#if defined(LIBYUV_USE_ST2)
// Read 16 Y, 8 UV, and write 8 YUYV.
void DetileToYUY2_NEON(const uint8_t* src_y,
ptrdiff_t src_y_tile_stride,
@@ -676,7 +686,7 @@
uint8_t* dst_yuy2,
int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"vld1.8 {q0}, [%0], %4 \n" // Load 16 Y
"pld [%0, #1792] \n"
"vld1.8 {q1}, [%1], %5 \n" // Load 8 UV
@@ -702,7 +712,7 @@
uint8_t* dst_yuy2,
int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"vld1.8 {q0}, [%0], %4 \n" // Load 16 Y
"vld1.8 {q1}, [%1], %5 \n" // Load 8 UV
"subs %3, %3, #16 \n"
@@ -724,13 +734,14 @@
void UnpackMT2T_NEON(const uint8_t* src, uint16_t* dst, size_t size) {
asm volatile(
- "1: \n"
+ "1: \n"
"vld1.8 {q14}, [%0]! \n" // Load lower bits.
"vld1.8 {q9}, [%0]! \n" // Load upper bits row
// by row.
"vld1.8 {q11}, [%0]! \n"
"vld1.8 {q13}, [%0]! \n"
"vld1.8 {q15}, [%0]! \n"
+ "subs %2, %2, #80 \n"
"vshl.u8 q8, q14, #6 \n" // Shift lower bit data
// appropriately.
"vshl.u8 q10, q14, #4 \n"
@@ -753,7 +764,6 @@
"vsri.u16 q15, q15, #10 \n"
"vstmia %1!, {q8-q15} \n" // Store pixel block (64
// pixels).
- "subs %2, %2, #80 \n"
"bgt 1b \n"
: "+r"(src), // %0
"+r"(dst), // %1
@@ -768,7 +778,7 @@
uint8_t* dst_uv,
int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"vld1.8 {q0}, [%0]! \n" // load U
"vld1.8 {q1}, [%1]! \n" // load V
"subs %3, %3, #16 \n" // 16 processed per loop
@@ -790,7 +800,7 @@
uint8_t* dst_b,
int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"vld3.8 {d0, d2, d4}, [%0]! \n" // load 8 RGB
"vld3.8 {d1, d3, d5}, [%0]! \n" // next 8 RGB
"subs %4, %4, #16 \n" // 16 processed per loop
@@ -815,7 +825,7 @@
uint8_t* dst_rgb,
int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"vld1.8 {q0}, [%0]! \n" // load R
"vld1.8 {q1}, [%1]! \n" // load G
"vld1.8 {q2}, [%2]! \n" // load B
@@ -841,7 +851,7 @@
uint8_t* dst_a,
int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"vld4.8 {d0, d2, d4, d6}, [%0]! \n" // load 8 ARGB
"vld4.8 {d1, d3, d5, d7}, [%0]! \n" // next 8 ARGB
"subs %5, %5, #16 \n" // 16 processed per loop
@@ -869,7 +879,7 @@
uint8_t* dst_argb,
int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"vld1.8 {q2}, [%0]! \n" // load R
"vld1.8 {q1}, [%1]! \n" // load G
"vld1.8 {q0}, [%2]! \n" // load B
@@ -896,7 +906,7 @@
uint8_t* dst_b,
int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"vld4.8 {d0, d2, d4, d6}, [%0]! \n" // load 8 ARGB
"vld4.8 {d1, d3, d5, d7}, [%0]! \n" // next 8 ARGB
"subs %4, %4, #16 \n" // 16 processed per loop
@@ -922,7 +932,7 @@
int width) {
asm volatile(
"vmov.u8 q3, #255 \n" // load A(255)
- "1: \n"
+ "1: \n"
"vld1.8 {q2}, [%0]! \n" // load R
"vld1.8 {q1}, [%1]! \n" // load G
"vld1.8 {q0}, [%2]! \n" // load B
@@ -950,10 +960,11 @@
asm volatile(
"vmov.u32 q14, #1023 \n"
"vdup.32 q15, %5 \n"
- "1: \n"
+ "1: \n"
"vld1.16 {d4}, [%2]! \n" // B
"vld1.16 {d2}, [%1]! \n" // G
"vld1.16 {d0}, [%0]! \n" // R
+ "subs %4, %4, #4 \n"
"vmovl.u16 q2, d4 \n" // B
"vmovl.u16 q1, d2 \n" // G
"vmovl.u16 q0, d0 \n" // R
@@ -966,7 +977,6 @@
"vsli.u32 q2, q1, #10 \n" // 00GB
"vsli.u32 q2, q0, #20 \n" // 0RGB
"vorr.u32 q2, #0xc0000000 \n" // ARGB (AR30)
- "subs %4, %4, #4 \n"
"vst1.8 {q2}, [%3]! \n"
"bgt 1b \n"
: "+r"(src_r), // %0
@@ -986,10 +996,11 @@
int width) {
asm volatile(
"vmov.u32 q14, #1023 \n"
- "1: \n"
+ "1: \n"
"vld1.16 {d4}, [%2]! \n" // B
"vld1.16 {d2}, [%1]! \n" // G
"vld1.16 {d0}, [%0]! \n" // R
+ "subs %4, %4, #4 \n"
"vmovl.u16 q2, d4 \n" // 000B
"vmovl.u16 q1, d2 \n" // G
"vmovl.u16 q0, d0 \n" // R
@@ -999,10 +1010,9 @@
"vsli.u32 q2, q1, #10 \n" // 00GB
"vsli.u32 q2, q0, #20 \n" // 0RGB
"vorr.u32 q2, #0xc0000000 \n" // ARGB (AR30)
- "subs %4, %4, #4 \n"
"vst1.8 {q2}, [%3]! \n"
"bgt 1b \n"
- "3: \n"
+ "3: \n"
: "+r"(src_r), // %0
"+r"(src_g), // %1
"+r"(src_b), // %2
@@ -1025,11 +1035,12 @@
"vdup.u16 q15, %6 \n"
"vdup.u16 q14, %7 \n"
- "1: \n"
+ "1: \n"
"vld1.16 {q2}, [%0]! \n" // R
"vld1.16 {q1}, [%1]! \n" // G
"vld1.16 {q0}, [%2]! \n" // B
"vld1.16 {q3}, [%3]! \n" // A
+ "subs %5, %5, #8 \n"
"vmin.u16 q2, q2, q14 \n"
"vmin.u16 q1, q1, q14 \n"
"vmin.u16 q0, q0, q14 \n"
@@ -1038,7 +1049,6 @@
"vshl.u16 q1, q1, q15 \n"
"vshl.u16 q0, q0, q15 \n"
"vshl.u16 q3, q3, q15 \n"
- "subs %5, %5, #8 \n"
"vst4.16 {d0, d2, d4, d6}, [%4]! \n"
"vst4.16 {d1, d3, d5, d7}, [%4]! \n"
"bgt 1b \n"
@@ -1066,17 +1076,17 @@
"vmov.u8 q3, #0xff \n" // A (0xffff)
"vdup.u16 q15, %5 \n"
"vdup.u16 q14, %6 \n"
- "1: \n"
+ "1: \n"
"vld1.16 {q2}, [%0]! \n" // R
"vld1.16 {q1}, [%1]! \n" // G
"vld1.16 {q0}, [%2]! \n" // B
+ "subs %4, %4, #8 \n"
"vmin.u16 q2, q2, q14 \n"
"vmin.u16 q1, q1, q14 \n"
"vmin.u16 q0, q0, q14 \n"
"vshl.u16 q2, q2, q15 \n"
"vshl.u16 q1, q1, q15 \n"
"vshl.u16 q0, q0, q15 \n"
- "subs %4, %4, #8 \n"
"vst4.16 {d0, d2, d4, d6}, [%3]! \n"
"vst4.16 {d1, d3, d5, d7}, [%3]! \n"
"bgt 1b \n"
@@ -1101,11 +1111,12 @@
asm volatile(
"vdup.16 q15, %6 \n"
- "1: \n"
+ "1: \n"
"vld1.16 {q2}, [%0]! \n" // R
"vld1.16 {q1}, [%1]! \n" // G
"vld1.16 {q0}, [%2]! \n" // B
"vld1.16 {q3}, [%3]! \n" // A
+ "subs %5, %5, #8 \n"
"vshl.u16 q2, q2, q15 \n"
"vshl.u16 q1, q1, q15 \n"
"vshl.u16 q0, q0, q15 \n"
@@ -1114,7 +1125,6 @@
"vqmovn.u16 d1, q1 \n"
"vqmovn.u16 d2, q2 \n"
"vqmovn.u16 d3, q3 \n"
- "subs %5, %5, #8 \n"
"vst4.8 {d0, d1, d2, d3}, [%4]! \n"
"bgt 1b \n"
: "+r"(src_r), // %0
@@ -1138,17 +1148,17 @@
"vdup.16 q15, %5 \n"
"vmov.u8 d6, #0xff \n" // A (0xff)
- "1: \n"
+ "1: \n"
"vld1.16 {q2}, [%0]! \n" // R
"vld1.16 {q1}, [%1]! \n" // G
"vld1.16 {q0}, [%2]! \n" // B
+ "subs %4, %4, #8 \n"
"vshl.u16 q2, q2, q15 \n"
"vshl.u16 q1, q1, q15 \n"
"vshl.u16 q0, q0, q15 \n"
"vqmovn.u16 d5, q2 \n"
"vqmovn.u16 d4, q1 \n"
"vqmovn.u16 d3, q0 \n"
- "subs %4, %4, #8 \n"
"vst4.u8 {d3, d4, d5, d6}, [%3]! \n"
"bgt 1b \n"
: "+r"(src_r), // %0
@@ -1163,7 +1173,7 @@
// Copy multiple of 32. vld4.8 allow unaligned and is fastest on a15.
void CopyRow_NEON(const uint8_t* src, uint8_t* dst, int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"vld1.8 {d0, d1, d2, d3}, [%0]! \n" // load 32
"subs %2, %2, #32 \n" // 32 processed per loop
"vst1.8 {d0, d1, d2, d3}, [%1]! \n" // store 32
@@ -1180,7 +1190,7 @@
void SetRow_NEON(uint8_t* dst, uint8_t v8, int width) {
asm volatile(
"vdup.8 q0, %2 \n" // duplicate 16 bytes
- "1: \n"
+ "1: \n"
"subs %1, %1, #16 \n" // 16 bytes per loop
"vst1.8 {q0}, [%0]! \n" // store
"bgt 1b \n"
@@ -1194,7 +1204,7 @@
void ARGBSetRow_NEON(uint8_t* dst, uint32_t v32, int width) {
asm volatile(
"vdup.u32 q0, %2 \n" // duplicate 4 ints
- "1: \n"
+ "1: \n"
"subs %1, %1, #4 \n" // 4 pixels per loop
"vst1.8 {q0}, [%0]! \n" // store
"bgt 1b \n"
@@ -1210,7 +1220,7 @@
"add %0, %0, %2 \n"
"sub %0, %0, #32 \n" // 32 bytes per loop
- "1: \n"
+ "1: \n"
"vld1.8 {q1, q2}, [%0], %3 \n" // src -= 32
"subs %2, #32 \n" // 32 pixels per loop.
"vrev64.8 q0, q2 \n"
@@ -1233,7 +1243,7 @@
"add %0, %0, %2, lsl #1 \n"
"sub %0, #16 \n"
- "1: \n"
+ "1: \n"
"vld2.8 {d0, d1}, [%0], r12 \n" // src -= 16
"subs %2, #8 \n" // 8 pixels per loop.
"vrev64.8 q0, q0 \n"
@@ -1256,7 +1266,7 @@
"add %0, %0, %3, lsl #1 \n"
"sub %0, #16 \n"
- "1: \n"
+ "1: \n"
"vld2.8 {d0, d1}, [%0], r12 \n" // src -= 16
"subs %3, #8 \n" // 8 pixels per loop.
"vrev64.8 q0, q0 \n"
@@ -1276,7 +1286,7 @@
"add %0, %0, %2, lsl #2 \n"
"sub %0, #32 \n"
- "1: \n"
+ "1: \n"
"vld4.8 {d0, d1, d2, d3}, [%0], %3 \n" // src -= 32
"subs %2, #8 \n" // 8 pixels per loop.
"vrev64.8 d0, d0 \n"
@@ -1297,7 +1307,7 @@
int width) {
src_rgb24 += width * 3 - 24;
asm volatile(
- "1: \n"
+ "1: \n"
"vld3.8 {d0, d1, d2}, [%0], %3 \n" // src -= 24
"subs %2, #8 \n" // 8 pixels per loop.
"vrev64.8 d0, d0 \n"
@@ -1317,7 +1327,7 @@
int width) {
asm volatile(
"vmov.u8 d4, #255 \n" // Alpha
- "1: \n"
+ "1: \n"
"vld3.8 {d1, d2, d3}, [%0]! \n" // load 8 pixels of RGB24.
"subs %2, %2, #8 \n" // 8 processed per loop.
"vst4.8 {d1, d2, d3, d4}, [%1]! \n" // store 8 pixels of ARGB.
@@ -1333,7 +1343,7 @@
void RAWToARGBRow_NEON(const uint8_t* src_raw, uint8_t* dst_argb, int width) {
asm volatile(
"vmov.u8 d4, #255 \n" // Alpha
- "1: \n"
+ "1: \n"
"vld3.8 {d1, d2, d3}, [%0]! \n" // load 8 pixels of RAW.
"subs %2, %2, #8 \n" // 8 processed per loop.
"vswp.u8 d1, d3 \n" // swap R, B
@@ -1350,7 +1360,7 @@
void RAWToRGBARow_NEON(const uint8_t* src_raw, uint8_t* dst_rgba, int width) {
asm volatile(
"vmov.u8 d0, #255 \n" // Alpha
- "1: \n"
+ "1: \n"
"vld3.8 {d1, d2, d3}, [%0]! \n" // load 8 pixels of RAW.
"subs %2, %2, #8 \n" // 8 processed per loop.
"vswp.u8 d1, d3 \n" // swap R, B
@@ -1365,7 +1375,7 @@
}
void RAWToRGB24Row_NEON(const uint8_t* src_raw, uint8_t* dst_rgb24, int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"vld3.8 {d1, d2, d3}, [%0]! \n" // load 8 pixels of RAW.
"subs %2, %2, #8 \n" // 8 processed per loop.
"vswp.u8 d1, d3 \n" // swap R, B
@@ -1397,7 +1407,7 @@
int width) {
asm volatile(
"vmov.u8 d3, #255 \n" // Alpha
- "1: \n"
+ "1: \n"
"vld1.8 {q0}, [%0]! \n" // load 8 RGB565 pixels.
"subs %2, %2, #8 \n" // 8 processed per loop.
RGB565TOARGB
@@ -1443,7 +1453,7 @@
int width) {
asm volatile(
"vmov.u8 d3, #255 \n" // Alpha
- "1: \n"
+ "1: \n"
"vld1.8 {q0}, [%0]! \n" // load 8 ARGB1555 pixels.
"subs %2, %2, #8 \n" // 8 processed per loop.
ARGB1555TOARGB
@@ -1472,7 +1482,7 @@
int width) {
asm volatile(
"vmov.u8 d3, #255 \n" // Alpha
- "1: \n"
+ "1: \n"
"vld1.8 {q0}, [%0]! \n" // load 8 ARGB4444 pixels.
"subs %2, %2, #8 \n" // 8 processed per loop.
ARGB4444TOARGB
@@ -1490,7 +1500,7 @@
uint8_t* dst_rgb24,
int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"vld4.8 {d0, d2, d4, d6}, [%0]! \n" // load 16 pixels of ARGB.
"vld4.8 {d1, d3, d5, d7}, [%0]! \n"
"subs %2, %2, #16 \n" // 16 processed per loop.
@@ -1507,7 +1517,7 @@
void ARGBToRAWRow_NEON(const uint8_t* src_argb, uint8_t* dst_raw, int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"vld4.8 {d1, d2, d3, d4}, [%0]! \n" // load 8 pixels of ARGB.
"subs %2, %2, #8 \n" // 8 processed per loop.
"vswp.u8 d1, d3 \n" // swap R, B
@@ -1523,7 +1533,7 @@
void YUY2ToYRow_NEON(const uint8_t* src_yuy2, uint8_t* dst_y, int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"vld2.8 {q0, q1}, [%0]! \n" // load 16 pixels of YUY2.
"subs %2, %2, #16 \n" // 16 processed per loop.
"vst1.8 {q0}, [%1]! \n" // store 16 pixels of Y.
@@ -1538,7 +1548,7 @@
void UYVYToYRow_NEON(const uint8_t* src_uyvy, uint8_t* dst_y, int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"vld2.8 {q0, q1}, [%0]! \n" // load 16 pixels of UYVY.
"subs %2, %2, #16 \n" // 16 processed per loop.
"vst1.8 {q1}, [%1]! \n" // store 16 pixels of Y.
@@ -1556,7 +1566,7 @@
uint8_t* dst_v,
int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"vld4.8 {d0, d1, d2, d3}, [%0]! \n" // load 16 pixels of YUY2.
"subs %3, %3, #16 \n" // 16 pixels = 8 UVs.
"vst1.8 {d1}, [%1]! \n" // store 8 U.
@@ -1576,7 +1586,7 @@
uint8_t* dst_v,
int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"vld4.8 {d0, d1, d2, d3}, [%0]! \n" // load 16 pixels of UYVY.
"subs %3, %3, #16 \n" // 16 pixels = 8 UVs.
"vst1.8 {d0}, [%1]! \n" // store 8 U.
@@ -1598,10 +1608,10 @@
int width) {
asm volatile(
"add %1, %0, %1 \n" // stride + src_yuy2
- "1: \n"
+ "1: \n"
"vld4.8 {d0, d1, d2, d3}, [%0]! \n" // load 16 pixels of YUY2.
- "subs %4, %4, #16 \n" // 16 pixels = 8 UVs.
"vld4.8 {d4, d5, d6, d7}, [%1]! \n" // load next row YUY2.
+ "subs %4, %4, #16 \n" // 16 pixels = 8 UVs.
"vrhadd.u8 d1, d1, d5 \n" // average rows of U
"vrhadd.u8 d3, d3, d7 \n" // average rows of V
"vst1.8 {d1}, [%2]! \n" // store 8 U.
@@ -1625,10 +1635,10 @@
int width) {
asm volatile(
"add %1, %0, %1 \n" // stride + src_uyvy
- "1: \n"
+ "1: \n"
"vld4.8 {d0, d1, d2, d3}, [%0]! \n" // load 16 pixels of UYVY.
- "subs %4, %4, #16 \n" // 16 pixels = 8 UVs.
"vld4.8 {d4, d5, d6, d7}, [%1]! \n" // load next row UYVY.
+ "subs %4, %4, #16 \n" // 16 pixels = 8 UVs.
"vrhadd.u8 d0, d0, d4 \n" // average rows of U
"vrhadd.u8 d2, d2, d6 \n" // average rows of V
"vst1.8 {d0}, [%2]! \n" // store 8 U.
@@ -1651,7 +1661,7 @@
int width) {
asm volatile(
"add %1, %0, %1 \n" // stride + src_yuy2
- "1: \n"
+ "1: \n"
"vld2.8 {q0, q1}, [%0]! \n" // load 16 pixels of YUY2.
"subs %3, %3, #16 \n" // 16 pixels = 8 UVs.
"vld2.8 {q2, q3}, [%1]! \n" // load next row YUY2.
@@ -1675,7 +1685,7 @@
int width) {
asm volatile(
"vld1.8 {q2}, [%3] \n" // shuffler
- "1: \n"
+ "1: \n"
"vld1.8 {q0}, [%0]! \n" // load 4 pixels.
"subs %2, %2, #4 \n" // 4 processed per loop
"vtbl.8 d2, {d0, d1}, d4 \n" // look up 2 first pixels
@@ -1696,7 +1706,7 @@
uint8_t* dst_yuy2,
int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"vld2.8 {d0, d2}, [%0]! \n" // load 16 Ys
"vld1.8 {d1}, [%1]! \n" // load 8 Us
"vld1.8 {d3}, [%2]! \n" // load 8 Vs
@@ -1718,7 +1728,7 @@
uint8_t* dst_uyvy,
int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"vld2.8 {d1, d3}, [%0]! \n" // load 16 Ys
"vld1.8 {d0}, [%1]! \n" // load 8 Us
"vld1.8 {d2}, [%2]! \n" // load 8 Vs
@@ -1738,7 +1748,7 @@
uint8_t* dst_rgb565,
int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"vld4.8 {d0, d2, d4, d6}, [%0]! \n" // load 8 pixels of ARGB.
"subs %2, %2, #8 \n" // 8 processed per loop.
ARGBTORGB565
@@ -1757,7 +1767,7 @@
int width) {
asm volatile(
"vdup.32 d7, %2 \n" // dither4
- "1: \n"
+ "1: \n"
"vld4.8 {d0, d2, d4, d6}, [%1]! \n" // load 8 pixels of ARGB.
"subs %3, %3, #8 \n" // 8 processed per loop.
"vqadd.u8 d0, d0, d7 \n"
@@ -1777,7 +1787,7 @@
uint8_t* dst_argb1555,
int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"vld4.8 {d0, d2, d4, d6}, [%0]! \n" // load 8 pixels of ARGB.
"subs %2, %2, #8 \n" // 8 processed per loop.
ARGBTOARGB1555
@@ -1796,7 +1806,7 @@
asm volatile(
"vmov.u8 d7, #0x0f \n" // bits to clear with
// vbic.
- "1: \n"
+ "1: \n"
"vld4.8 {d0, d2, d4, d6}, [%0]! \n" // load 8 pixels of ARGB.
"subs %2, %2, #8 \n" // 8 processed per loop.
ARGBTOARGB4444
@@ -1813,7 +1823,7 @@
uint8_t* dst_a,
int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"vld4.8 {d0, d2, d4, d6}, [%0]! \n" // load 8 ARGB pixels
"vld4.8 {d1, d3, d5, d7}, [%0]! \n" // load next 8 ARGB pixels
"subs %2, %2, #16 \n" // 16 processed per loop
@@ -1827,28 +1837,30 @@
);
}
+// Coefficients expressed as negatives to allow 128
struct RgbUVConstants {
- uint8_t kRGBToU[4];
- uint8_t kRGBToV[4];
+ int8_t kRGBToU[4];
+ int8_t kRGBToV[4];
};
// 8x1 pixels.
-void ARGBToUV444MatrixRow_NEON(const uint8_t* src_argb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width,
- const struct RgbUVConstants* rgbuvconstants) {
+static void ARGBToUV444MatrixRow_NEON(
+ const uint8_t* src_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct RgbUVConstants* rgbuvconstants) {
asm volatile(
-
"vld1.8 {d0}, [%4] \n" // load rgbuvconstants
"vdup.u8 d24, d0[0] \n" // UB 0.875 coefficient
"vdup.u8 d25, d0[1] \n" // UG -0.5781 coefficient
"vdup.u8 d26, d0[2] \n" // UR -0.2969 coefficient
"vdup.u8 d27, d0[4] \n" // VB -0.1406 coefficient
"vdup.u8 d28, d0[5] \n" // VG -0.7344 coefficient
- "vmov.u16 q15, #0x8080 \n" // 128.5
+ "vneg.s8 d24, d24 \n"
+ "vmov.u16 q15, #0x8000 \n" // 128.0
- "1: \n"
+ "1: \n"
"vld4.8 {d0, d1, d2, d3}, [%0]! \n" // load 8 ARGB pixels.
"subs %3, %3, #8 \n" // 8 processed per loop.
"vmull.u8 q2, d0, d24 \n" // B
@@ -1859,8 +1871,8 @@
"vmlsl.u8 q3, d1, d28 \n" // G
"vmlsl.u8 q3, d0, d27 \n" // B
- "vaddhn.u16 d0, q2, q15 \n" // +128 -> unsigned
- "vaddhn.u16 d1, q3, q15 \n" // +128 -> unsigned
+ "vaddhn.u16 d0, q2, q15 \n" // signed -> unsigned
+ "vaddhn.u16 d1, q3, q15 \n"
"vst1.8 {d0}, [%1]! \n" // store 8 pixels U.
"vst1.8 {d1}, [%2]! \n" // store 8 pixels V.
@@ -1874,42 +1886,42 @@
"q15");
}
-// RGB to bt601 coefficients
+// RGB to BT601 coefficients
// UB 0.875 coefficient = 112
-// UG -0.5781 coefficient = 74
-// UR -0.2969 coefficient = 38
-// VB -0.1406 coefficient = 18
-// VG -0.7344 coefficient = 94
-// VR 0.875 coefficient = 112 (ignored)
+// UG -0.5781 coefficient = -74
+// UR -0.2969 coefficient = -38
+// VB -0.1406 coefficient = -18
+// VG -0.7344 coefficient = -94
+// VR 0.875 coefficient = 112
-static const struct RgbUVConstants kRgb24I601UVConstants = {{112, 74, 38, 0},
- {18, 94, 112, 0}};
-
-// RGB to JPeg coefficients
-// UB coeff 0.500 = 127
-// UG coeff -0.33126 = 84
-// UR coeff -0.16874 = 43
-// VB coeff -0.08131 = 20
-// VG coeff -0.41869 = 107
-// VR coeff 0.500 = 127 (ignored)
-
-static const struct RgbUVConstants kRgb24JPegUVConstants = {{127, 84, 43, 0},
- {20, 107, 127, 0}};
+static const struct RgbUVConstants kARGBI601UVConstants = {{-112, 74, 38, 0},
+ {18, 94, -112, 0}};
void ARGBToUV444Row_NEON(const uint8_t* src_argb,
uint8_t* dst_u,
uint8_t* dst_v,
int width) {
ARGBToUV444MatrixRow_NEON(src_argb, dst_u, dst_v, width,
- &kRgb24I601UVConstants);
+ &kARGBI601UVConstants);
}
+// RGB to JPEG coefficients
+// UB 0.500 coefficient = 128
+// UG -0.33126 coefficient = -85
+// UR -0.16874 coefficient = -43
+// VB -0.08131 coefficient = -21
+// VG -0.41869 coefficient = -107
+// VR 0.500 coefficient = 128
+
+static const struct RgbUVConstants kARGBJPEGUVConstants = {{-128, 85, 43, 0},
+ {21, 107, -128, 0}};
+
void ARGBToUVJ444Row_NEON(const uint8_t* src_argb,
uint8_t* dst_u,
uint8_t* dst_v,
int width) {
ARGBToUV444MatrixRow_NEON(src_argb, dst_u, dst_v, width,
- &kRgb24JPegUVConstants);
+ &kARGBJPEGUVConstants);
}
// clang-format off
@@ -1933,15 +1945,16 @@
int width) {
asm volatile (
"add %1, %0, %1 \n" // src_stride + src_argb
- "vmov.s16 q10, #112 / 2 \n" // UB / VR 0.875 coefficient
- "vmov.s16 q11, #74 / 2 \n" // UG -0.5781 coefficient
- "vmov.s16 q12, #38 / 2 \n" // UR -0.2969 coefficient
- "vmov.s16 q13, #18 / 2 \n" // VB -0.1406 coefficient
- "vmov.s16 q14, #94 / 2 \n" // VG -0.7344 coefficient
- "vmov.u16 q15, #0x8080 \n" // 128.5
- "1: \n"
+ "vmov.s16 q10, #112 \n" // UB/VR 0.875 coefficient
+ "vmov.s16 q11, #74 \n" // UG -0.5781 coefficient
+ "vmov.s16 q12, #38 \n" // UR -0.2969 coefficient
+ "vmov.s16 q13, #18 \n" // VB -0.1406 coefficient
+ "vmov.s16 q14, #94 \n" // VG -0.7344 coefficient
+ "vmov.u16 q15, #0x8000 \n" // 128.0
+ "1: \n"
"vld4.8 {d0, d2, d4, d6}, [%0]! \n" // load 8 ARGB pixels.
"vld4.8 {d1, d3, d5, d7}, [%0]! \n" // load next 8 ARGB pixels.
+ "subs %4, %4, #16 \n" // 16 processed per loop.
"vpaddl.u8 q0, q0 \n" // B 16 bytes -> 8 shorts.
"vpaddl.u8 q1, q1 \n" // G 16 bytes -> 8 shorts.
"vpaddl.u8 q2, q2 \n" // R 16 bytes -> 8 shorts.
@@ -1951,11 +1964,10 @@
"vpadal.u8 q1, q5 \n" // G 16 bytes -> 8 shorts.
"vpadal.u8 q2, q6 \n" // R 16 bytes -> 8 shorts.
- "vrshr.u16 q0, q0, #1 \n" // 2x average
- "vrshr.u16 q1, q1, #1 \n"
- "vrshr.u16 q2, q2, #1 \n"
+ "vrshr.u16 q0, q0, #2 \n" // average of 4
+ "vrshr.u16 q1, q1, #2 \n"
+ "vrshr.u16 q2, q2, #2 \n"
- "subs %4, %4, #16 \n" // 16 processed per loop.
RGBTOUV(q0, q1, q2)
"vst1.8 {d0}, [%2]! \n" // store 8 pixels U.
"vst1.8 {d1}, [%3]! \n" // store 8 pixels V.
@@ -1971,7 +1983,6 @@
);
}
-// TODO(fbarchard): Subsample match Intel code.
void ARGBToUVJRow_NEON(const uint8_t* src_argb,
int src_stride_argb,
uint8_t* dst_u,
@@ -1979,15 +1990,16 @@
int width) {
asm volatile (
"add %1, %0, %1 \n" // src_stride + src_argb
- "vmov.s16 q10, #127 / 2 \n" // UB / VR 0.500 coefficient
- "vmov.s16 q11, #84 / 2 \n" // UG -0.33126 coefficient
- "vmov.s16 q12, #43 / 2 \n" // UR -0.16874 coefficient
- "vmov.s16 q13, #20 / 2 \n" // VB -0.08131 coefficient
- "vmov.s16 q14, #107 / 2 \n" // VG -0.41869 coefficient
- "vmov.u16 q15, #0x8080 \n" // 128.5
- "1: \n"
+ "vmov.s16 q10, #128 \n" // UB/VR 0.500 coefficient
+ "vmov.s16 q11, #85 \n" // UG -0.33126 coefficient
+ "vmov.s16 q12, #43 \n" // UR -0.16874 coefficient
+ "vmov.s16 q13, #21 \n" // VB -0.08131 coefficient
+ "vmov.s16 q14, #107 \n" // VG -0.41869 coefficient
+ "vmov.u16 q15, #0x8000 \n" // 128.0
+ "1: \n"
"vld4.8 {d0, d2, d4, d6}, [%0]! \n" // load 8 ARGB pixels.
"vld4.8 {d1, d3, d5, d7}, [%0]! \n" // load next 8 ARGB pixels.
+ "subs %4, %4, #16 \n" // 16 processed per loop.
"vpaddl.u8 q0, q0 \n" // B 16 bytes -> 8 shorts.
"vpaddl.u8 q1, q1 \n" // G 16 bytes -> 8 shorts.
"vpaddl.u8 q2, q2 \n" // R 16 bytes -> 8 shorts.
@@ -1997,11 +2009,10 @@
"vpadal.u8 q1, q5 \n" // G 16 bytes -> 8 shorts.
"vpadal.u8 q2, q6 \n" // R 16 bytes -> 8 shorts.
- "vrshr.u16 q0, q0, #1 \n" // 2x average
- "vrshr.u16 q1, q1, #1 \n"
- "vrshr.u16 q2, q2, #1 \n"
+ "vrshr.u16 q0, q0, #2 \n" // average of 4
+ "vrshr.u16 q1, q1, #2 \n"
+ "vrshr.u16 q2, q2, #2 \n"
- "subs %4, %4, #16 \n" // 16 processed per loop.
RGBTOUV(q0, q1, q2)
"vst1.8 {d0}, [%2]! \n" // store 8 pixels U.
"vst1.8 {d1}, [%3]! \n" // store 8 pixels V.
@@ -2024,15 +2035,16 @@
int width) {
asm volatile (
"add %1, %0, %1 \n" // src_stride + src_argb
- "vmov.s16 q10, #127 / 2 \n" // UB / VR 0.500 coefficient
- "vmov.s16 q11, #84 / 2 \n" // UG -0.33126 coefficient
- "vmov.s16 q12, #43 / 2 \n" // UR -0.16874 coefficient
- "vmov.s16 q13, #20 / 2 \n" // VB -0.08131 coefficient
- "vmov.s16 q14, #107 / 2 \n" // VG -0.41869 coefficient
- "vmov.u16 q15, #0x8080 \n" // 128.5
- "1: \n"
+ "vmov.s16 q10, #128 \n" // UB/VR 0.500 coefficient
+ "vmov.s16 q11, #85 \n" // UG -0.33126 coefficient
+ "vmov.s16 q12, #43 \n" // UR -0.16874 coefficient
+ "vmov.s16 q13, #21 \n" // VB -0.08131 coefficient
+ "vmov.s16 q14, #107 \n" // VG -0.41869 coefficient
+ "vmov.u16 q15, #0x8000 \n" // 128.0
+ "1: \n"
"vld4.8 {d0, d2, d4, d6}, [%0]! \n" // load 8 ABGR pixels.
"vld4.8 {d1, d3, d5, d7}, [%0]! \n" // load next 8 ABGR pixels.
+ "subs %4, %4, #16 \n" // 16 processed per loop.
"vpaddl.u8 q0, q0 \n" // R 16 bytes -> 8 shorts.
"vpaddl.u8 q1, q1 \n" // G 16 bytes -> 8 shorts.
"vpaddl.u8 q2, q2 \n" // B 16 bytes -> 8 shorts.
@@ -2042,11 +2054,10 @@
"vpadal.u8 q1, q5 \n" // G 16 bytes -> 8 shorts.
"vpadal.u8 q2, q6 \n" // B 16 bytes -> 8 shorts.
- "vrshr.u16 q0, q0, #1 \n" // 2x average
- "vrshr.u16 q1, q1, #1 \n"
- "vrshr.u16 q2, q2, #1 \n"
+ "vrshr.u16 q0, q0, #2 \n" // average of 4
+ "vrshr.u16 q1, q1, #2 \n"
+ "vrshr.u16 q2, q2, #2 \n"
- "subs %4, %4, #16 \n" // 16 processed per loop.
RGBTOUV(q2, q1, q0)
"vst1.8 {d0}, [%2]! \n" // store 8 pixels U.
"vst1.8 {d1}, [%3]! \n" // store 8 pixels V.
@@ -2062,7 +2073,6 @@
);
}
-// TODO(fbarchard): Subsample match C code.
void RGB24ToUVJRow_NEON(const uint8_t* src_rgb24,
int src_stride_rgb24,
uint8_t* dst_u,
@@ -2070,15 +2080,16 @@
int width) {
asm volatile (
"add %1, %0, %1 \n" // src_stride + src_rgb24
- "vmov.s16 q10, #127 / 2 \n" // UB / VR 0.500 coefficient
- "vmov.s16 q11, #84 / 2 \n" // UG -0.33126 coefficient
- "vmov.s16 q12, #43 / 2 \n" // UR -0.16874 coefficient
- "vmov.s16 q13, #20 / 2 \n" // VB -0.08131 coefficient
- "vmov.s16 q14, #107 / 2 \n" // VG -0.41869 coefficient
- "vmov.u16 q15, #0x8080 \n" // 128.5
- "1: \n"
+ "vmov.s16 q10, #128 \n" // UB/VR 0.500 coefficient
+ "vmov.s16 q11, #85 \n" // UG -0.33126 coefficient
+ "vmov.s16 q12, #43 \n" // UR -0.16874 coefficient
+ "vmov.s16 q13, #21 \n" // VB -0.08131 coefficient
+ "vmov.s16 q14, #107 \n" // VG -0.41869 coefficient
+ "vmov.u16 q15, #0x8000 \n" // 128.0
+ "1: \n"
"vld3.8 {d0, d2, d4}, [%0]! \n" // load 8 RGB24 pixels.
"vld3.8 {d1, d3, d5}, [%0]! \n" // load next 8 RGB24 pixels.
+ "subs %4, %4, #16 \n" // 16 processed per loop.
"vpaddl.u8 q0, q0 \n" // B 16 bytes -> 8 shorts.
"vpaddl.u8 q1, q1 \n" // G 16 bytes -> 8 shorts.
"vpaddl.u8 q2, q2 \n" // R 16 bytes -> 8 shorts.
@@ -2088,11 +2099,10 @@
"vpadal.u8 q1, q5 \n" // G 16 bytes -> 8 shorts.
"vpadal.u8 q2, q6 \n" // R 16 bytes -> 8 shorts.
- "vrshr.u16 q0, q0, #1 \n" // 2x average
- "vrshr.u16 q1, q1, #1 \n"
- "vrshr.u16 q2, q2, #1 \n"
+ "vrshr.u16 q0, q0, #2 \n" // average of 4
+ "vrshr.u16 q1, q1, #2 \n"
+ "vrshr.u16 q2, q2, #2 \n"
- "subs %4, %4, #16 \n" // 16 processed per loop.
RGBTOUV(q0, q1, q2)
"vst1.8 {d0}, [%2]! \n" // store 8 pixels U.
"vst1.8 {d1}, [%3]! \n" // store 8 pixels V.
@@ -2108,7 +2118,6 @@
);
}
-// TODO(fbarchard): Subsample match C code.
void RAWToUVJRow_NEON(const uint8_t* src_raw,
int src_stride_raw,
uint8_t* dst_u,
@@ -2116,15 +2125,16 @@
int width) {
asm volatile (
"add %1, %0, %1 \n" // src_stride + src_raw
- "vmov.s16 q10, #127 / 2 \n" // UB / VR 0.500 coefficient
- "vmov.s16 q11, #84 / 2 \n" // UG -0.33126 coefficient
- "vmov.s16 q12, #43 / 2 \n" // UR -0.16874 coefficient
- "vmov.s16 q13, #20 / 2 \n" // VB -0.08131 coefficient
- "vmov.s16 q14, #107 / 2 \n" // VG -0.41869 coefficient
- "vmov.u16 q15, #0x8080 \n" // 128.5
- "1: \n"
+ "vmov.s16 q10, #128 \n" // UB/VR 0.500 coefficient
+ "vmov.s16 q11, #85 \n" // UG -0.33126 coefficient
+ "vmov.s16 q12, #43 \n" // UR -0.16874 coefficient
+ "vmov.s16 q13, #21 \n" // VB -0.08131 coefficient
+ "vmov.s16 q14, #107 \n" // VG -0.41869 coefficient
+ "vmov.u16 q15, #0x8000 \n" // 128.0
+ "1: \n"
"vld3.8 {d0, d2, d4}, [%0]! \n" // load 8 RAW pixels.
"vld3.8 {d1, d3, d5}, [%0]! \n" // load next 8 RAW pixels.
+ "subs %4, %4, #16 \n" // 16 processed per loop.
"vpaddl.u8 q0, q0 \n" // B 16 bytes -> 8 shorts.
"vpaddl.u8 q1, q1 \n" // G 16 bytes -> 8 shorts.
"vpaddl.u8 q2, q2 \n" // R 16 bytes -> 8 shorts.
@@ -2134,11 +2144,10 @@
"vpadal.u8 q1, q5 \n" // G 16 bytes -> 8 shorts.
"vpadal.u8 q2, q6 \n" // R 16 bytes -> 8 shorts.
- "vrshr.u16 q0, q0, #1 \n" // 2x average
- "vrshr.u16 q1, q1, #1 \n"
- "vrshr.u16 q2, q2, #1 \n"
+ "vrshr.u16 q0, q0, #2 \n" // average of 4
+ "vrshr.u16 q1, q1, #2 \n"
+ "vrshr.u16 q2, q2, #2 \n"
- "subs %4, %4, #16 \n" // 16 processed per loop.
RGBTOUV(q2, q1, q0)
"vst1.8 {d0}, [%2]! \n" // store 8 pixels U.
"vst1.8 {d1}, [%3]! \n" // store 8 pixels V.
@@ -2161,15 +2170,16 @@
int width) {
asm volatile (
"add %1, %0, %1 \n" // src_stride + src_bgra
- "vmov.s16 q10, #112 / 2 \n" // UB / VR 0.875 coefficient
- "vmov.s16 q11, #74 / 2 \n" // UG -0.5781 coefficient
- "vmov.s16 q12, #38 / 2 \n" // UR -0.2969 coefficient
- "vmov.s16 q13, #18 / 2 \n" // VB -0.1406 coefficient
- "vmov.s16 q14, #94 / 2 \n" // VG -0.7344 coefficient
- "vmov.u16 q15, #0x8080 \n" // 128.5
- "1: \n"
+ "vmov.s16 q10, #112 \n" // UB/VR 0.875 coefficient
+ "vmov.s16 q11, #74 \n" // UG -0.5781 coefficient
+ "vmov.s16 q12, #38 \n" // UR -0.2969 coefficient
+ "vmov.s16 q13, #18 \n" // VB -0.1406 coefficient
+ "vmov.s16 q14, #94 \n" // VG -0.7344 coefficient
+ "vmov.u16 q15, #0x8000 \n" // 128.0
+ "1: \n"
"vld4.8 {d0, d2, d4, d6}, [%0]! \n" // load 8 BGRA pixels.
"vld4.8 {d1, d3, d5, d7}, [%0]! \n" // load next 8 BGRA pixels.
+ "subs %4, %4, #16 \n" // 16 processed per loop.
"vpaddl.u8 q3, q3 \n" // B 16 bytes -> 8 shorts.
"vpaddl.u8 q2, q2 \n" // G 16 bytes -> 8 shorts.
"vpaddl.u8 q1, q1 \n" // R 16 bytes -> 8 shorts.
@@ -2179,18 +2189,17 @@
"vpadal.u8 q2, q6 \n" // G 16 bytes -> 8 shorts.
"vpadal.u8 q1, q5 \n" // R 16 bytes -> 8 shorts.
- "vrshr.u16 q1, q1, #1 \n" // 2x average
- "vrshr.u16 q2, q2, #1 \n"
- "vrshr.u16 q3, q3, #1 \n"
+ "vrshr.u16 q1, q1, #2 \n" // average of 4
+ "vrshr.u16 q2, q2, #2 \n"
+ "vrshr.u16 q3, q3, #2 \n"
- "subs %4, %4, #16 \n" // 16 processed per loop.
RGBTOUV(q3, q2, q1)
"vst1.8 {d0}, [%2]! \n" // store 8 pixels U.
"vst1.8 {d1}, [%3]! \n" // store 8 pixels V.
"bgt 1b \n"
: "+r"(src_bgra), // %0
"+r"(src_stride_bgra), // %1
- "+r"(dst_u), // %2
+ "+r"(dst_u), // %2-
"+r"(dst_v), // %3
"+r"(width) // %4
:
@@ -2206,15 +2215,16 @@
int width) {
asm volatile (
"add %1, %0, %1 \n" // src_stride + src_abgr
- "vmov.s16 q10, #112 / 2 \n" // UB / VR 0.875 coefficient
- "vmov.s16 q11, #74 / 2 \n" // UG -0.5781 coefficient
- "vmov.s16 q12, #38 / 2 \n" // UR -0.2969 coefficient
- "vmov.s16 q13, #18 / 2 \n" // VB -0.1406 coefficient
- "vmov.s16 q14, #94 / 2 \n" // VG -0.7344 coefficient
- "vmov.u16 q15, #0x8080 \n" // 128.5
- "1: \n"
+ "vmov.s16 q10, #112 \n" // UB/VR 0.875 coefficient
+ "vmov.s16 q11, #74 \n" // UG -0.5781 coefficient
+ "vmov.s16 q12, #38 \n" // UR -0.2969 coefficient
+ "vmov.s16 q13, #18 \n" // VB -0.1406 coefficient
+ "vmov.s16 q14, #94 \n" // VG -0.7344 coefficient
+ "vmov.u16 q15, #0x8000 \n" // 128.0
+ "1: \n"
"vld4.8 {d0, d2, d4, d6}, [%0]! \n" // load 8 ABGR pixels.
"vld4.8 {d1, d3, d5, d7}, [%0]! \n" // load next 8 ABGR pixels.
+ "subs %4, %4, #16 \n" // 16 processed per loop.
"vpaddl.u8 q2, q2 \n" // B 16 bytes -> 8 shorts.
"vpaddl.u8 q1, q1 \n" // G 16 bytes -> 8 shorts.
"vpaddl.u8 q0, q0 \n" // R 16 bytes -> 8 shorts.
@@ -2224,11 +2234,10 @@
"vpadal.u8 q1, q5 \n" // G 16 bytes -> 8 shorts.
"vpadal.u8 q0, q4 \n" // R 16 bytes -> 8 shorts.
- "vrshr.u16 q0, q0, #1 \n" // 2x average
- "vrshr.u16 q1, q1, #1 \n"
- "vrshr.u16 q2, q2, #1 \n"
+ "vrshr.u16 q0, q0, #2 \n" // average of 4
+ "vrshr.u16 q1, q1, #2 \n"
+ "vrshr.u16 q2, q2, #2 \n"
- "subs %4, %4, #16 \n" // 16 processed per loop.
RGBTOUV(q2, q1, q0)
"vst1.8 {d0}, [%2]! \n" // store 8 pixels U.
"vst1.8 {d1}, [%3]! \n" // store 8 pixels V.
@@ -2251,15 +2260,16 @@
int width) {
asm volatile (
"add %1, %0, %1 \n" // src_stride + src_rgba
- "vmov.s16 q10, #112 / 2 \n" // UB / VR 0.875 coefficient
- "vmov.s16 q11, #74 / 2 \n" // UG -0.5781 coefficient
- "vmov.s16 q12, #38 / 2 \n" // UR -0.2969 coefficient
- "vmov.s16 q13, #18 / 2 \n" // VB -0.1406 coefficient
- "vmov.s16 q14, #94 / 2 \n" // VG -0.7344 coefficient
- "vmov.u16 q15, #0x8080 \n" // 128.5
- "1: \n"
+ "vmov.s16 q10, #112 \n" // UB/VR 0.875 coefficient
+ "vmov.s16 q11, #74 \n" // UG -0.5781 coefficient
+ "vmov.s16 q12, #38 \n" // UR -0.2969 coefficient
+ "vmov.s16 q13, #18 \n" // VB -0.1406 coefficient
+ "vmov.s16 q14, #94 \n" // VG -0.7344 coefficient
+ "vmov.u16 q15, #0x8000 \n" // 128.0
+ "1: \n"
"vld4.8 {d0, d2, d4, d6}, [%0]! \n" // load 8 RGBA pixels.
"vld4.8 {d1, d3, d5, d7}, [%0]! \n" // load next 8 RGBA pixels.
+ "subs %4, %4, #16 \n" // 16 processed per loop.
"vpaddl.u8 q0, q1 \n" // B 16 bytes -> 8 shorts.
"vpaddl.u8 q1, q2 \n" // G 16 bytes -> 8 shorts.
"vpaddl.u8 q2, q3 \n" // R 16 bytes -> 8 shorts.
@@ -2269,11 +2279,10 @@
"vpadal.u8 q1, q6 \n" // G 16 bytes -> 8 shorts.
"vpadal.u8 q2, q7 \n" // R 16 bytes -> 8 shorts.
- "vrshr.u16 q0, q0, #1 \n" // 2x average
- "vrshr.u16 q1, q1, #1 \n"
- "vrshr.u16 q2, q2, #1 \n"
+ "vrshr.u16 q0, q0, #2 \n" // average of 4
+ "vrshr.u16 q1, q1, #2 \n"
+ "vrshr.u16 q2, q2, #2 \n"
- "subs %4, %4, #16 \n" // 16 processed per loop.
RGBTOUV(q0, q1, q2)
"vst1.8 {d0}, [%2]! \n" // store 8 pixels U.
"vst1.8 {d1}, [%3]! \n" // store 8 pixels V.
@@ -2296,15 +2305,16 @@
int width) {
asm volatile (
"add %1, %0, %1 \n" // src_stride + src_rgb24
- "vmov.s16 q10, #112 / 2 \n" // UB / VR 0.875 coefficient
- "vmov.s16 q11, #74 / 2 \n" // UG -0.5781 coefficient
- "vmov.s16 q12, #38 / 2 \n" // UR -0.2969 coefficient
- "vmov.s16 q13, #18 / 2 \n" // VB -0.1406 coefficient
- "vmov.s16 q14, #94 / 2 \n" // VG -0.7344 coefficient
- "vmov.u16 q15, #0x8080 \n" // 128.5
- "1: \n"
+ "vmov.s16 q10, #112 \n" // UB/VR 0.875 coefficient
+ "vmov.s16 q11, #74 \n" // UG -0.5781 coefficient
+ "vmov.s16 q12, #38 \n" // UR -0.2969 coefficient
+ "vmov.s16 q13, #18 \n" // VB -0.1406 coefficient
+ "vmov.s16 q14, #94 \n" // VG -0.7344 coefficient
+ "vmov.u16 q15, #0x8000 \n" // 128.0
+ "1: \n"
"vld3.8 {d0, d2, d4}, [%0]! \n" // load 8 RGB24 pixels.
"vld3.8 {d1, d3, d5}, [%0]! \n" // load next 8 RGB24 pixels.
+ "subs %4, %4, #16 \n" // 16 processed per loop.
"vpaddl.u8 q0, q0 \n" // B 16 bytes -> 8 shorts.
"vpaddl.u8 q1, q1 \n" // G 16 bytes -> 8 shorts.
"vpaddl.u8 q2, q2 \n" // R 16 bytes -> 8 shorts.
@@ -2314,11 +2324,10 @@
"vpadal.u8 q1, q5 \n" // G 16 bytes -> 8 shorts.
"vpadal.u8 q2, q6 \n" // R 16 bytes -> 8 shorts.
- "vrshr.u16 q0, q0, #1 \n" // 2x average
- "vrshr.u16 q1, q1, #1 \n"
- "vrshr.u16 q2, q2, #1 \n"
+ "vrshr.u16 q0, q0, #2 \n" // average of 4
+ "vrshr.u16 q1, q1, #2 \n"
+ "vrshr.u16 q2, q2, #2 \n"
- "subs %4, %4, #16 \n" // 16 processed per loop.
RGBTOUV(q0, q1, q2)
"vst1.8 {d0}, [%2]! \n" // store 8 pixels U.
"vst1.8 {d1}, [%3]! \n" // store 8 pixels V.
@@ -2341,15 +2350,16 @@
int width) {
asm volatile (
"add %1, %0, %1 \n" // src_stride + src_raw
- "vmov.s16 q10, #112 / 2 \n" // UB / VR 0.875 coefficient
- "vmov.s16 q11, #74 / 2 \n" // UG -0.5781 coefficient
- "vmov.s16 q12, #38 / 2 \n" // UR -0.2969 coefficient
- "vmov.s16 q13, #18 / 2 \n" // VB -0.1406 coefficient
- "vmov.s16 q14, #94 / 2 \n" // VG -0.7344 coefficient
- "vmov.u16 q15, #0x8080 \n" // 128.5
- "1: \n"
+ "vmov.s16 q10, #112 \n" // UB/VR 0.875 coefficient
+ "vmov.s16 q11, #74 \n" // UG -0.5781 coefficient
+ "vmov.s16 q12, #38 \n" // UR -0.2969 coefficient
+ "vmov.s16 q13, #18 \n" // VB -0.1406 coefficient
+ "vmov.s16 q14, #94 \n" // VG -0.7344 coefficient
+ "vmov.u16 q15, #0x8000 \n" // 128.0
+ "1: \n"
"vld3.8 {d0, d2, d4}, [%0]! \n" // load 8 RAW pixels.
"vld3.8 {d1, d3, d5}, [%0]! \n" // load next 8 RAW pixels.
+ "subs %4, %4, #16 \n" // 16 processed per loop.
"vpaddl.u8 q2, q2 \n" // B 16 bytes -> 8 shorts.
"vpaddl.u8 q1, q1 \n" // G 16 bytes -> 8 shorts.
"vpaddl.u8 q0, q0 \n" // R 16 bytes -> 8 shorts.
@@ -2359,11 +2369,10 @@
"vpadal.u8 q1, q5 \n" // G 16 bytes -> 8 shorts.
"vpadal.u8 q0, q4 \n" // R 16 bytes -> 8 shorts.
- "vrshr.u16 q0, q0, #1 \n" // 2x average
- "vrshr.u16 q1, q1, #1 \n"
- "vrshr.u16 q2, q2, #1 \n"
+ "vrshr.u16 q0, q0, #2 \n" // average of 4
+ "vrshr.u16 q1, q1, #2 \n"
+ "vrshr.u16 q2, q2, #2 \n"
- "subs %4, %4, #16 \n" // 16 processed per loop.
RGBTOUV(q2, q1, q0)
"vst1.8 {d0}, [%2]! \n" // store 8 pixels U.
"vst1.8 {d1}, [%3]! \n" // store 8 pixels V.
@@ -2387,15 +2396,15 @@
int width) {
asm volatile(
"add %1, %0, %1 \n" // src_stride + src_argb
- "vmov.s16 q10, #112 / 2 \n" // UB / VR 0.875
- // coefficient
- "vmov.s16 q11, #74 / 2 \n" // UG -0.5781 coefficient
- "vmov.s16 q12, #38 / 2 \n" // UR -0.2969 coefficient
- "vmov.s16 q13, #18 / 2 \n" // VB -0.1406 coefficient
- "vmov.s16 q14, #94 / 2 \n" // VG -0.7344 coefficient
- "vmov.u16 q15, #0x8080 \n" // 128.5
- "1: \n"
+ "vmov.s16 q10, #112 \n" // UB/VR 0.875 coefficient
+ "vmov.s16 q11, #74 \n" // UG -0.5781 coefficient
+ "vmov.s16 q12, #38 \n" // UR -0.2969 coefficient
+ "vmov.s16 q13, #18 \n" // VB -0.1406 coefficient
+ "vmov.s16 q14, #94 \n" // VG -0.7344 coefficient
+ "vmov.u16 q15, #0x8000 \n" // 128.0
+ "1: \n"
"vld1.8 {q0}, [%0]! \n" // load 8 RGB565 pixels.
+ "subs %4, %4, #16 \n" // 16 processed per loop.
RGB565TOARGB
"vpaddl.u8 d8, d0 \n" // B 8 bytes -> 4 shorts.
"vpaddl.u8 d10, d1 \n" // G 8 bytes -> 4 shorts.
@@ -2417,11 +2426,10 @@
"vpadal.u8 d11, d1 \n" // G 8 bytes -> 4 shorts.
"vpadal.u8 d13, d2 \n" // R 8 bytes -> 4 shorts.
- "vrshr.u16 q4, q4, #1 \n" // 2x average
- "vrshr.u16 q5, q5, #1 \n"
- "vrshr.u16 q6, q6, #1 \n"
+ "vrshr.u16 q4, q4, #2 \n" // average of 4
+ "vrshr.u16 q5, q5, #2 \n"
+ "vrshr.u16 q6, q6, #2 \n"
- "subs %4, %4, #16 \n" // 16 processed per loop.
"vmul.s16 q8, q4, q10 \n" // B
"vmls.s16 q8, q5, q11 \n" // G
"vmls.s16 q8, q6, q12 \n" // R
@@ -2453,15 +2461,15 @@
int width) {
asm volatile(
"add %1, %0, %1 \n" // src_stride + src_argb
- "vmov.s16 q10, #112 / 2 \n" // UB / VR 0.875
- // coefficient
- "vmov.s16 q11, #74 / 2 \n" // UG -0.5781 coefficient
- "vmov.s16 q12, #38 / 2 \n" // UR -0.2969 coefficient
- "vmov.s16 q13, #18 / 2 \n" // VB -0.1406 coefficient
- "vmov.s16 q14, #94 / 2 \n" // VG -0.7344 coefficient
- "vmov.u16 q15, #0x8080 \n" // 128.5
- "1: \n"
+ "vmov.s16 q10, #112 \n" // UB/VR 0.875 coefficient
+ "vmov.s16 q11, #74 \n" // UG -0.5781 coefficient
+ "vmov.s16 q12, #38 \n" // UR -0.2969 coefficient
+ "vmov.s16 q13, #18 \n" // VB -0.1406 coefficient
+ "vmov.s16 q14, #94 \n" // VG -0.7344 coefficient
+ "vmov.u16 q15, #0x8000 \n" // 128.0
+ "1: \n"
"vld1.8 {q0}, [%0]! \n" // load 8 ARGB1555 pixels.
+ "subs %4, %4, #16 \n" // 16 processed per loop.
RGB555TOARGB
"vpaddl.u8 d8, d0 \n" // B 8 bytes -> 4 shorts.
"vpaddl.u8 d10, d1 \n" // G 8 bytes -> 4 shorts.
@@ -2483,11 +2491,10 @@
"vpadal.u8 d11, d1 \n" // G 8 bytes -> 4 shorts.
"vpadal.u8 d13, d2 \n" // R 8 bytes -> 4 shorts.
- "vrshr.u16 q4, q4, #1 \n" // 2x average
- "vrshr.u16 q5, q5, #1 \n"
- "vrshr.u16 q6, q6, #1 \n"
+ "vrshr.u16 q4, q4, #2 \n" // average of 4
+ "vrshr.u16 q5, q5, #2 \n"
+ "vrshr.u16 q6, q6, #2 \n"
- "subs %4, %4, #16 \n" // 16 processed per loop.
"vmul.s16 q8, q4, q10 \n" // B
"vmls.s16 q8, q5, q11 \n" // G
"vmls.s16 q8, q6, q12 \n" // R
@@ -2517,17 +2524,17 @@
uint8_t* dst_u,
uint8_t* dst_v,
int width) {
- asm volatile(
+ asm volatile (
"add %1, %0, %1 \n" // src_stride + src_argb
- "vmov.s16 q10, #112 / 2 \n" // UB / VR 0.875
- // coefficient
- "vmov.s16 q11, #74 / 2 \n" // UG -0.5781 coefficient
- "vmov.s16 q12, #38 / 2 \n" // UR -0.2969 coefficient
- "vmov.s16 q13, #18 / 2 \n" // VB -0.1406 coefficient
- "vmov.s16 q14, #94 / 2 \n" // VG -0.7344 coefficient
- "vmov.u16 q15, #0x8080 \n" // 128.5
- "1: \n"
+ "vmov.s16 q10, #112 \n" // UB/VR 0.875 coefficient
+ "vmov.s16 q11, #74 \n" // UG -0.5781 coefficient
+ "vmov.s16 q12, #38 \n" // UR -0.2969 coefficient
+ "vmov.s16 q13, #18 \n" // VB -0.1406 coefficient
+ "vmov.s16 q14, #94 \n" // VG -0.7344 coefficient
+ "vmov.u16 q15, #0x8000 \n" // 128.0
+ "1: \n"
"vld1.8 {q0}, [%0]! \n" // load 8 ARGB4444 pixels.
+ "subs %4, %4, #16 \n" // 16 processed per loop.
ARGB4444TOARGB
"vpaddl.u8 d8, d0 \n" // B 8 bytes -> 4 shorts.
"vpaddl.u8 d10, d1 \n" // G 8 bytes -> 4 shorts.
@@ -2549,11 +2556,10 @@
"vpadal.u8 d11, d1 \n" // G 8 bytes -> 4 shorts.
"vpadal.u8 d13, d2 \n" // R 8 bytes -> 4 shorts.
- "vrshr.u16 q0, q4, #1 \n" // 2x average
- "vrshr.u16 q1, q5, #1 \n"
- "vrshr.u16 q2, q6, #1 \n"
+ "vrshr.u16 q0, q4, #2 \n" // average of 4
+ "vrshr.u16 q1, q5, #2 \n"
+ "vrshr.u16 q2, q6, #2 \n"
- "subs %4, %4, #16 \n" // 16 processed per loop.
RGBTOUV(q0, q1, q2)
"vst1.8 {d0}, [%2]! \n" // store 8 pixels U.
"vst1.8 {d1}, [%3]! \n" // store 8 pixels V.
@@ -2574,7 +2580,7 @@
"vmov.u8 d25, #129 \n" // G * 0.5078 coefficient
"vmov.u8 d26, #66 \n" // R * 0.2578 coefficient
"vmov.u8 d27, #16 \n" // Add 16 constant
- "1: \n"
+ "1: \n"
"vld1.8 {q0}, [%0]! \n" // load 8 RGB565 pixels.
"subs %2, %2, #8 \n" // 8 processed per loop.
RGB565TOARGB
@@ -2600,7 +2606,7 @@
"vmov.u8 d25, #129 \n" // G * 0.5078 coefficient
"vmov.u8 d26, #66 \n" // R * 0.2578 coefficient
"vmov.u8 d27, #16 \n" // Add 16 constant
- "1: \n"
+ "1: \n"
"vld1.8 {q0}, [%0]! \n" // load 8 ARGB1555 pixels.
"subs %2, %2, #8 \n" // 8 processed per loop.
ARGB1555TOARGB
@@ -2626,7 +2632,7 @@
"vmov.u8 d25, #129 \n" // G * 0.5078 coefficient
"vmov.u8 d26, #66 \n" // R * 0.2578 coefficient
"vmov.u8 d27, #16 \n" // Add 16 constant
- "1: \n"
+ "1: \n"
"vld1.8 {q0}, [%0]! \n" // load 8 ARGB4444 pixels.
"subs %2, %2, #8 \n" // 8 processed per loop.
ARGB4444TOARGB
@@ -2648,12 +2654,12 @@
uint16_t* dst_ar64,
int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"vld1.8 {q0}, [%0]! \n"
"vld1.8 {q2}, [%0]! \n"
+ "subs %2, %2, #8 \n" // 8 processed per loop.
"vmov.u8 q1, q0 \n"
"vmov.u8 q3, q2 \n"
- "subs %2, %2, #8 \n" // 8 processed per loop.
"vst2.8 {q0, q1}, [%1]! \n" // store 4 pixels
"vst2.8 {q2, q3}, [%1]! \n" // store 4 pixels
"bgt 1b \n"
@@ -2673,16 +2679,16 @@
asm volatile(
"vld1.8 {q4}, [%3] \n" // shuffler
- "1: \n"
+ "1: \n"
"vld1.8 {q0}, [%0]! \n"
"vld1.8 {q2}, [%0]! \n"
+ "subs %2, %2, #8 \n" // 8 processed per loop.
"vtbl.8 d2, {d0, d1}, d8 \n"
"vtbl.8 d3, {d0, d1}, d9 \n"
"vtbl.8 d6, {d4, d5}, d8 \n"
"vtbl.8 d7, {d4, d5}, d9 \n"
"vmov.u8 q0, q1 \n"
"vmov.u8 q2, q3 \n"
- "subs %2, %2, #8 \n" // 8 processed per loop.
"vst2.8 {q0, q1}, [%1]! \n" // store 4 pixels
"vst2.8 {q2, q3}, [%1]! \n" // store 4 pixels
"bgt 1b \n"
@@ -2697,16 +2703,16 @@
uint8_t* dst_argb,
int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"vld1.16 {q0}, [%0]! \n"
"vld1.16 {q1}, [%0]! \n"
"vld1.16 {q2}, [%0]! \n"
"vld1.16 {q3}, [%0]! \n"
+ "subs %2, %2, #8 \n" // 8 processed per loop.
"vshrn.u16 d0, q0, #8 \n"
"vshrn.u16 d1, q1, #8 \n"
"vshrn.u16 d4, q2, #8 \n"
"vshrn.u16 d5, q3, #8 \n"
- "subs %2, %2, #8 \n" // 8 processed per loop.
"vst1.8 {q0}, [%1]! \n" // store 4 pixels
"vst1.8 {q2}, [%1]! \n" // store 4 pixels
"bgt 1b \n"
@@ -2725,16 +2731,16 @@
asm volatile(
"vld1.8 {d8}, [%3] \n" // shuffler
- "1: \n"
+ "1: \n"
"vld1.16 {q0}, [%0]! \n"
"vld1.16 {q1}, [%0]! \n"
"vld1.16 {q2}, [%0]! \n"
"vld1.16 {q3}, [%0]! \n"
+ "subs %2, %2, #8 \n" // 8 processed per loop.
"vtbl.8 d0, {d0, d1}, d8 \n"
"vtbl.8 d1, {d2, d3}, d8 \n"
"vtbl.8 d4, {d4, d5}, d8 \n"
"vtbl.8 d5, {d6, d7}, d8 \n"
- "subs %2, %2, #8 \n" // 8 processed per loop.
"vst1.8 {q0}, [%1]! \n" // store 4 pixels
"vst1.8 {q2}, [%1]! \n" // store 4 pixels
"bgt 1b \n"
@@ -2754,10 +2760,11 @@
// B * 0.1140 coefficient = 29
// G * 0.5870 coefficient = 150
// R * 0.2990 coefficient = 77
-// Add 0.5 = 0x80
-static const struct RgbConstants kRgb24JPEGConstants = {{29, 150, 77, 0}, 128};
+// Add 0.5
+static const struct RgbConstants kRgb24JPEGConstants = {{29, 150, 77, 0},
+ 0x0080};
-static const struct RgbConstants kRawJPEGConstants = {{77, 150, 29, 0}, 128};
+static const struct RgbConstants kRawJPEGConstants = {{77, 150, 29, 0}, 0x0080};
// RGB to BT.601 coefficients
// B * 0.1016 coefficient = 25
@@ -2771,17 +2778,17 @@
static const struct RgbConstants kRawI601Constants = {{66, 129, 25, 0}, 0x1080};
// ARGB expects first 3 values to contain RGB and 4th value is ignored.
-void ARGBToYMatrixRow_NEON(const uint8_t* src_argb,
- uint8_t* dst_y,
- int width,
- const struct RgbConstants* rgbconstants) {
+static void ARGBToYMatrixRow_NEON(const uint8_t* src_argb,
+ uint8_t* dst_y,
+ int width,
+ const struct RgbConstants* rgbconstants) {
asm volatile(
"vld1.8 {d0}, [%3] \n" // load rgbconstants
"vdup.u8 d20, d0[0] \n"
"vdup.u8 d21, d0[1] \n"
"vdup.u8 d22, d0[2] \n"
"vdup.u16 q12, d0[2] \n"
- "1: \n"
+ "1: \n"
"vld4.8 {d0, d2, d4, d6}, [%0]! \n" // load 16 pixels of ARGB
"vld4.8 {d1, d3, d5, d7}, [%0]! \n"
"subs %2, %2, #16 \n" // 16 processed per loop.
@@ -2821,17 +2828,17 @@
// RGBA expects first value to be A and ignored, then 3 values to contain RGB.
// Same code as ARGB, except the LD4
-void RGBAToYMatrixRow_NEON(const uint8_t* src_rgba,
- uint8_t* dst_y,
- int width,
- const struct RgbConstants* rgbconstants) {
+static void RGBAToYMatrixRow_NEON(const uint8_t* src_rgba,
+ uint8_t* dst_y,
+ int width,
+ const struct RgbConstants* rgbconstants) {
asm volatile(
"vld1.8 {d0}, [%3] \n" // load rgbconstants
"vdup.u8 d20, d0[0] \n"
"vdup.u8 d21, d0[1] \n"
"vdup.u8 d22, d0[2] \n"
"vdup.u16 q12, d0[2] \n"
- "1: \n"
+ "1: \n"
"vld4.8 {d0, d2, d4, d6}, [%0]! \n" // load 16 pixels of RGBA
"vld4.8 {d1, d3, d5, d7}, [%0]! \n"
"subs %2, %2, #16 \n" // 16 processed per loop.
@@ -2865,17 +2872,17 @@
RGBAToYMatrixRow_NEON(src_bgra, dst_y, width, &kRawI601Constants);
}
-void RGBToYMatrixRow_NEON(const uint8_t* src_rgb,
- uint8_t* dst_y,
- int width,
- const struct RgbConstants* rgbconstants) {
+static void RGBToYMatrixRow_NEON(const uint8_t* src_rgb,
+ uint8_t* dst_y,
+ int width,
+ const struct RgbConstants* rgbconstants) {
asm volatile(
"vld1.8 {d0}, [%3] \n" // load rgbconstants
"vdup.u8 d20, d0[0] \n"
"vdup.u8 d21, d0[1] \n"
"vdup.u8 d22, d0[2] \n"
"vdup.u16 q12, d0[2] \n"
- "1: \n"
+ "1: \n"
"vld3.8 {d2, d4, d6}, [%0]! \n" // load 16 pixels of
// RGB24.
"vld3.8 {d3, d5, d7}, [%0]! \n"
@@ -2932,7 +2939,7 @@
"rsb %4, #256 \n"
"vdup.8 d4, %4 \n"
// General purpose row blend.
- "1: \n"
+ "1: \n"
"vld1.8 {q0}, [%1]! \n"
"vld1.8 {q1}, [%2]! \n"
"subs %3, %3, #16 \n"
@@ -2947,7 +2954,7 @@
"b 99f \n"
// Blend 50 / 50.
- "50: \n"
+ "50: \n"
"vld1.8 {q0}, [%1]! \n"
"vld1.8 {q1}, [%2]! \n"
"subs %3, %3, #16 \n"
@@ -2957,13 +2964,13 @@
"b 99f \n"
// Blend 100 / 0 - Copy row unchanged.
- "100: \n"
+ "100: \n"
"vld1.8 {q0}, [%1]! \n"
"subs %3, %3, #16 \n"
"vst1.8 {q0}, [%0]! \n"
"bgt 100b \n"
- "99: \n"
+ "99: \n"
: "+r"(dst_ptr), // %0
"+r"(src_ptr), // %1
"+r"(src_stride), // %2
@@ -2992,7 +2999,7 @@
"vdup.16 d17, %4 \n"
"vdup.16 d16, %5 \n"
// General purpose row blend.
- "1: \n"
+ "1: \n"
"vld1.16 {q0}, [%1]! \n"
"vld1.16 {q1}, [%2]! \n"
"subs %3, %3, #8 \n"
@@ -3007,7 +3014,7 @@
"b 99f \n"
// Blend 50 / 50.
- "50: \n"
+ "50: \n"
"vld1.16 {q0}, [%1]! \n"
"vld1.16 {q1}, [%2]! \n"
"subs %3, %3, #8 \n"
@@ -3017,13 +3024,13 @@
"b 99f \n"
// Blend 100 / 0 - Copy row unchanged.
- "100: \n"
+ "100: \n"
"vld1.16 {q0}, [%1]! \n"
"subs %3, %3, #8 \n"
"vst1.16 {q0}, [%0]! \n"
"bgt 100b \n"
- "99: \n"
+ "99: \n"
: "+r"(dst_ptr), // %0
"+r"(src_ptr), // %1
"+r"(src_ptr1), // %2
@@ -3042,7 +3049,7 @@
"subs %3, #8 \n"
"blt 89f \n"
// Blend 8 pixels.
- "8: \n"
+ "8: \n"
"vld4.8 {d0, d1, d2, d3}, [%0]! \n" // load 8 pixels of ARGB0.
"vld4.8 {d4, d5, d6, d7}, [%1]! \n" // load 8 pixels of ARGB1.
"subs %3, %3, #8 \n" // 8 processed per loop.
@@ -3060,12 +3067,12 @@
"vst4.8 {d0, d1, d2, d3}, [%2]! \n" // store 8 pixels of ARGB.
"bge 8b \n"
- "89: \n"
+ "89: \n"
"adds %3, #8-1 \n"
"blt 99f \n"
// Blend 1 pixels.
- "1: \n"
+ "1: \n"
"vld4.8 {d0[0],d1[0],d2[0],d3[0]}, [%0]! \n" // load 1 pixel ARGB0.
"vld4.8 {d4[0],d5[0],d6[0],d7[0]}, [%1]! \n" // load 1 pixel ARGB1.
"subs %3, %3, #1 \n" // 1 processed per loop.
@@ -3083,7 +3090,7 @@
"vst4.8 {d0[0],d1[0],d2[0],d3[0]}, [%2]! \n" // store 1 pixel.
"bge 1b \n"
- "99: \n"
+ "99: \n"
: "+r"(src_argb), // %0
"+r"(src_argb1), // %1
@@ -3101,7 +3108,7 @@
"vmov.u16 q15, #0x00ff \n" // 255 for rounding up
// Attenuate 8 pixels.
- "1: \n"
+ "1: \n"
"vld4.8 {d0, d1, d2, d3}, [%0]! \n" // load 8 pixels of ARGB.
"subs %2, %2, #8 \n" // 8 processed per loop.
"vmull.u8 q10, d0, d3 \n" // b * a
@@ -3133,7 +3140,7 @@
"vdup.u16 q10, %4 \n" // interval add
// 8 pixel loop.
- "1: \n"
+ "1: \n"
"vld4.8 {d0, d2, d4, d6}, [%0] \n" // load 8 pixels of ARGB.
"subs %1, %1, #8 \n" // 8 processed per loop.
"vmovl.u8 q0, d0 \n" // b (0 .. 255)
@@ -3174,7 +3181,7 @@
"vshr.u16 q0, q0, #1 \n" // scale / 2.
// 8 pixel loop.
- "1: \n"
+ "1: \n"
"vld4.8 {d20, d22, d24, d26}, [%0]! \n" // load 8 pixels of ARGB.
"subs %2, %2, #8 \n" // 8 processed per loop.
"vmovl.u8 q10, d20 \n" // b (0 .. 255)
@@ -3206,7 +3213,7 @@
"vmov.u8 d24, #29 \n" // B * 0.1140 coefficient
"vmov.u8 d25, #150 \n" // G * 0.5870 coefficient
"vmov.u8 d26, #77 \n" // R * 0.2990 coefficient
- "1: \n"
+ "1: \n"
"vld4.8 {d0, d1, d2, d3}, [%0]! \n" // load 8 ARGB pixels.
"subs %2, %2, #8 \n" // 8 processed per loop.
"vmull.u8 q2, d0, d24 \n" // B
@@ -3239,7 +3246,7 @@
"vmov.u8 d28, #24 \n" // BB coefficient
"vmov.u8 d29, #98 \n" // BG coefficient
"vmov.u8 d30, #50 \n" // BR coefficient
- "1: \n"
+ "1: \n"
"vld4.8 {d0, d1, d2, d3}, [%0] \n" // load 8 ARGB pixels.
"subs %1, %1, #8 \n" // 8 processed per loop.
"vmull.u8 q2, d0, d20 \n" // B to Sepia B
@@ -3275,7 +3282,7 @@
"vmovl.s8 q0, d4 \n" // B,G coefficients s16.
"vmovl.s8 q1, d5 \n" // R,A coefficients s16.
- "1: \n"
+ "1: \n"
"vld4.8 {d16, d18, d20, d22}, [%0]! \n" // load 8 ARGB pixels.
"subs %2, %2, #8 \n" // 8 processed per loop.
"vmovl.u8 q8, d16 \n" // b (0 .. 255) 16 bit
@@ -3331,7 +3338,7 @@
int width) {
asm volatile(
// 8 pixel loop.
- "1: \n"
+ "1: \n"
"vld4.8 {d0, d2, d4, d6}, [%0]! \n" // load 8 ARGB pixels.
"vld4.8 {d1, d3, d5, d7}, [%1]! \n" // load 8 more ARGB
"subs %3, %3, #8 \n" // 8 processed per loop.
@@ -3360,7 +3367,7 @@
int width) {
asm volatile(
// 8 pixel loop.
- "1: \n"
+ "1: \n"
"vld4.8 {d0, d1, d2, d3}, [%0]! \n" // load 8 ARGB pixels.
"vld4.8 {d4, d5, d6, d7}, [%1]! \n" // load 8 more ARGB
"subs %3, %3, #8 \n" // 8 processed per loop.
@@ -3383,7 +3390,7 @@
int width) {
asm volatile(
// 8 pixel loop.
- "1: \n"
+ "1: \n"
"vld4.8 {d0, d1, d2, d3}, [%0]! \n" // load 8 ARGB pixels.
"vld4.8 {d4, d5, d6, d7}, [%1]! \n" // load 8 more ARGB
"subs %3, %3, #8 \n" // 8 processed per loop.
@@ -3411,7 +3418,7 @@
asm volatile(
"vmov.u8 d3, #255 \n" // alpha
// 8 pixel loop.
- "1: \n"
+ "1: \n"
"vld1.8 {d0}, [%0]! \n" // load 8 sobelx.
"vld1.8 {d1}, [%1]! \n" // load 8 sobely.
"subs %3, %3, #8 \n" // 8 processed per loop.
@@ -3435,7 +3442,7 @@
int width) {
asm volatile(
// 16 pixel loop.
- "1: \n"
+ "1: \n"
"vld1.8 {q0}, [%0]! \n" // load 16 sobelx.
"vld1.8 {q1}, [%1]! \n" // load 16 sobely.
"subs %3, %3, #16 \n" // 16 processed per loop.
@@ -3462,7 +3469,7 @@
asm volatile(
"vmov.u8 d3, #255 \n" // alpha
// 8 pixel loop.
- "1: \n"
+ "1: \n"
"vld1.8 {d2}, [%0]! \n" // load 8 sobelx.
"vld1.8 {d0}, [%1]! \n" // load 8 sobely.
"subs %3, %3, #8 \n" // 8 processed per loop.
@@ -3487,9 +3494,10 @@
uint8_t* dst_sobelx,
int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"vld1.8 {d0}, [%0],%5 \n" // top
"vld1.8 {d1}, [%0],%6 \n"
+ "subs %4, %4, #8 \n" // 8 pixels
"vsubl.u8 q0, d0, d1 \n"
"vld1.8 {d2}, [%1],%5 \n" // center * 2
"vld1.8 {d3}, [%1],%6 \n"
@@ -3498,7 +3506,6 @@
"vadd.s16 q0, q0, q1 \n"
"vld1.8 {d2}, [%2],%5 \n" // bottom
"vld1.8 {d3}, [%2],%6 \n"
- "subs %4, %4, #8 \n" // 8 pixels
"vsubl.u8 q1, d2, d3 \n"
"vadd.s16 q0, q0, q1 \n"
"vabs.s16 q0, q0 \n"
@@ -3525,9 +3532,10 @@
uint8_t* dst_sobely,
int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"vld1.8 {d0}, [%0],%4 \n" // left
"vld1.8 {d1}, [%1],%4 \n"
+ "subs %3, %3, #8 \n" // 8 pixels
"vsubl.u8 q0, d0, d1 \n"
"vld1.8 {d2}, [%0],%4 \n" // center * 2
"vld1.8 {d3}, [%1],%4 \n"
@@ -3536,7 +3544,6 @@
"vadd.s16 q0, q0, q1 \n"
"vld1.8 {d2}, [%0],%5 \n" // right
"vld1.8 {d3}, [%1],%5 \n"
- "subs %3, %3, #8 \n" // 8 pixels
"vsubl.u8 q1, d2, d3 \n"
"vadd.s16 q0, q0, q1 \n"
"vabs.s16 q0, q0 \n"
@@ -3554,59 +3561,41 @@
}
// %y passes a float as a scalar vector for vector * scalar multiply.
-// the regoster must be d0 to d15 and indexed with [0] or [1] to access
+// the register must be d0 to d15 and indexed with [0] or [1] to access
// the float in the first or second float of the d-reg
-void HalfFloat1Row_NEON(const uint16_t* src,
- uint16_t* dst,
- float /*unused*/,
- int width) {
- asm volatile(
-
- "1: \n"
- "vld1.8 {q1}, [%0]! \n" // load 8 shorts
- "subs %2, %2, #8 \n" // 8 pixels per loop
- "vmovl.u16 q2, d2 \n" // 8 int's
- "vmovl.u16 q3, d3 \n"
- "vcvt.f32.u32 q2, q2 \n" // 8 floats
- "vcvt.f32.u32 q3, q3 \n"
- "vmul.f32 q2, q2, %y3 \n" // adjust exponent
- "vmul.f32 q3, q3, %y3 \n"
- "vqshrn.u32 d2, q2, #13 \n" // isolate halffloat
- "vqshrn.u32 d3, q3, #13 \n"
- "vst1.8 {q1}, [%1]! \n"
- "bgt 1b \n"
- : "+r"(src), // %0
- "+r"(dst), // %1
- "+r"(width) // %2
- : "w"(1.9259299444e-34f) // %3
- : "cc", "memory", "q1", "q2", "q3");
-}
-
void HalfFloatRow_NEON(const uint16_t* src,
uint16_t* dst,
float scale,
int width) {
asm volatile(
- "1: \n"
- "vld1.8 {q1}, [%0]! \n" // load 8 shorts
- "subs %2, %2, #8 \n" // 8 pixels per loop
- "vmovl.u16 q2, d2 \n" // 8 int's
- "vmovl.u16 q3, d3 \n"
- "vcvt.f32.u32 q2, q2 \n" // 8 floats
- "vcvt.f32.u32 q3, q3 \n"
- "vmul.f32 q2, q2, %y3 \n" // adjust exponent
- "vmul.f32 q3, q3, %y3 \n"
- "vqshrn.u32 d2, q2, #13 \n" // isolate halffloat
- "vqshrn.u32 d3, q3, #13 \n"
- "vst1.8 {q1}, [%1]! \n"
+ "1: \n"
+ "vld1.16 {q0, q1}, [%0]! \n" // load 16 shorts
+ "subs %2, %2, #16 \n" // 16 pixels per loop
+ "vmovl.u16 q8, d0 \n"
+ "vmovl.u16 q9, d1 \n"
+ "vmovl.u16 q10, d2 \n"
+ "vmovl.u16 q11, d3 \n"
+ "vcvt.f32.u32 q8, q8 \n"
+ "vcvt.f32.u32 q9, q9 \n"
+ "vcvt.f32.u32 q10, q10 \n"
+ "vcvt.f32.u32 q11, q11 \n"
+ "vmul.f32 q8, q8, %y3 \n" // adjust exponent
+ "vmul.f32 q9, q9, %y3 \n"
+ "vmul.f32 q10, q10, %y3 \n"
+ "vmul.f32 q11, q11, %y3 \n"
+ "vqshrn.u32 d0, q8, #13 \n" // isolate halffloat
+ "vqshrn.u32 d1, q9, #13 \n"
+ "vqshrn.u32 d2, q10, #13 \n"
+ "vqshrn.u32 d3, q11, #13 \n"
+ "vst1.16 {q0, q1}, [%1]! \n" // store 16 fp16
"bgt 1b \n"
: "+r"(src), // %0
"+r"(dst), // %1
"+r"(width) // %2
: "w"(scale * 1.9259299444e-34f) // %3
- : "cc", "memory", "q1", "q2", "q3");
+ : "cc", "memory", "q0", "q1", "q8", "q9", "q10", "q11");
}
void ByteToFloatRow_NEON(const uint8_t* src,
@@ -3615,7 +3604,7 @@
int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"vld1.8 {d2}, [%0]! \n" // load 8 bytes
"subs %2, %2, #8 \n" // 8 pixels per loop
"vmovl.u8 q1, d2 \n" // 8 shorts
@@ -3646,9 +3635,10 @@
"vmov.u16 d6, #4 \n" // constant 4
"vmov.u16 d7, #6 \n" // constant 6
- "1: \n"
+ "1: \n"
"vld1.16 {q1}, [%0]! \n" // load 8 samples, 5 rows
"vld1.16 {q2}, [%4]! \n"
+ "subs %6, %6, #8 \n" // 8 processed per loop
"vaddl.u16 q0, d2, d4 \n" // * 1
"vaddl.u16 q1, d3, d5 \n" // * 1
"vld1.16 {q2}, [%1]! \n"
@@ -3660,7 +3650,6 @@
"vld1.16 {q2}, [%3]! \n"
"vmlal.u16 q0, d4, d6 \n" // * 4
"vmlal.u16 q1, d5, d6 \n" // * 4
- "subs %6, %6, #8 \n" // 8 processed per loop
"vst1.32 {q0, q1}, [%5]! \n" // store 8 samples
"bgt 1b \n"
: "+r"(src0), // %0
@@ -3683,9 +3672,10 @@
"vmov.u32 q10, #4 \n" // constant 4
"vmov.u32 q11, #6 \n" // constant 6
- "1: \n"
+ "1: \n"
"vld1.32 {q0, q1}, [%0]! \n" // load 12 source samples
"vld1.32 {q2}, [%0] \n"
+ "subs %5, %5, #8 \n" // 8 processed per loop
"vadd.u32 q0, q0, q1 \n" // * 1
"vadd.u32 q1, q1, q2 \n" // * 1
"vld1.32 {q2, q3}, [%2]! \n"
@@ -3697,7 +3687,6 @@
"vadd.u32 q3, q3, q9 \n"
"vmla.u32 q0, q2, q10 \n" // * 4
"vmla.u32 q1, q3, q10 \n" // * 4
- "subs %5, %5, #8 \n" // 8 processed per loop
"vqshrn.u32 d0, q0, #8 \n" // round and pack
"vqshrn.u32 d1, q1, #8 \n"
"vst1.u16 {q0}, [%4]! \n" // store 8 samples
@@ -3718,14 +3707,14 @@
uint8_t* dst_yuv24,
int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"vld1.8 {q2}, [%0]! \n" // load 16 Y values
"vld2.8 {d0, d2}, [%1]! \n" // load 8 VU values
+ "subs %3, %3, #16 \n" // 16 pixels per loop
"vmov d1, d0 \n"
"vzip.u8 d0, d1 \n" // VV
"vmov d3, d2 \n"
"vzip.u8 d2, d3 \n" // UU
- "subs %3, %3, #16 \n" // 16 pixels per loop
"vst3.8 {d0, d2, d4}, [%2]! \n" // store 16 YUV pixels
"vst3.8 {d1, d3, d5}, [%2]! \n"
"bgt 1b \n"
@@ -3743,10 +3732,11 @@
int width) {
asm volatile(
"add %1, %0, %1 \n" // src_stride + src_AYUV
- "1: \n"
+ "1: \n"
"vld4.8 {d0, d2, d4, d6}, [%0]! \n" // load 8 AYUV pixels.
"vld4.8 {d1, d3, d5, d7}, [%0]! \n" // load next 8 AYUV
// pixels.
+ "subs %3, %3, #16 \n" // 16 processed per loop.
"vpaddl.u8 q0, q0 \n" // V 16 bytes -> 8 shorts.
"vpaddl.u8 q1, q1 \n" // U 16 bytes -> 8 shorts.
"vld4.8 {d8, d10, d12, d14}, [%1]! \n" // load 8 more AYUV
@@ -3757,7 +3747,6 @@
"vpadal.u8 q1, q5 \n" // G 16 bytes -> 8 shorts.
"vqrshrun.s16 d1, q0, #2 \n" // 2x2 average
"vqrshrun.s16 d0, q1, #2 \n"
- "subs %3, %3, #16 \n" // 16 processed per loop.
"vst2.8 {d0, d1}, [%2]! \n" // store 8 pixels UV.
"bgt 1b \n"
: "+r"(src_ayuv), // %0
@@ -3774,10 +3763,11 @@
int width) {
asm volatile(
"add %1, %0, %1 \n" // src_stride + src_AYUV
- "1: \n"
+ "1: \n"
"vld4.8 {d0, d2, d4, d6}, [%0]! \n" // load 8 AYUV pixels.
"vld4.8 {d1, d3, d5, d7}, [%0]! \n" // load next 8 AYUV
// pixels.
+ "subs %3, %3, #16 \n" // 16 processed per loop.
"vpaddl.u8 q0, q0 \n" // V 16 bytes -> 8 shorts.
"vpaddl.u8 q1, q1 \n" // U 16 bytes -> 8 shorts.
"vld4.8 {d8, d10, d12, d14}, [%1]! \n" // load 8 more AYUV
@@ -3788,7 +3778,6 @@
"vpadal.u8 q1, q5 \n" // G 16 bytes -> 8 shorts.
"vqrshrun.s16 d0, q0, #2 \n" // 2x2 average
"vqrshrun.s16 d1, q1, #2 \n"
- "subs %3, %3, #16 \n" // 16 processed per loop.
"vst2.8 {d0, d1}, [%2]! \n" // store 8 pixels VU.
"bgt 1b \n"
: "+r"(src_ayuv), // %0
@@ -3803,7 +3792,7 @@
// Similar to ARGBExtractAlphaRow_NEON
void AYUVToYRow_NEON(const uint8_t* src_ayuv, uint8_t* dst_y, int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"vld4.8 {d0, d2, d4, d6}, [%0]! \n" // load 8 AYUV pixels
"vld4.8 {d1, d3, d5, d7}, [%0]! \n" // load next 8 AYUV pixels
"subs %2, %2, #16 \n" // 16 processed per loop
@@ -3819,11 +3808,11 @@
// Convert UV plane of NV12 to VU of NV21.
void SwapUVRow_NEON(const uint8_t* src_uv, uint8_t* dst_vu, int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"vld2.8 {d0, d2}, [%0]! \n" // load 16 UV values
"vld2.8 {d1, d3}, [%0]! \n"
- "vorr.u8 q2, q0, q0 \n" // move U after V
"subs %2, %2, #16 \n" // 16 pixels per loop
+ "vmov.u8 q2, q0 \n" // move U after V
"vst2.8 {q1, q2}, [%1]! \n" // store 16 VU pixels
"bgt 1b \n"
: "+r"(src_uv), // %0
@@ -3842,18 +3831,18 @@
const uint8_t* src_u_1 = src_u + src_stride_u;
const uint8_t* src_v_1 = src_v + src_stride_v;
asm volatile(
- "1: \n"
+ "1: \n"
"vld1.8 {q0}, [%0]! \n" // load 16 U values
"vld1.8 {q1}, [%2]! \n" // load 16 V values
"vld1.8 {q2}, [%1]! \n"
"vld1.8 {q3}, [%3]! \n"
+ "subs %5, %5, #16 \n" // 16 src pixels per loop
"vpaddl.u8 q0, q0 \n" // half size
"vpaddl.u8 q1, q1 \n"
"vpadal.u8 q0, q2 \n"
"vpadal.u8 q1, q3 \n"
"vqrshrn.u16 d0, q0, #2 \n"
"vqrshrn.u16 d1, q1, #2 \n"
- "subs %5, %5, #16 \n" // 16 src pixels per loop
"vst2.8 {d0, d1}, [%4]! \n" // store 8 UV pixels
"bgt 1b \n"
: "+r"(src_u), // %0
@@ -3874,11 +3863,11 @@
int shift = depth - 16; // Negative for right shift.
asm volatile(
"vdup.16 q2, %4 \n"
- "1: \n"
+ "1: \n"
"vld2.16 {q0, q1}, [%0]! \n" // load 8 UV
+ "subs %3, %3, #8 \n" // 8 src pixels per loop
"vshl.u16 q0, q0, q2 \n"
"vshl.u16 q1, q1, q2 \n"
- "subs %3, %3, #8 \n" // 8 src pixels per loop
"vst1.16 {q0}, [%1]! \n" // store 8 U pixels
"vst1.16 {q1}, [%2]! \n" // store 8 V pixels
"bgt 1b \n"
@@ -3898,12 +3887,12 @@
int shift = 16 - depth;
asm volatile(
"vdup.16 q2, %4 \n"
- "1: \n"
+ "1: \n"
"vld1.16 {q0}, [%0]! \n" // load 8 U
"vld1.16 {q1}, [%1]! \n" // load 8 V
+ "subs %3, %3, #8 \n" // 8 src pixels per loop
"vshl.u16 q0, q0, q2 \n"
"vshl.u16 q1, q1, q2 \n"
- "subs %3, %3, #8 \n" // 8 src pixels per loop
"vst2.16 {q0, q1}, [%2]! \n" // store 8 UV pixels
"bgt 1b \n"
: "+r"(src_u), // %0
@@ -3920,14 +3909,14 @@
int width) {
asm volatile(
"vdup.16 q2, %3 \n"
- "1: \n"
+ "1: \n"
"vld1.16 {q0}, [%0]! \n"
"vld1.16 {q1}, [%0]! \n"
+ "subs %2, %2, #16 \n" // 16 src pixels per loop
"vmul.u16 q0, q0, q2 \n"
"vmul.u16 q1, q1, q2 \n"
"vst1.16 {q0}, [%1]! \n"
"vst1.16 {q1}, [%1]! \n"
- "subs %2, %2, #16 \n" // 16 src pixels per loop
"bgt 1b \n"
: "+r"(src_y), // %0
"+r"(dst_y), // %1
@@ -3942,8 +3931,9 @@
int width) {
asm volatile(
"vdup.16 d8, %3 \n"
- "1: \n"
+ "1: \n"
"vld1.16 {q2, q3}, [%0]! \n"
+ "subs %2, %2, #16 \n" // 16 src pixels per loop
"vmull.u16 q0, d4, d8 \n"
"vmull.u16 q1, d5, d8 \n"
"vmull.u16 q2, d6, d8 \n"
@@ -3953,7 +3943,6 @@
"vshrn.u32 d2, q2, #16 \n"
"vshrn.u32 d3, q3, #16 \n"
"vst1.16 {q0, q1}, [%1]! \n" // store 16 pixels
- "subs %2, %2, #16 \n" // 16 src pixels per loop
"bgt 1b \n"
: "+r"(src_y), // %0
"+r"(dst_y), // %1
@@ -3974,14 +3963,14 @@
int shift = 15 - __builtin_clz((int32_t)scale); // Negative shl is shr
asm volatile(
"vdup.16 q2, %3 \n"
- "1: \n"
+ "1: \n"
"vld1.16 {q0}, [%0]! \n"
"vld1.16 {q1}, [%0]! \n"
+ "subs %2, %2, #16 \n" // 16 src pixels per loop
"vshl.u16 q0, q0, q2 \n" // shr = q2 is negative
"vshl.u16 q1, q1, q2 \n"
"vqmovn.u16 d0, q0 \n"
"vqmovn.u16 d1, q1 \n"
- "subs %2, %2, #16 \n" // 16 src pixels per loop
"vst1.8 {q0}, [%1]! \n"
"bgt 1b \n"
: "+r"(src_y), // %0
@@ -3991,6 +3980,41 @@
: "cc", "memory", "q0", "q1", "q2");
}
+// Use scale to convert J420 to I420
+// scale parameter is 8.8 fixed point but limited to 0 to 255
+// Function is based on DivideRow, but adds a bias
+// Does not clamp
+void Convert8To8Row_NEON(const uint8_t* src_y,
+ uint8_t* dst_y,
+ int scale,
+ int bias,
+ int width) {
+ asm volatile(
+ "vdup.8 d8, %3 \n"
+ "vdup.8 q5, %4 \n"
+ "1: \n"
+ "vld1.8 {q2, q3}, [%0]! \n"
+ "subs %2, %2, #32 \n" // 32 src pixels per loop
+ "vmull.u8 q0, d4, d8 \n"
+ "vmull.u8 q1, d5, d8 \n"
+ "vmull.u8 q2, d6, d8 \n"
+ "vmull.u8 q3, d7, d8 \n"
+ "vshrn.u16 d0, q0, #8 \n"
+ "vshrn.u16 d1, q1, #8 \n"
+ "vshrn.u16 d2, q2, #8 \n"
+ "vshrn.u16 d3, q3, #8 \n"
+ "vadd.u8 q0, q0, q5 \n"
+ "vadd.u8 q1, q1, q5 \n"
+ "vst1.8 {q0, q1}, [%1]! \n" // store 32 pixels
+ "bgt 1b \n"
+ : "+r"(src_y), // %0
+ "+r"(dst_y), // %1
+ "+r"(width) // %2
+ : "r"(scale), // %3
+ "r"(bias) // %4
+ : "cc", "memory", "q0", "q1", "q2", "q3", "d8", "q5");
+}
+
#endif // !defined(LIBYUV_DISABLE_NEON) && defined(__ARM_NEON__)..
#ifdef __cplusplus
diff --git a/source/row_neon64.cc b/source/row_neon64.cc
index 1679f87..0a0427c 100644
--- a/source/row_neon64.cc
+++ b/source/row_neon64.cc
@@ -28,35 +28,96 @@
// Read 8 Y, 4 U and 4 V from 422
#define READYUV422 \
"ldr d0, [%[src_y]], #8 \n" \
- "ld1 {v1.s}[0], [%[src_u]], #4 \n" \
- "ld1 {v1.s}[1], [%[src_v]], #4 \n" \
+ "ldr s1, [%[src_u]], #4 \n" \
+ "ldr s2, [%[src_v]], #4 \n" \
"zip1 v0.16b, v0.16b, v0.16b \n" \
"prfm pldl1keep, [%[src_y], 448] \n" \
- "zip1 v1.16b, v1.16b, v1.16b \n" \
+ "zip1 v1.8b, v1.8b, v1.8b \n" \
+ "zip1 v2.8b, v2.8b, v2.8b \n" \
"prfm pldl1keep, [%[src_u], 128] \n" \
"prfm pldl1keep, [%[src_v], 128] \n"
+// Read 8 Y, 4 U and 4 V from 210
+#define READYUV210 \
+ "ldr q2, [%[src_y]], #16 \n" \
+ "ldr d1, [%[src_u]], #8 \n" \
+ "ldr d3, [%[src_v]], #8 \n" \
+ "shl v0.8h, v2.8h, #6 \n" \
+ "usra v0.8h, v2.8h, #4 \n" \
+ "prfm pldl1keep, [%[src_y], 448] \n" \
+ "zip1 v2.8h, v3.8h, v3.8h \n" \
+ "zip1 v3.8h, v1.8h, v1.8h \n" \
+ "uqshrn v1.8b, v3.8h, #2 \n" \
+ "uqshrn2 v1.16b, v2.8h, #2 \n" \
+ "prfm pldl1keep, [%[src_u], 128] \n" \
+ "prfm pldl1keep, [%[src_v], 128] \n"
+
+// Read 8 Y, 4 U and 4 V interleaved from 210
+#define READYUVP210 \
+ "ldr q0, [%[src_y]], #16 \n" \
+ "ldr q1, [%[src_uv]], #16 \n" \
+ "prfm pldl1keep, [%[src_y], 448] \n" \
+ "tbl v1.16b, {v1.16b}, v2.16b \n"
+
+// Read 8 Y, 4 U and 4 V from 212
+#define READYUV212 \
+ "ldr q2, [%[src_y]], #16 \n" \
+ "ldr d1, [%[src_u]], #8 \n" \
+ "ldr d3, [%[src_v]], #8 \n" \
+ "shl v0.8h, v2.8h, #4 \n" \
+ "usra v0.8h, v2.8h, #8 \n" \
+ "prfm pldl1keep, [%[src_y], 448] \n" \
+ "zip1 v2.8h, v3.8h, v3.8h \n" \
+ "zip1 v3.8h, v1.8h, v1.8h \n" \
+ "uqshrn v1.8b, v3.8h, #4 \n" \
+ "uqshrn2 v1.16b, v2.8h, #4 \n" \
+ "prfm pldl1keep, [%[src_u], 128] \n" \
+ "prfm pldl1keep, [%[src_v], 128] \n"
+
+// Read 8 Y, 8 U and 8 V from 410
+#define READYUV410 \
+ "ldr q1, [%[src_y]], #16 \n" \
+ "ldr q2, [%[src_u]], #16 \n" \
+ "ldr q3, [%[src_v]], #16 \n" \
+ "shl v0.8h, v1.8h, #6 \n" \
+ "usra v0.8h, v1.8h, #4 \n" \
+ "prfm pldl1keep, [%[src_y], 448] \n" \
+ "uqshrn v1.8b, v2.8h, #2 \n" \
+ "uqshrn2 v1.16b, v3.8h, #2 \n" \
+ "prfm pldl1keep, [%[src_u], 128] \n" \
+ "prfm pldl1keep, [%[src_v], 128] \n"
+
+// Read 8 Y, 8 U and 8 V interleaved from 410
+#define READYUVP410 \
+ "ldr q0, [%[src_y]], #16 \n" \
+ "ldp q4, q5, [%[src_uv]], #32 \n" \
+ "prfm pldl1keep, [%[src_y], 448] \n" \
+ "tbl v1.16b, {v4.16b, v5.16b}, v2.16b \n"
+
// Read 8 Y, 8 U and 8 V from 444
#define READYUV444 \
"ldr d0, [%[src_y]], #8 \n" \
- "ld1 {v1.d}[0], [%[src_u]], #8 \n" \
+ "ldr d1, [%[src_u]], #8 \n" \
+ "ldr d2, [%[src_v]], #8 \n" \
"prfm pldl1keep, [%[src_y], 448] \n" \
- "ld1 {v1.d}[1], [%[src_v]], #8 \n" \
"prfm pldl1keep, [%[src_u], 448] \n" \
"zip1 v0.16b, v0.16b, v0.16b \n" \
"prfm pldl1keep, [%[src_v], 448] \n"
-// Read 8 Y, and set 4 U and 4 V to 128
+// Read 8 Y
#define READYUV400 \
"ldr d0, [%[src_y]], #8 \n" \
- "movi v1.16b, #128 \n" \
"prfm pldl1keep, [%[src_y], 448] \n" \
"zip1 v0.16b, v0.16b, v0.16b \n"
static const uvec8 kNV12Table = {0, 0, 2, 2, 4, 4, 6, 6,
1, 1, 3, 3, 5, 5, 7, 7};
+static const uvec8 kNV12InterleavedTable = {0, 0, 4, 4, 8, 8, 12, 12,
+ 2, 2, 6, 6, 10, 10, 14, 14};
static const uvec8 kNV21Table = {1, 1, 3, 3, 5, 5, 7, 7,
0, 0, 2, 2, 4, 4, 6, 6};
+static const uvec8 kNV21InterleavedTable = {1, 1, 5, 5, 9, 9, 13, 13,
+ 3, 3, 7, 7, 11, 11, 15, 15};
// Read 8 Y and 4 UV from NV12 or NV21
#define READNV12 \
@@ -68,17 +129,17 @@
"prfm pldl1keep, [%[src_uv], 448] \n"
// Read 8 YUY2
-#define READYUY2 \
- "ld2 {v0.8b, v1.8b}, [%[src_yuy2]], #16 \n" \
- "zip1 v0.16b, v0.16b, v0.16b \n" \
- "prfm pldl1keep, [%[src_yuy2], 448] \n" \
- "tbl v1.16b, {v1.16b}, v2.16b \n"
+#define READYUY2 \
+ "ld1 {v3.16b}, [%[src_yuy2]], #16 \n" \
+ "trn1 v0.16b, v3.16b, v3.16b \n" \
+ "prfm pldl1keep, [%[src_yuy2], 448] \n" \
+ "tbl v1.16b, {v3.16b}, v2.16b \n"
// Read 8 UYVY
-#define READUYVY \
- "ld2 {v3.8b, v4.8b}, [%[src_uyvy]], #16 \n" \
- "zip1 v0.16b, v4.16b, v4.16b \n" \
- "prfm pldl1keep, [%[src_uyvy], 448] \n" \
+#define READUYVY \
+ "ld1 {v3.16b}, [%[src_uyvy]], #16 \n" \
+ "trn2 v0.16b, v3.16b, v3.16b \n" \
+ "prfm pldl1keep, [%[src_uyvy], 448] \n" \
"tbl v1.16b, {v3.16b}, v2.16b \n"
// UB VR UG VG
@@ -91,14 +152,14 @@
// v17.8h: G
// v18.8h: R
-// Convert from YUV to 2.14 fixed point RGB
-#define YUVTORGB \
+// Convert from YUV (NV12 or NV21) to 2.14 fixed point RGB.
+// Similar to I4XXTORGB but U/V components are in the low/high halves of v1.
+#define NVTORGB \
"umull2 v3.4s, v0.8h, v24.8h \n" \
"umull v6.8h, v1.8b, v30.8b \n" \
"umull v0.4s, v0.4h, v24.4h \n" \
"umlal2 v6.8h, v1.16b, v31.16b \n" /* DG */ \
- "uqshrn v0.4h, v0.4s, #16 \n" \
- "uqshrn2 v0.8h, v3.4s, #16 \n" /* Y */ \
+ "uzp2 v0.8h, v0.8h, v3.8h \n" /* Y */ \
"umull v4.8h, v1.8b, v28.8b \n" /* DB */ \
"umull2 v5.8h, v1.16b, v29.16b \n" /* DR */ \
"add v17.8h, v0.8h, v26.8h \n" /* G */ \
@@ -108,15 +169,69 @@
"uqsub v16.8h, v16.8h, v25.8h \n" /* B */ \
"uqsub v18.8h, v18.8h, v27.8h \n" /* R */
+// Convert from YUV (I444 or I420) to 2.14 fixed point RGB.
+// Similar to NVTORGB but U/V components are in v1/v2.
+#define I4XXTORGB \
+ "umull2 v3.4s, v0.8h, v24.8h \n" \
+ "umull v6.8h, v1.8b, v30.8b \n" \
+ "umull v0.4s, v0.4h, v24.4h \n" \
+ "umlal v6.8h, v2.8b, v31.8b \n" /* DG */ \
+ "uzp2 v0.8h, v0.8h, v3.8h \n" /* Y */ \
+ "umull v4.8h, v1.8b, v28.8b \n" /* DB */ \
+ "umull v5.8h, v2.8b, v29.8b \n" /* DR */ \
+ "add v17.8h, v0.8h, v26.8h \n" /* G */ \
+ "add v16.8h, v0.8h, v4.8h \n" /* B */ \
+ "add v18.8h, v0.8h, v5.8h \n" /* R */ \
+ "uqsub v17.8h, v17.8h, v6.8h \n" /* G */ \
+ "uqsub v16.8h, v16.8h, v25.8h \n" /* B */ \
+ "uqsub v18.8h, v18.8h, v27.8h \n" /* R */
+
+// Convert from YUV I400 to 2.14 fixed point RGB
+#define I400TORGB \
+ "umull2 v3.4s, v0.8h, v24.8h \n" \
+ "umull v0.4s, v0.4h, v24.4h \n" \
+ "uzp2 v0.8h, v0.8h, v3.8h \n" /* Y */ \
+ "add v17.8h, v0.8h, v26.8h \n" /* G */ \
+ "add v16.8h, v0.8h, v4.8h \n" /* B */ \
+ "add v18.8h, v0.8h, v5.8h \n" /* R */ \
+ "uqsub v17.8h, v17.8h, v6.8h \n" /* G */ \
+ "uqsub v16.8h, v16.8h, v25.8h \n" /* B */ \
+ "uqsub v18.8h, v18.8h, v27.8h \n" /* R */
+
// Convert from 2.14 fixed point RGB To 8 bit RGB
#define RGBTORGB8 \
"uqshrn v17.8b, v17.8h, #6 \n" \
"uqshrn v16.8b, v16.8h, #6 \n" \
"uqshrn v18.8b, v18.8h, #6 \n"
-#define YUVTORGB_REGS \
- "v0", "v1", "v3", "v4", "v5", "v6", "v7", "v16", "v17", "v18", "v24", "v25", \
- "v26", "v27", "v28", "v29", "v30", "v31"
+// Convert from 2.14 fixed point RGB to 8 bit RGB, placing the results in the
+// top half of each lane.
+#define RGBTORGB8_TOP \
+ "uqshl v17.8h, v17.8h, #2 \n" \
+ "uqshl v16.8h, v16.8h, #2 \n" \
+ "uqshl v18.8h, v18.8h, #2 \n"
+
+// Store 2.14 fixed point RGB as AR30 elements
+#define STOREAR30 \
+ /* Inputs: \
+ * v16.8h: xxbbbbbbbbbbxxxx \
+ * v17.8h: xxggggggggggxxxx \
+ * v18.8h: xxrrrrrrrrrrxxxx \
+ * v22.8h: 0011111111110000 (umin limit) \
+ * v23.8h: 1100000000000000 (alpha) \
+ */ \
+ "uqshl v0.8h, v16.8h, #2 \n" /* bbbbbbbbbbxxxxxx */ \
+ "uqshl v1.8h, v17.8h, #2 \n" /* ggggggggggxxxxxx */ \
+ "umin v6.8h, v18.8h, v22.8h \n" /* 00rrrrrrrrrrxxxx */ \
+ "shl v4.8h, v1.8h, #4 \n" /* ggggggxxxxxx0000 */ \
+ "orr v5.16b, v6.16b, v23.16b \n" /* 11rrrrrrrrrrxxxx */ \
+ "sri v4.8h, v0.8h, #6 \n" /* ggggggbbbbbbbbbb */ \
+ "sri v5.8h, v1.8h, #12 \n" /* 11rrrrrrrrrrgggg */ \
+ "st2 {v4.8h, v5.8h}, [%[dst_ar30]], #32 \n"
+
+#define YUVTORGB_REGS \
+ "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16", "v17", "v18", "v24", \
+ "v25", "v26", "v27", "v28", "v29", "v30", "v31"
void I444ToARGBRow_NEON(const uint8_t* src_y,
const uint8_t* src_u,
@@ -127,9 +242,9 @@
asm volatile(
YUVTORGB_SETUP
"movi v19.8b, #255 \n" /* A */
- "1: \n" READYUV444 YUVTORGB
- RGBTORGB8
- "subs %w[width], %w[width], #8 \n"
+ "1: \n" //
+ READYUV444
+ "subs %w[width], %w[width], #8 \n" I4XXTORGB RGBTORGB8
"st4 {v16.8b,v17.8b,v18.8b,v19.8b}, [%[dst_argb]], #32 \n"
"b.gt 1b \n"
: [src_y] "+r"(src_y), // %[src_y]
@@ -150,9 +265,9 @@
int width) {
asm volatile(
YUVTORGB_SETUP
- "1: \n" READYUV444 YUVTORGB
- RGBTORGB8
- "subs %w[width], %w[width], #8 \n"
+ "1: \n" //
+ READYUV444
+ "subs %w[width], %w[width], #8 \n" I4XXTORGB RGBTORGB8
"st3 {v16.8b,v17.8b,v18.8b}, [%[dst_rgb24]], #24 \n"
"b.gt 1b \n"
: [src_y] "+r"(src_y), // %[src_y]
@@ -165,18 +280,103 @@
: "cc", "memory", YUVTORGB_REGS);
}
-void I422ToARGBRow_NEON(const uint8_t* src_y,
- const uint8_t* src_u,
- const uint8_t* src_v,
+void I210ToAR30Row_NEON(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ const uvec8* uv_coeff = &yuvconstants->kUVCoeff;
+ const vec16* rgb_coeff = &yuvconstants->kRGBCoeffBias;
+ uint16_t limit = 0x3ff0;
+ uint16_t alpha = 0xc000;
+ asm volatile(YUVTORGB_SETUP
+ "dup v22.8h, %w[limit] \n"
+ "dup v23.8h, %w[alpha] \n"
+ "1: \n" //
+ READYUV210
+ "subs %w[width], %w[width], #8 \n" NVTORGB STOREAR30
+ "b.gt 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_ar30] "+r"(dst_ar30), // %[dst_ar30]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(uv_coeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(rgb_coeff), // %[kRGBCoeffBias]
+ [limit] "r"(limit), // %[limit]
+ [alpha] "r"(alpha) // %[alpha]
+ : "cc", "memory", YUVTORGB_REGS, "v22", "v23");
+}
+
+void I410ToAR30Row_NEON(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ const uvec8* uv_coeff = &yuvconstants->kUVCoeff;
+ const vec16* rgb_coeff = &yuvconstants->kRGBCoeffBias;
+ uint16_t limit = 0x3ff0;
+ uint16_t alpha = 0xc000;
+ asm volatile(YUVTORGB_SETUP
+ "dup v22.8h, %w[limit] \n"
+ "dup v23.8h, %w[alpha] \n"
+ "1: \n" //
+ READYUV410
+ "subs %w[width], %w[width], #8 \n" NVTORGB STOREAR30
+ "b.gt 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_ar30] "+r"(dst_ar30), // %[dst_ar30]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(uv_coeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(rgb_coeff), // %[kRGBCoeffBias]
+ [limit] "r"(limit), // %[limit]
+ [alpha] "r"(alpha) // %[alpha]
+ : "cc", "memory", YUVTORGB_REGS, "v22", "v23");
+}
+
+void I212ToAR30Row_NEON(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ const uvec8* uv_coeff = &yuvconstants->kUVCoeff;
+ const vec16* rgb_coeff = &yuvconstants->kRGBCoeffBias;
+ const uint16_t limit = 0x3ff0;
+ asm volatile(YUVTORGB_SETUP
+ "dup v22.8h, %w[limit] \n"
+ "movi v23.8h, #0xc0, lsl #8 \n" // A
+ "1: \n" //
+ READYUV212
+ "subs %w[width], %w[width], #8 \n" NVTORGB STOREAR30
+ "b.gt 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_ar30] "+r"(dst_ar30), // %[dst_ar30]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(uv_coeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(rgb_coeff), // %[kRGBCoeffBias]
+ [limit] "r"(limit) // %[limit]
+ : "cc", "memory", YUVTORGB_REGS, "v22", "v23");
+}
+
+void I210ToARGBRow_NEON(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
int width) {
asm volatile(
YUVTORGB_SETUP
- "movi v19.8b, #255 \n" /* A */
- "1: \n" READYUV422 YUVTORGB
- RGBTORGB8
- "subs %w[width], %w[width], #8 \n"
+ "movi v19.8b, #255 \n"
+ "1: \n" //
+ READYUV210
+ "subs %w[width], %w[width], #8 \n" NVTORGB RGBTORGB8
"st4 {v16.8b,v17.8b,v18.8b,v19.8b}, [%[dst_argb]], #32 \n"
"b.gt 1b \n"
: [src_y] "+r"(src_y), // %[src_y]
@@ -189,6 +389,220 @@
: "cc", "memory", YUVTORGB_REGS, "v19");
}
+void I410ToARGBRow_NEON(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ asm volatile(
+ YUVTORGB_SETUP
+ "movi v19.8b, #255 \n"
+ "1: \n" //
+ READYUV410
+ "subs %w[width], %w[width], #8 \n" NVTORGB RGBTORGB8
+ "st4 {v16.8b,v17.8b,v18.8b,v19.8b}, [%[dst_argb]], #32 \n"
+ "b.gt 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_REGS, "v19");
+}
+
+void I212ToARGBRow_NEON(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ const uvec8* uv_coeff = &yuvconstants->kUVCoeff;
+ const vec16* rgb_coeff = &yuvconstants->kRGBCoeffBias;
+ asm volatile(
+ YUVTORGB_SETUP
+ "movi v19.8b, #255 \n"
+ "1: \n" //
+ READYUV212
+ "subs %w[width], %w[width], #8 \n" NVTORGB RGBTORGB8
+ "st4 {v16.8b,v17.8b,v18.8b,v19.8b}, [%[dst_argb]], #32 \n"
+ "b.gt 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(uv_coeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(rgb_coeff) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_REGS, "v19");
+}
+
+void I422ToARGBRow_NEON(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ asm volatile(
+ YUVTORGB_SETUP
+ "movi v19.8b, #255 \n" /* A */
+ "1: \n" //
+ READYUV422
+ "subs %w[width], %w[width], #8 \n" I4XXTORGB RGBTORGB8
+ "st4 {v16.8b,v17.8b,v18.8b,v19.8b}, [%[dst_argb]], #32 \n"
+ "b.gt 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_REGS, "v19");
+}
+
+uint8_t kP210LoadShuffleIndices[] = {1, 1, 5, 5, 9, 9, 13, 13,
+ 3, 3, 7, 7, 11, 11, 15, 15};
+
+void P210ToARGBRow_NEON(const uint16_t* src_y,
+ const uint16_t* src_uv,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ const uvec8* uv_coeff = &yuvconstants->kUVCoeff;
+ const vec16* rgb_coeff = &yuvconstants->kRGBCoeffBias;
+ asm volatile(
+ YUVTORGB_SETUP
+ "movi v19.8b, #255 \n"
+ "ldr q2, [%[kIndices]] \n"
+ "1: \n" //
+ READYUVP210
+ "subs %w[width], %w[width], #8 \n" NVTORGB RGBTORGB8
+ "st4 {v16.8b, v17.8b, v18.8b, v19.8b}, [%[dst_argb]], #32 \n"
+ "b.gt 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_uv] "+r"(src_uv), // %[src_uv]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(uv_coeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(rgb_coeff), // %[kRGBCoeffBias]
+ [kIndices] "r"(kP210LoadShuffleIndices) // %[kIndices]
+ : "cc", "memory", YUVTORGB_REGS, "v19");
+}
+
+uint8_t kP410LoadShuffleIndices[] = {1, 5, 9, 13, 17, 21, 25, 29,
+ 3, 7, 11, 15, 19, 23, 27, 31};
+
+void P410ToARGBRow_NEON(const uint16_t* src_y,
+ const uint16_t* src_uv,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ const uvec8* uv_coeff = &yuvconstants->kUVCoeff;
+ const vec16* rgb_coeff = &yuvconstants->kRGBCoeffBias;
+ asm volatile(
+ YUVTORGB_SETUP
+ "movi v19.8b, #255 \n"
+ "ldr q2, [%[kIndices]] \n"
+ "1: \n" //
+ READYUVP410
+ "subs %w[width], %w[width], #8 \n" NVTORGB RGBTORGB8
+ "st4 {v16.8b, v17.8b, v18.8b, v19.8b}, [%[dst_argb]], #32 \n"
+ "b.gt 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_uv] "+r"(src_uv), // %[src_uv]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(uv_coeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(rgb_coeff), // %[kRGBCoeffBias]
+ [kIndices] "r"(kP410LoadShuffleIndices) // %[kIndices]
+ : "cc", "memory", YUVTORGB_REGS, "v19");
+}
+
+void P210ToAR30Row_NEON(const uint16_t* src_y,
+ const uint16_t* src_uv,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ const uvec8* uv_coeff = &yuvconstants->kUVCoeff;
+ const vec16* rgb_coeff = &yuvconstants->kRGBCoeffBias;
+ const uint16_t limit = 0x3ff0;
+ asm volatile(YUVTORGB_SETUP
+ "dup v22.8h, %w[limit] \n"
+ "movi v23.8h, #0xc0, lsl #8 \n" // A
+ "ldr q2, [%[kIndices]] \n"
+ "1: \n" //
+ READYUVP210
+ "subs %w[width], %w[width], #8 \n" NVTORGB STOREAR30
+ "b.gt 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_uv] "+r"(src_uv), // %[src_uv]
+ [dst_ar30] "+r"(dst_ar30), // %[dst_ar30]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(uv_coeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(rgb_coeff), // %[kRGBCoeffBias]
+ [limit] "r"(limit), // %[limit]
+ [kIndices] "r"(kP210LoadShuffleIndices) // %[kIndices]
+ : "cc", "memory", YUVTORGB_REGS, "v22", "v23");
+}
+
+void P410ToAR30Row_NEON(const uint16_t* src_y,
+ const uint16_t* src_uv,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ const uvec8* uv_coeff = &yuvconstants->kUVCoeff;
+ const vec16* rgb_coeff = &yuvconstants->kRGBCoeffBias;
+ uint16_t limit = 0x3ff0;
+ asm volatile(YUVTORGB_SETUP
+ "dup v22.8h, %w[limit] \n"
+ "movi v23.8h, #0xc0, lsl #8 \n" // A
+ "ldr q2, [%[kIndices]] \n"
+ "1: \n" //
+ READYUVP410
+ "subs %w[width], %w[width], #8 \n" NVTORGB STOREAR30
+ "b.gt 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_uv] "+r"(src_uv), // %[src_uv]
+ [dst_ar30] "+r"(dst_ar30), // %[dst_ar30]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(uv_coeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(rgb_coeff), // %[kRGBCoeffBias]
+ [limit] "r"(limit), // %[limit]
+ [kIndices] "r"(kP410LoadShuffleIndices) // %[kIndices]
+ : "cc", "memory", YUVTORGB_REGS, "v22", "v23");
+}
+
+void I422ToAR30Row_NEON(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ const uvec8* uv_coeff = &yuvconstants->kUVCoeff;
+ const vec16* rgb_coeff = &yuvconstants->kRGBCoeffBias;
+ const uint16_t limit = 0x3ff0;
+ asm volatile(
+ YUVTORGB_SETUP
+ "dup v22.8h, %w[limit] \n"
+ "movi v23.8h, #0xc0, lsl #8 \n" // A
+ "1: \n" //
+ READYUV422
+ "subs %w[width], %w[width], #8 \n" I4XXTORGB STOREAR30
+ "b.gt 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_ar30] "+r"(dst_ar30), // %[dst_ar30]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(uv_coeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(rgb_coeff), // %[kRGBCoeffBias]
+ [limit] "r"(limit) // %[limit]
+ : "cc", "memory", YUVTORGB_REGS, "v22", "v23");
+}
+
void I444AlphaToARGBRow_NEON(const uint8_t* src_y,
const uint8_t* src_u,
const uint8_t* src_v,
@@ -198,10 +612,62 @@
int width) {
asm volatile(
YUVTORGB_SETUP
- "1: \n"
+ "1: \n"
"ld1 {v19.8b}, [%[src_a]], #8 \n" READYUV444
- "prfm pldl1keep, [%[src_a], 448] \n" YUVTORGB RGBTORGB8
"subs %w[width], %w[width], #8 \n"
+ "prfm pldl1keep, [%[src_a], 448] \n" I4XXTORGB RGBTORGB8
+ "st4 {v16.8b,v17.8b,v18.8b,v19.8b}, [%[dst_argb]], #32 \n"
+ "b.gt 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [src_a] "+r"(src_a), // %[src_a]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_REGS, "v19");
+}
+
+void I410AlphaToARGBRow_NEON(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ const uint16_t* src_a,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ asm volatile(
+ YUVTORGB_SETUP
+ "1: \n"
+ "ld1 {v19.16b}, [%[src_a]], #16 \n" READYUV410
+ "subs %w[width], %w[width], #8 \n"
+ "uqshrn v19.8b, v19.8h, #2 \n" NVTORGB RGBTORGB8
+ "st4 {v16.8b, v17.8b, v18.8b, v19.8b}, [%[dst_argb]], #32 \n"
+ "b.gt 1b \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [src_a] "+r"(src_a), // %[src_a]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width) // %[width]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_REGS, "v19");
+}
+
+void I210AlphaToARGBRow_NEON(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ const uint16_t* src_a,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ asm volatile(
+ YUVTORGB_SETUP
+ "1: \n"
+ "ld1 {v19.16b}, [%[src_a]], #16 \n" READYUV210
+ "subs %w[width], %w[width], #8 \n"
+ "uqshrn v19.8b, v19.8h, #2 \n" NVTORGB RGBTORGB8
"st4 {v16.8b,v17.8b,v18.8b,v19.8b}, [%[dst_argb]], #32 \n"
"b.gt 1b \n"
: [src_y] "+r"(src_y), // %[src_y]
@@ -224,10 +690,10 @@
int width) {
asm volatile(
YUVTORGB_SETUP
- "1: \n"
+ "1: \n"
"ld1 {v19.8b}, [%[src_a]], #8 \n" READYUV422
- "prfm pldl1keep, [%[src_a], 448] \n" YUVTORGB RGBTORGB8
"subs %w[width], %w[width], #8 \n"
+ "prfm pldl1keep, [%[src_a], 448] \n" I4XXTORGB RGBTORGB8
"st4 {v16.8b,v17.8b,v18.8b,v19.8b}, [%[dst_argb]], #32 \n"
"b.gt 1b \n"
: [src_y] "+r"(src_y), // %[src_y]
@@ -250,9 +716,9 @@
asm volatile(
YUVTORGB_SETUP
"movi v15.8b, #255 \n" /* A */
- "1: \n" READYUV422 YUVTORGB
- RGBTORGB8
- "subs %w[width], %w[width], #8 \n"
+ "1: \n" //
+ READYUV422
+ "subs %w[width], %w[width], #8 \n" I4XXTORGB RGBTORGB8
"st4 {v15.8b,v16.8b,v17.8b,v18.8b}, [%[dst_rgba]], #32 \n"
"b.gt 1b \n"
: [src_y] "+r"(src_y), // %[src_y]
@@ -273,9 +739,9 @@
int width) {
asm volatile(
YUVTORGB_SETUP
- "1: \n" READYUV422 YUVTORGB
- RGBTORGB8
- "subs %w[width], %w[width], #8 \n"
+ "1: \n" //
+ READYUV422
+ "subs %w[width], %w[width], #8 \n" I4XXTORGB RGBTORGB8
"st3 {v16.8b,v17.8b,v18.8b}, [%[dst_rgb24]], #24 \n"
"b.gt 1b \n"
: [src_y] "+r"(src_y), // %[src_y]
@@ -288,12 +754,24 @@
: "cc", "memory", YUVTORGB_REGS);
}
-#define ARGBTORGB565 \
- "shll v18.8h, v18.8b, #8 \n" /* R */ \
- "shll v17.8h, v17.8b, #8 \n" /* G */ \
- "shll v16.8h, v16.8b, #8 \n" /* B */ \
- "sri v18.8h, v17.8h, #5 \n" /* RG */ \
- "sri v18.8h, v16.8h, #11 \n" /* RGB */
+#define ARGBTORGB565 \
+ /* Inputs: \
+ * v16: bbbbbxxx \
+ * v17: ggggggxx \
+ * v18: rrrrrxxx */ \
+ "shll v18.8h, v18.8b, #8 \n" /* rrrrrrxx00000000 */ \
+ "shll v17.8h, v17.8b, #8 \n" /* gggggxxx00000000 */ \
+ "shll v16.8h, v16.8b, #8 \n" /* bbbbbbxx00000000 */ \
+ "sri v18.8h, v17.8h, #5 \n" /* rrrrrgggggg00000 */ \
+ "sri v18.8h, v16.8h, #11 \n" /* rrrrrggggggbbbbb */
+
+#define ARGBTORGB565_FROM_TOP \
+ /* Inputs: \
+ * v16: bbbbbxxxxxxxxxxx \
+ * v17: ggggggxxxxxxxxxx \
+ * v18: rrrrrxxxxxxxxxxx */ \
+ "sri v18.8h, v17.8h, #5 \n" /* rrrrrgggggg00000 */ \
+ "sri v18.8h, v16.8h, #11 \n" /* rrrrrggggggbbbbb */
void I422ToRGB565Row_NEON(const uint8_t* src_y,
const uint8_t* src_u,
@@ -303,8 +781,10 @@
int width) {
asm volatile(
YUVTORGB_SETUP
- "1: \n" READYUV422 YUVTORGB
- RGBTORGB8 "subs %w[width], %w[width], #8 \n" ARGBTORGB565
+ "1: \n" //
+ READYUV422
+ "subs %w[width], %w[width], #8 \n" I4XXTORGB RGBTORGB8_TOP
+ ARGBTORGB565_FROM_TOP
"st1 {v18.8h}, [%[dst_rgb565]], #16 \n" // store 8 pixels RGB565.
"b.gt 1b \n"
: [src_y] "+r"(src_y), // %[src_y]
@@ -317,14 +797,22 @@
: "cc", "memory", YUVTORGB_REGS);
}
-#define ARGBTOARGB1555 \
- "shll v0.8h, v19.8b, #8 \n" /* A */ \
- "shll v18.8h, v18.8b, #8 \n" /* R */ \
- "shll v17.8h, v17.8b, #8 \n" /* G */ \
- "shll v16.8h, v16.8b, #8 \n" /* B */ \
- "sri v0.8h, v18.8h, #1 \n" /* AR */ \
- "sri v0.8h, v17.8h, #6 \n" /* ARG */ \
- "sri v0.8h, v16.8h, #11 \n" /* ARGB */
+#define ARGBTOARGB1555 \
+ /* Inputs: \
+ * v16: gggggxxxbbbbbxxx v17: axxxxxxxrrrrrxxx */ \
+ "shl v1.8h, v16.8h, #8 \n" /* bbbbbxxx00000000 */ \
+ "shl v2.8h, v17.8h, #8 \n" /* rrrrrxxx00000000 */ \
+ "sri v17.8h, v2.8h, #1 \n" /* arrrrrxxxrrrrxxx */ \
+ "sri v17.8h, v16.8h, #6 \n" /* arrrrrgggggxxxbb */ \
+ "sri v17.8h, v1.8h, #11 \n" /* arrrrrgggggbbbbb */
+
+#define ARGBTOARGB1555_FROM_TOP \
+ /* Inputs: \
+ * v16: bbbbbxxxxxxxxxxx v17: gggggxxxxxxxxxxx \
+ * v18: rrrrrxxxxxxxxxxx v19: axxxxxxxxxxxxxxx */ \
+ "sri v19.8h, v18.8h, #1 \n" /* arrrrrxxxxxxxxxx */ \
+ "sri v19.8h, v17.8h, #6 \n" /* arrrrrgggggxxxxx */ \
+ "sri v19.8h, v16.8h, #11 \n" /* arrrrrgggggbbbbb */
void I422ToARGB1555Row_NEON(const uint8_t* src_y,
const uint8_t* src_u,
@@ -334,12 +822,12 @@
int width) {
asm volatile(
YUVTORGB_SETUP
- "movi v19.8b, #255 \n"
- "1: \n" READYUV422 YUVTORGB
- RGBTORGB8
- "subs %w[width], %w[width], #8 \n" ARGBTOARGB1555
- "st1 {v0.8h}, [%[dst_argb1555]], #16 \n" // store 8 pixels
- // RGB565.
+ "movi v19.8h, #0x80, lsl #8 \n"
+ "1: \n" //
+ READYUV422 "subs %w[width], %w[width], #8 \n" //
+ I4XXTORGB RGBTORGB8_TOP ARGBTOARGB1555_FROM_TOP
+ "st1 {v19.8h}, [%[dst_argb1555]], #16 \n" // store 8 pixels
+ // RGB1555.
"b.gt 1b \n"
: [src_y] "+r"(src_y), // %[src_y]
[src_u] "+r"(src_u), // %[src_u]
@@ -351,15 +839,11 @@
: "cc", "memory", YUVTORGB_REGS, "v19");
}
-#define ARGBTOARGB4444 \
- /* Input v16.8b<=B, v17.8b<=G, v18.8b<=R, v19.8b<=A, v23.8b<=0x0f */ \
- "ushr v16.8b, v16.8b, #4 \n" /* B */ \
- "bic v17.8b, v17.8b, v23.8b \n" /* G */ \
- "ushr v18.8b, v18.8b, #4 \n" /* R */ \
- "bic v19.8b, v19.8b, v23.8b \n" /* A */ \
- "orr v0.8b, v16.8b, v17.8b \n" /* BG */ \
- "orr v1.8b, v18.8b, v19.8b \n" /* RA */ \
- "zip1 v0.16b, v0.16b, v1.16b \n" /* BGRA */
+#define ARGBTOARGB4444 \
+ /* Input v16.8b<=B, v17.8b<=G, v18.8b<=R, v19.8b<=A */ \
+ "sri v17.8b, v16.8b, #4 \n" /* BG */ \
+ "sri v19.8b, v18.8b, #4 \n" /* RA */ \
+ "zip1 v0.16b, v17.16b, v19.16b \n" /* BGRA */
void I422ToARGB4444Row_NEON(const uint8_t* src_y,
const uint8_t* src_u,
@@ -369,11 +853,9 @@
int width) {
asm volatile(
YUVTORGB_SETUP
- "movi v23.16b, #0x0f \n" // bits to clear with
- // vbic.
- "1: \n" READYUV422 YUVTORGB
- RGBTORGB8
- "subs %w[width], %w[width], #8 \n"
+ "1: \n" //
+ READYUV422
+ "subs %w[width], %w[width], #8 \n" I4XXTORGB RGBTORGB8
"movi v19.8b, #255 \n" ARGBTOARGB4444
"st1 {v0.8h}, [%[dst_argb4444]], #16 \n" // store 8
// pixels
@@ -386,7 +868,7 @@
[width] "+r"(width) // %[width]
: [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
- : "cc", "memory", YUVTORGB_REGS, "v19", "v23");
+ : "cc", "memory", YUVTORGB_REGS, "v19");
}
void I400ToARGBRow_NEON(const uint8_t* src_y,
@@ -395,10 +877,15 @@
int width) {
asm volatile(
YUVTORGB_SETUP
+ "movi v1.16b, #128 \n"
"movi v19.8b, #255 \n"
- "1: \n" READYUV400 YUVTORGB
- RGBTORGB8
- "subs %w[width], %w[width], #8 \n"
+ "umull v6.8h, v1.8b, v30.8b \n"
+ "umlal2 v6.8h, v1.16b, v31.16b \n" /* DG */
+ "umull v4.8h, v1.8b, v28.8b \n" /* DB */
+ "umull2 v5.8h, v1.16b, v29.16b \n" /* DR */
+ "1: \n" //
+ READYUV400 I400TORGB
+ "subs %w[width], %w[width], #8 \n" RGBTORGB8
"st4 {v16.8b,v17.8b,v18.8b,v19.8b}, [%[dst_argb]], #32 \n"
"b.gt 1b \n"
: [src_y] "+r"(src_y), // %[src_y]
@@ -409,16 +896,16 @@
: "cc", "memory", YUVTORGB_REGS, "v19");
}
-#if LIBYUV_USE_ST4
+#if defined(LIBYUV_USE_ST4)
void J400ToARGBRow_NEON(const uint8_t* src_y, uint8_t* dst_argb, int width) {
asm volatile(
"movi v23.8b, #255 \n"
- "1: \n"
+ "1: \n"
"ld1 {v20.8b}, [%0], #8 \n"
- "prfm pldl1keep, [%0, 448] \n"
- "orr v21.8b, v20.8b, v20.8b \n"
- "orr v22.8b, v20.8b, v20.8b \n"
"subs %w2, %w2, #8 \n"
+ "prfm pldl1keep, [%0, 448] \n"
+ "mov v21.8b, v20.8b \n"
+ "mov v22.8b, v20.8b \n"
"st4 {v20.8b,v21.8b,v22.8b,v23.8b}, [%1], #32 \n"
"b.gt 1b \n"
: "+r"(src_y), // %0
@@ -431,7 +918,7 @@
void J400ToARGBRow_NEON(const uint8_t* src_y, uint8_t* dst_argb, int width) {
asm volatile(
"movi v20.8b, #255 \n"
- "1: \n"
+ "1: \n"
"ldr d16, [%0], #8 \n"
"subs %w2, %w2, #8 \n"
"zip1 v18.16b, v16.16b, v16.16b \n" // YY
@@ -458,8 +945,8 @@
YUVTORGB_SETUP
"movi v19.8b, #255 \n"
"ldr q2, [%[kNV12Table]] \n"
- "1: \n" READNV12 YUVTORGB RGBTORGB8
- "subs %w[width], %w[width], #8 \n"
+ "1: \n" //
+ READNV12 "subs %w[width], %w[width], #8 \n" NVTORGB RGBTORGB8
"st4 {v16.8b,v17.8b,v18.8b,v19.8b}, [%[dst_argb]], #32 \n"
"b.gt 1b \n"
: [src_y] "+r"(src_y), // %[src_y]
@@ -481,8 +968,8 @@
YUVTORGB_SETUP
"movi v19.8b, #255 \n"
"ldr q2, [%[kNV12Table]] \n"
- "1: \n" READNV12 YUVTORGB RGBTORGB8
- "subs %w[width], %w[width], #8 \n"
+ "1: \n" //
+ READNV12 "subs %w[width], %w[width], #8 \n" NVTORGB RGBTORGB8
"st4 {v16.8b,v17.8b,v18.8b,v19.8b}, [%[dst_argb]], #32 \n"
"b.gt 1b \n"
: [src_y] "+r"(src_y), // %[src_y]
@@ -503,8 +990,8 @@
asm volatile(
YUVTORGB_SETUP
"ldr q2, [%[kNV12Table]] \n"
- "1: \n" READNV12 YUVTORGB RGBTORGB8
- "subs %w[width], %w[width], #8 \n"
+ "1: \n" //
+ READNV12 "subs %w[width], %w[width], #8 \n" NVTORGB RGBTORGB8
"st3 {v16.8b,v17.8b,v18.8b}, [%[dst_rgb24]], #24 \n"
"b.gt 1b \n"
: [src_y] "+r"(src_y), // %[src_y]
@@ -525,8 +1012,8 @@
asm volatile(
YUVTORGB_SETUP
"ldr q2, [%[kNV12Table]] \n"
- "1: \n" READNV12 YUVTORGB RGBTORGB8
- "subs %w[width], %w[width], #8 \n"
+ "1: \n" //
+ READNV12 "subs %w[width], %w[width], #8 \n" NVTORGB RGBTORGB8
"st3 {v16.8b,v17.8b,v18.8b}, [%[dst_rgb24]], #24 \n"
"b.gt 1b \n"
: [src_y] "+r"(src_y), // %[src_y]
@@ -547,8 +1034,10 @@
asm volatile(
YUVTORGB_SETUP
"ldr q2, [%[kNV12Table]] \n"
- "1: \n" READNV12 YUVTORGB RGBTORGB8
- "subs %w[width], %w[width], #8 \n" ARGBTORGB565
+ "1: \n" //
+ READNV12
+ "subs %w[width], %w[width], #8 \n" NVTORGB RGBTORGB8_TOP
+ ARGBTORGB565_FROM_TOP
"st1 {v18.8h}, [%[dst_rgb565]], #16 \n" // store 8
// pixels
// RGB565.
@@ -570,9 +1059,9 @@
asm volatile(
YUVTORGB_SETUP
"movi v19.8b, #255 \n"
- "ldr q2, [%[kNV12Table]] \n"
- "1: \n" READYUY2 YUVTORGB RGBTORGB8
- "subs %w[width], %w[width], #8 \n"
+ "ldr q2, [%[kNV21InterleavedTable]] \n"
+ "1: \n" //
+ READYUY2 "subs %w[width], %w[width], #8 \n" NVTORGB RGBTORGB8
"st4 {v16.8b,v17.8b,v18.8b,v19.8b}, [%[dst_argb]], #32 \n"
"b.gt 1b \n"
: [src_yuy2] "+r"(src_yuy2), // %[src_yuy2]
@@ -580,7 +1069,7 @@
[width] "+r"(width) // %[width]
: [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
- [kNV12Table] "r"(&kNV12Table)
+ [kNV21InterleavedTable] "r"(&kNV21InterleavedTable)
: "cc", "memory", YUVTORGB_REGS, "v2", "v19");
}
@@ -591,9 +1080,9 @@
asm volatile(
YUVTORGB_SETUP
"movi v19.8b, #255 \n"
- "ldr q2, [%[kNV12Table]] \n"
- "1: \n" READUYVY YUVTORGB RGBTORGB8
- "subs %w[width], %w[width], #8 \n"
+ "ldr q2, [%[kNV12InterleavedTable]] \n"
+ "1: \n" //
+ READUYVY "subs %w[width], %w[width], #8 \n" NVTORGB RGBTORGB8
"st4 {v16.8b,v17.8b,v18.8b,v19.8b}, [%[dst_argb]], #32 \n"
"b.gt 1b \n"
: [src_uyvy] "+r"(src_uyvy), // %[src_yuy2]
@@ -601,7 +1090,7 @@
[width] "+r"(width) // %[width]
: [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
[kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias), // %[kRGBCoeffBias]
- [kNV12Table] "r"(&kNV12Table)
+ [kNV12InterleavedTable] "r"(&kNV12InterleavedTable)
: "cc", "memory", YUVTORGB_REGS, "v2", "v19");
}
@@ -611,7 +1100,7 @@
uint8_t* dst_v,
int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"ld2 {v0.16b,v1.16b}, [%0], #32 \n" // load 16 pairs of UV
"subs %w3, %w3, #16 \n" // 16 processed per loop
"prfm pldl1keep, [%0, 448] \n"
@@ -636,7 +1125,7 @@
uint8_t* dst,
int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"ld1 {v0.16b}, [%0], %3 \n" // load 16 bytes
"subs %w2, %w2, #16 \n" // 16 processed per loop
"prfm pldl1keep, [%0, 1792] \n" // 7 tiles of 256b ahead
@@ -656,7 +1145,7 @@
uint16_t* dst,
int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"ld1 {v0.8h,v1.8h}, [%0], %3 \n" // load 16 pixels
"subs %w2, %w2, #16 \n" // 16 processed per loop
"prfm pldl1keep, [%0, 3584] \n" // 7 tiles of 512b ahead
@@ -677,7 +1166,7 @@
uint8_t* dst_v,
int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"ld2 {v0.8b,v1.8b}, [%0], %4 \n"
"subs %w3, %w3, #16 \n"
"prfm pldl1keep, [%0, 1792] \n"
@@ -693,7 +1182,7 @@
);
}
-#if LIBYUV_USE_ST2
+#if defined(LIBYUV_USE_ST2)
// Read 16 Y, 8 UV, and write 8 YUY2
void DetileToYUY2_NEON(const uint8_t* src_y,
ptrdiff_t src_y_tile_stride,
@@ -702,12 +1191,12 @@
uint8_t* dst_yuy2,
int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"ld1 {v0.16b}, [%0], %4 \n" // load 16 Ys
+ "subs %w3, %w3, #16 \n" // store 8 YUY2
"prfm pldl1keep, [%0, 1792] \n"
"ld1 {v1.16b}, [%1], %5 \n" // load 8 UVs
"prfm pldl1keep, [%1, 1792] \n"
- "subs %w3, %w3, #16 \n" // store 8 YUY2
"st2 {v0.16b,v1.16b}, [%2], #32 \n"
"b.gt 1b \n"
: "+r"(src_y), // %0
@@ -728,7 +1217,7 @@
uint8_t* dst_yuy2,
int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"ld1 {v0.16b}, [%0], %4 \n" // load 16 Ys
"ld1 {v1.16b}, [%1], %5 \n" // load 8 UVs
"subs %w3, %w3, #16 \n"
@@ -753,13 +1242,13 @@
// tinyurl.com/mtk-10bit-video-format for format documentation.
void UnpackMT2T_NEON(const uint8_t* src, uint16_t* dst, size_t size) {
asm volatile(
- "1: \n"
+ "1: \n"
"ld1 {v7.16b}, [%0], #16 \n"
"ld1 {v0.16b-v3.16b}, [%0], #64 \n"
+ "subs %2, %2, #80 \n"
"shl v4.16b, v7.16b, #6 \n"
"shl v5.16b, v7.16b, #4 \n"
"shl v6.16b, v7.16b, #2 \n"
- "subs %2, %2, #80 \n"
"zip1 v16.16b, v4.16b, v0.16b \n"
"zip1 v18.16b, v5.16b, v1.16b \n"
"zip1 v20.16b, v6.16b, v2.16b \n"
@@ -787,14 +1276,14 @@
"v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23");
}
-#if LIBYUV_USE_ST2
+#if defined(LIBYUV_USE_ST2)
// Reads 16 U's and V's and writes out 16 pairs of UV.
void MergeUVRow_NEON(const uint8_t* src_u,
const uint8_t* src_v,
uint8_t* dst_uv,
int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"ld1 {v0.16b}, [%0], #16 \n" // load U
"ld1 {v1.16b}, [%1], #16 \n" // load V
"subs %w3, %w3, #16 \n" // 16 processed per loop
@@ -819,10 +1308,10 @@
int shift = 16 - depth;
asm volatile(
"dup v2.8h, %w4 \n"
- "1: \n"
+ "1: \n"
"ld1 {v0.8h}, [%0], #16 \n" // load 8 U
- "subs %w3, %w3, #8 \n" // 8 src pixels per loop
"ld1 {v1.8h}, [%1], #16 \n" // load 8 V
+ "subs %w3, %w3, #8 \n" // 8 src pixels per loop
"ushl v0.8h, v0.8h, v2.8h \n"
"prfm pldl1keep, [%0, 448] \n"
"ushl v1.8h, v1.8h, v2.8h \n"
@@ -843,7 +1332,7 @@
uint8_t* dst_uv,
int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"ld1 {v0.16b}, [%0], #16 \n" // load U
"ld1 {v1.16b}, [%1], #16 \n" // load V
"subs %w3, %w3, #16 \n" // 16 processed per loop
@@ -870,10 +1359,10 @@
int shift = 16 - depth;
asm volatile(
"dup v4.8h, %w4 \n"
- "1: \n"
+ "1: \n"
"ld1 {v0.8h}, [%0], #16 \n" // load 8 U
- "subs %w3, %w3, #8 \n" // 8 src pixels per loop
"ld1 {v1.8h}, [%1], #16 \n" // load 8 V
+ "subs %w3, %w3, #8 \n" // 8 src pixels per loop
"ushl v0.8h, v0.8h, v4.8h \n"
"ushl v1.8h, v1.8h, v4.8h \n"
"prfm pldl1keep, [%0, 448] \n"
@@ -898,7 +1387,7 @@
uint8_t* dst_b,
int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"ld3 {v0.16b,v1.16b,v2.16b}, [%0], #48 \n" // load 16 RGB
"subs %w4, %w4, #16 \n" // 16 processed per loop
"prfm pldl1keep, [%0, 448] \n"
@@ -923,7 +1412,7 @@
uint8_t* dst_rgb,
int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"ld1 {v0.16b}, [%0], #16 \n" // load R
"ld1 {v1.16b}, [%1], #16 \n" // load G
"ld1 {v2.16b}, [%2], #16 \n" // load B
@@ -951,7 +1440,7 @@
uint8_t* dst_a,
int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"ld4 {v0.16b,v1.16b,v2.16b,v3.16b}, [%0], #64 \n" // load 16 ARGB
"subs %w5, %w5, #16 \n" // 16 processed per loop
"prfm pldl1keep, [%0, 448] \n"
@@ -971,7 +1460,7 @@
);
}
-#if LIBYUV_USE_ST4
+#if defined(LIBYUV_USE_ST4)
// Reads 16 planar R's, G's, B's and A's and writes out 16 packed ARGB at a time
void MergeARGBRow_NEON(const uint8_t* src_r,
const uint8_t* src_g,
@@ -980,7 +1469,7 @@
uint8_t* dst_argb,
int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"ld1 {v0.16b}, [%2], #16 \n" // load B
"ld1 {v1.16b}, [%1], #16 \n" // load G
"ld1 {v2.16b}, [%0], #16 \n" // load R
@@ -1011,7 +1500,7 @@
uint8_t* dst_argb,
int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"ld1 {v0.16b}, [%2], #16 \n" // load B
"ld1 {v1.16b}, [%1], #16 \n" // load G
"ld1 {v2.16b}, [%0], #16 \n" // load R
@@ -1051,7 +1540,7 @@
uint8_t* dst_b,
int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"ld4 {v0.16b,v1.16b,v2.16b,v3.16b}, [%0], #64 \n" // load 16 ARGB
"subs %w4, %w4, #16 \n" // 16 processed per loop
"prfm pldl1keep, [%0, 448] \n"
@@ -1077,7 +1566,7 @@
int width) {
asm volatile(
"movi v3.16b, #255 \n" // load A(255)
- "1: \n"
+ "1: \n"
"ld1 {v2.16b}, [%0], #16 \n" // load R
"ld1 {v1.16b}, [%1], #16 \n" // load G
"ld1 {v0.16b}, [%2], #16 \n" // load B
@@ -1108,10 +1597,11 @@
"movi v30.16b, #255 \n"
"ushr v30.4s, v30.4s, #22 \n" // 1023
"dup v31.4s, %w5 \n"
- "1: \n"
+ "1: \n"
"ldr d2, [%2], #8 \n" // B
"ldr d1, [%1], #8 \n" // G
"ldr d0, [%0], #8 \n" // R
+ "subs %w4, %w4, #4 \n"
"ushll v2.4s, v2.4h, #0 \n" // B
"ushll v1.4s, v1.4h, #0 \n" // G
"ushll v0.4s, v0.4h, #0 \n" // R
@@ -1124,7 +1614,6 @@
"sli v2.4s, v1.4s, #10 \n" // 00GB
"sli v2.4s, v0.4s, #20 \n" // 0RGB
"orr v2.4s, #0xc0, lsl #24 \n" // ARGB (AR30)
- "subs %w4, %w4, #4 \n"
"str q2, [%3], #16 \n"
"b.gt 1b \n"
: "+r"(src_r), // %0
@@ -1142,32 +1631,34 @@
uint8_t* dst_ar30,
int /* depth */,
int width) {
+ // Neon has no "shift left and accumulate/orr", so use a multiply-add to
+ // perform the shift instead.
+ int limit = 1023;
asm volatile(
- "movi v30.16b, #255 \n"
- "ushr v30.4s, v30.4s, #22 \n" // 1023
- "1: \n"
- "ldr d2, [%2], #8 \n" // B
- "ldr d1, [%1], #8 \n" // G
- "ldr d0, [%0], #8 \n" // R
- "ushll v2.4s, v2.4h, #0 \n" // 000B
- "ushll v1.4s, v1.4h, #0 \n" // G
- "ushll v0.4s, v0.4h, #0 \n" // R
- "umin v2.4s, v2.4s, v30.4s \n"
- "umin v1.4s, v1.4s, v30.4s \n"
- "umin v0.4s, v0.4s, v30.4s \n"
- "sli v2.4s, v1.4s, #10 \n" // 00GB
- "sli v2.4s, v0.4s, #20 \n" // 0RGB
- "orr v2.4s, #0xc0, lsl #24 \n" // ARGB (AR30)
- "subs %w4, %w4, #4 \n"
- "str q2, [%3], #16 \n"
+ "dup v5.8h, %w[limit] \n"
+ "movi v6.8h, #16 \n" // 1 << 4
+ "movi v7.8h, #4, lsl #8 \n" // 1 << 10
+ "1: \n"
+ "ldr q0, [%0], #16 \n" // xxxxxxRrrrrrrrrr
+ "ldr q1, [%1], #16 \n" // xxxxxxGggggggggg
+ "ldr q2, [%2], #16 \n" // xxxxxxBbbbbbbbbb
+ "subs %w4, %w4, #8 \n"
+ "umin v0.8h, v0.8h, v5.8h \n" // 000000Rrrrrrrrrr
+ "umin v1.8h, v1.8h, v5.8h \n" // 000000Gggggggggg
+ "movi v4.8h, #0xc0, lsl #8 \n" // 1100000000000000
+ "umin v3.8h, v2.8h, v5.8h \n" // 000000Bbbbbbbbbb
+ "mla v4.8h, v0.8h, v6.8h \n" // 11Rrrrrrrrrr0000
+ "mla v3.8h, v1.8h, v7.8h \n" // ggggggBbbbbbbbbb
+ "usra v4.8h, v1.8h, #6 \n" // 11RrrrrrrrrrGggg
+ "st2 {v3.8h, v4.8h}, [%3], #32 \n"
"b.gt 1b \n"
: "+r"(src_r), // %0
"+r"(src_g), // %1
"+r"(src_b), // %2
"+r"(dst_ar30), // %3
"+r"(width) // %4
- :
- : "memory", "cc", "v0", "v1", "v2", "v30");
+ : [limit] "r"(limit)
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7");
}
void MergeAR64Row_NEON(const uint16_t* src_r,
@@ -1183,11 +1674,12 @@
"dup v30.8h, %w7 \n"
"dup v31.8h, %w6 \n"
- "1: \n"
+ "1: \n"
"ldr q2, [%0], #16 \n" // R
"ldr q1, [%1], #16 \n" // G
"ldr q0, [%2], #16 \n" // B
"ldr q3, [%3], #16 \n" // A
+ "subs %w5, %w5, #8 \n"
"umin v2.8h, v2.8h, v30.8h \n"
"prfm pldl1keep, [%0, 448] \n"
"umin v1.8h, v1.8h, v30.8h \n"
@@ -1200,7 +1692,6 @@
"ushl v1.8h, v1.8h, v31.8h \n"
"ushl v0.8h, v0.8h, v31.8h \n"
"ushl v3.8h, v3.8h, v31.8h \n"
- "subs %w5, %w5, #8 \n"
"st4 {v0.8h, v1.8h, v2.8h, v3.8h}, [%4], #64 \n"
"b.gt 1b \n"
: "+r"(src_r), // %0
@@ -1228,10 +1719,11 @@
"dup v30.8h, %w6 \n"
"dup v31.8h, %w5 \n"
- "1: \n"
+ "1: \n"
"ldr q2, [%0], #16 \n" // R
"ldr q1, [%1], #16 \n" // G
"ldr q0, [%2], #16 \n" // B
+ "subs %w4, %w4, #8 \n"
"umin v2.8h, v2.8h, v30.8h \n"
"prfm pldl1keep, [%0, 448] \n"
"umin v1.8h, v1.8h, v30.8h \n"
@@ -1241,7 +1733,6 @@
"ushl v2.8h, v2.8h, v31.8h \n"
"ushl v1.8h, v1.8h, v31.8h \n"
"ushl v0.8h, v0.8h, v31.8h \n"
- "subs %w4, %w4, #8 \n"
"st4 {v0.8h, v1.8h, v2.8h, v3.8h}, [%3], #64 \n"
"b.gt 1b \n"
: "+r"(src_r), // %0
@@ -1261,33 +1752,31 @@
uint8_t* dst_argb,
int depth,
int width) {
- int shift = 8 - depth;
+ // Shift is 8 - depth, +8 so the result is in the top half of each lane.
+ int shift = 16 - depth;
asm volatile(
-
"dup v31.8h, %w6 \n"
- "1: \n"
- "ldr q2, [%0], #16 \n" // R
+ "1: \n"
+ "ldr q0, [%0], #16 \n" // B
"ldr q1, [%1], #16 \n" // G
- "ldr q0, [%2], #16 \n" // B
+ "ldr q2, [%2], #16 \n" // R
"ldr q3, [%3], #16 \n" // A
- "ushl v2.8h, v2.8h, v31.8h \n"
- "prfm pldl1keep, [%0, 448] \n"
- "ushl v1.8h, v1.8h, v31.8h \n"
- "prfm pldl1keep, [%1, 448] \n"
- "ushl v0.8h, v0.8h, v31.8h \n"
- "prfm pldl1keep, [%2, 448] \n"
- "ushl v3.8h, v3.8h, v31.8h \n"
- "prfm pldl1keep, [%3, 448] \n"
- "uqxtn v2.8b, v2.8h \n"
- "uqxtn v1.8b, v1.8h \n"
- "uqxtn v0.8b, v0.8h \n"
- "uqxtn v3.8b, v3.8h \n"
"subs %w5, %w5, #8 \n"
- "st4 {v0.8b, v1.8b, v2.8b, v3.8b}, [%4], #32 \n"
+ "uqshl v0.8h, v0.8h, v31.8h \n"
+ "prfm pldl1keep, [%0, 448] \n"
+ "uqshl v1.8h, v1.8h, v31.8h \n"
+ "prfm pldl1keep, [%1, 448] \n"
+ "uqshl v2.8h, v2.8h, v31.8h \n"
+ "prfm pldl1keep, [%2, 448] \n"
+ "uqshl v3.8h, v3.8h, v31.8h \n"
+ "prfm pldl1keep, [%3, 448] \n"
+ "trn2 v0.16b, v0.16b, v1.16b \n"
+ "trn2 v1.16b, v2.16b, v3.16b \n"
+ "st2 {v0.8h, v1.8h}, [%4], #32 \n"
"b.gt 1b \n"
- : "+r"(src_r), // %0
+ : "+r"(src_b), // %0
"+r"(src_g), // %1
- "+r"(src_b), // %2
+ "+r"(src_r), // %2
"+r"(src_a), // %3
"+r"(dst_argb), // %4
"+r"(width) // %5
@@ -1301,30 +1790,29 @@
uint8_t* dst_argb,
int depth,
int width) {
- int shift = 8 - depth;
+ // Shift is 8 - depth, +8 so the result is in the top half of each lane.
+ int shift = 16 - depth;
asm volatile(
-
"dup v31.8h, %w5 \n"
- "movi v3.8b, #0xff \n" // A (0xff)
- "1: \n"
- "ldr q2, [%0], #16 \n" // R
+ "movi v3.16b, #0xff \n" // A (0xff)
+ "1: \n"
+ "ldr q0, [%0], #16 \n" // B
"ldr q1, [%1], #16 \n" // G
- "ldr q0, [%2], #16 \n" // B
- "ushl v2.8h, v2.8h, v31.8h \n"
- "prfm pldl1keep, [%0, 448] \n"
- "ushl v1.8h, v1.8h, v31.8h \n"
- "prfm pldl1keep, [%1, 448] \n"
- "ushl v0.8h, v0.8h, v31.8h \n"
- "prfm pldl1keep, [%2, 448] \n"
- "uqxtn v2.8b, v2.8h \n"
- "uqxtn v1.8b, v1.8h \n"
- "uqxtn v0.8b, v0.8h \n"
+ "ldr q2, [%2], #16 \n" // R
"subs %w4, %w4, #8 \n"
- "st4 {v0.8b, v1.8b, v2.8b, v3.8b}, [%3], #32 \n"
+ "uqshl v0.8h, v0.8h, v31.8h \n"
+ "prfm pldl1keep, [%0, 448] \n"
+ "uqshl v1.8h, v1.8h, v31.8h \n"
+ "prfm pldl1keep, [%1, 448] \n"
+ "uqshl v2.8h, v2.8h, v31.8h \n"
+ "prfm pldl1keep, [%2, 448] \n"
+ "trn2 v0.16b, v0.16b, v1.16b \n"
+ "trn2 v1.16b, v2.16b, v3.16b \n"
+ "st2 {v0.8h, v1.8h}, [%3], #32 \n"
"b.gt 1b \n"
- : "+r"(src_r), // %0
+ : "+r"(src_b), // %0
"+r"(src_g), // %1
- "+r"(src_b), // %2
+ "+r"(src_r), // %2
"+r"(dst_argb), // %3
"+r"(width) // %4
: "r"(shift) // %5
@@ -1334,7 +1822,7 @@
// Copy multiple of 32.
void CopyRow_NEON(const uint8_t* src, uint8_t* dst, int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"ldp q0, q1, [%0], #32 \n"
"prfm pldl1keep, [%0, 448] \n"
"subs %w2, %w2, #32 \n" // 32 processed per loop
@@ -1352,7 +1840,7 @@
void SetRow_NEON(uint8_t* dst, uint8_t v8, int width) {
asm volatile(
"dup v0.16b, %w2 \n" // duplicate 16 bytes
- "1: \n"
+ "1: \n"
"subs %w1, %w1, #16 \n" // 16 bytes per loop
"st1 {v0.16b}, [%0], #16 \n" // store
"b.gt 1b \n"
@@ -1365,7 +1853,7 @@
void ARGBSetRow_NEON(uint8_t* dst, uint32_t v32, int width) {
asm volatile(
"dup v0.4s, %w2 \n" // duplicate 4 ints
- "1: \n"
+ "1: \n"
"subs %w1, %w1, #4 \n" // 4 ints per loop
"st1 {v0.16b}, [%0], #16 \n" // store
"b.gt 1b \n"
@@ -1385,7 +1873,7 @@
"ld1 {v3.16b}, [%3] \n" // shuffler
"add %0, %0, %w2, sxtw \n"
"sub %0, %0, #32 \n"
- "1: \n"
+ "1: \n"
"ldr q2, [%0, 16] \n"
"ldr q1, [%0], -32 \n" // src -= 32
"subs %w2, %w2, #32 \n" // 32 pixels per loop.
@@ -1410,7 +1898,7 @@
"ld1 {v4.16b}, [%3] \n" // shuffler
"add %0, %0, %w2, sxtw #1 \n"
"sub %0, %0, #32 \n"
- "1: \n"
+ "1: \n"
"ldr q1, [%0, 16] \n"
"ldr q0, [%0], -32 \n" // src -= 32
"subs %w2, %w2, #16 \n" // 16 pixels per loop.
@@ -1434,7 +1922,7 @@
"ld1 {v4.16b}, [%4] \n" // shuffler
"add %0, %0, %w3, sxtw #1 \n"
"sub %0, %0, #32 \n"
- "1: \n"
+ "1: \n"
"ldr q1, [%0, 16] \n"
"ldr q0, [%0], -32 \n" // src -= 32
"subs %w3, %w3, #16 \n" // 16 pixels per loop.
@@ -1463,7 +1951,7 @@
"ld1 {v4.16b}, [%3] \n" // shuffler
"add %0, %0, %w2, sxtw #2 \n"
"sub %0, %0, #32 \n"
- "1: \n"
+ "1: \n"
"ldr q1, [%0, 16] \n"
"ldr q0, [%0], -32 \n" // src -= 32
"subs %w2, %w2, #8 \n" // 8 pixels per loop.
@@ -1487,7 +1975,7 @@
"add %0, %0, %w2, sxtw \n"
"sub %0, %0, #48 \n"
- "1: \n"
+ "1: \n"
"ld3 {v0.16b, v1.16b, v2.16b}, [%0], %3 \n" // src -= 48
"subs %w2, %w2, #16 \n" // 16 pixels per loop.
"tbl v0.16b, {v0.16b}, v3.16b \n"
@@ -1508,11 +1996,11 @@
int width) {
asm volatile(
"movi v4.8b, #255 \n" // Alpha
- "1: \n"
+ "1: \n"
"ld3 {v1.8b,v2.8b,v3.8b}, [%0], #24 \n" // load 8 pixels of
// RGB24.
+ "subs %w2, %w2, #8 \n" // 8 processed per loop.
"prfm pldl1keep, [%0, 448] \n"
- "subs %w2, %w2, #8 \n" // 8 processed per loop.
"st4 {v1.8b,v2.8b,v3.8b,v4.8b}, [%1], #32 \n" // store 8 ARGB
"b.gt 1b \n"
: "+r"(src_rgb24), // %0
@@ -1526,12 +2014,12 @@
void RAWToARGBRow_NEON(const uint8_t* src_raw, uint8_t* dst_argb, int width) {
asm volatile(
"movi v5.8b, #255 \n" // Alpha
- "1: \n"
+ "1: \n"
"ld3 {v0.8b,v1.8b,v2.8b}, [%0], #24 \n" // read r g b
"subs %w2, %w2, #8 \n" // 8 processed per loop.
- "orr v3.8b, v1.8b, v1.8b \n" // move g
+ "mov v3.8b, v1.8b \n" // move g
"prfm pldl1keep, [%0, 448] \n"
- "orr v4.8b, v0.8b, v0.8b \n" // move r
+ "mov v4.8b, v0.8b \n" // move r
"st4 {v2.8b,v3.8b,v4.8b,v5.8b}, [%1], #32 \n" // store b g r a
"b.gt 1b \n"
: "+r"(src_raw), // %0
@@ -1545,12 +2033,12 @@
void RAWToRGBARow_NEON(const uint8_t* src_raw, uint8_t* dst_rgba, int width) {
asm volatile(
"movi v0.8b, #255 \n" // Alpha
- "1: \n"
+ "1: \n"
"ld3 {v3.8b,v4.8b,v5.8b}, [%0], #24 \n" // read r g b
"subs %w2, %w2, #8 \n" // 8 processed per loop.
- "orr v2.8b, v4.8b, v4.8b \n" // move g
+ "mov v2.8b, v4.8b \n" // move g
"prfm pldl1keep, [%0, 448] \n"
- "orr v1.8b, v5.8b, v5.8b \n" // move r
+ "mov v1.8b, v5.8b \n" // move r
"st4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%1], #32 \n" // store a b g r
"b.gt 1b \n"
: "+r"(src_raw), // %0
@@ -1563,12 +2051,12 @@
void RAWToRGB24Row_NEON(const uint8_t* src_raw, uint8_t* dst_rgb24, int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"ld3 {v0.8b,v1.8b,v2.8b}, [%0], #24 \n" // read r g b
"subs %w2, %w2, #8 \n" // 8 processed per loop.
- "orr v3.8b, v1.8b, v1.8b \n" // move g
+ "mov v3.8b, v1.8b \n" // move g
"prfm pldl1keep, [%0, 448] \n"
- "orr v4.8b, v0.8b, v0.8b \n" // move r
+ "mov v4.8b, v0.8b \n" // move r
"st3 {v2.8b,v3.8b,v4.8b}, [%1], #24 \n" // store b g r
"b.gt 1b \n"
: "+r"(src_raw), // %0
@@ -1579,29 +2067,28 @@
);
}
-#define RGB565TOARGB \
- "shrn v6.8b, v0.8h, #5 \n" /* G xxGGGGGG */ \
- "shl v6.8b, v6.8b, #2 \n" /* G GGGGGG00 upper 6 */ \
- "ushr v4.8b, v6.8b, #6 \n" /* G 000000GG lower 2 */ \
- "orr v1.8b, v4.8b, v6.8b \n" /* G */ \
- "xtn v2.8b, v0.8h \n" /* B xxxBBBBB */ \
- "ushr v0.8h, v0.8h, #11 \n" /* R 000RRRRR */ \
- "xtn2 v2.16b,v0.8h \n" /* R in upper part */ \
- "shl v2.16b, v2.16b, #3 \n" /* R,B BBBBB000 upper 5 */ \
- "ushr v0.16b, v2.16b, #5 \n" /* R,B 00000BBB lower 3 */ \
- "orr v0.16b, v0.16b, v2.16b \n" /* R,B */ \
- "dup v2.2D, v0.D[1] \n" /* R */
+#define RGB565TOARGB \
+ /* Input: v0/v4.8h: RRRRRGGGGGGBBBBB */ \
+ "shrn v1.8b, v0.8h, #3 \n" /* G GGGGGGxx */ \
+ "shrn2 v1.16b, v4.8h, #3 \n" /* G GGGGGGxx */ \
+ "uzp2 v2.16b, v0.16b, v4.16b \n" /* R RRRRRxxx */ \
+ "uzp1 v0.16b, v0.16b, v4.16b \n" /* B xxxBBBBB */ \
+ "sri v1.16b, v1.16b, #6 \n" /* G GGGGGGGG, fill 2 */ \
+ "shl v0.16b, v0.16b, #3 \n" /* B BBBBB000 */ \
+ "sri v2.16b, v2.16b, #5 \n" /* R RRRRRRRR, fill 3 */ \
+ "sri v0.16b, v0.16b, #5 \n" /* R BBBBBBBB, fill 3 */
void RGB565ToARGBRow_NEON(const uint8_t* src_rgb565,
uint8_t* dst_argb,
int width) {
asm volatile(
- "movi v3.8b, #255 \n" // Alpha
- "1: \n"
- "ld1 {v0.16b}, [%0], #16 \n" // load 8 RGB565 pixels.
- "subs %w2, %w2, #8 \n" // 8 processed per loop.
+ "movi v3.16b, #255 \n" // Alpha
+ "1: \n"
+ "ldp q0, q4, [%0], #32 \n" // load 16 RGB565 pixels
+ "subs %w2, %w2, #16 \n" // 16 processed per loop
"prfm pldl1keep, [%0, 448] \n" RGB565TOARGB
- "st4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%1], #32 \n" // store 8 ARGB
+ "st4 {v0.16b,v1.16b,v2.16b,v3.16b}, [%1] \n" // store 16 ARGB
+ "add %1, %1, #64 \n"
"b.gt 1b \n"
: "+r"(src_rgb565), // %0
"+r"(dst_argb), // %1
@@ -1611,86 +2098,79 @@
);
}
-#define ARGB1555TOARGB \
- "ushr v2.8h, v0.8h, #10 \n" /* R xxxRRRRR */ \
- "shl v2.8h, v2.8h, #3 \n" /* R RRRRR000 upper 5 */ \
- "xtn v3.8b, v2.8h \n" /* RRRRR000 AAAAAAAA */ \
- \
- "sshr v2.8h, v0.8h, #15 \n" /* A AAAAAAAA */ \
- "xtn2 v3.16b, v2.8h \n" \
- \
- "xtn v2.8b, v0.8h \n" /* B xxxBBBBB */ \
- "shrn2 v2.16b,v0.8h, #5 \n" /* G xxxGGGGG */ \
- \
- "ushr v1.16b, v3.16b, #5 \n" /* R,A 00000RRR lower 3 */ \
- "shl v0.16b, v2.16b, #3 \n" /* B,G BBBBB000 upper 5 */ \
- "ushr v2.16b, v0.16b, #5 \n" /* B,G 00000BBB lower 3 */ \
- \
- "orr v0.16b, v0.16b, v2.16b \n" /* B,G */ \
- "orr v2.16b, v1.16b, v3.16b \n" /* R,A */ \
- "dup v1.2D, v0.D[1] \n" \
- "dup v3.2D, v2.D[1] \n"
+#define ARGB1555TOARGB \
+ /* Input: ARRRRRGGGGGBBBBB */ \
+ "shrn v2.8b, v0.8h, #7 \n" /* RRRRRxxx */ \
+ "uzp1 v29.16b, v0.16b, v4.16b \n" /* xxxBBBBB */ \
+ "shrn v1.8b, v0.8h, #2 \n" /* GGGGGxxx */ \
+ "uzp2 v3.16b, v0.16b, v4.16b \n" /* Axxxxxxx */ \
+ "shrn2 v2.16b, v4.8h, #7 \n" /* RRRRRxxx */ \
+ "shl v0.16b, v29.16b, #3 \n" /* BBBBB000 */ \
+ "shrn2 v1.16b, v4.8h, #2 \n" /* GGGGGxxx */ \
+ "sshr v3.16b, v3.16b, #7 \n" /* AAAAAAAA */ \
+ "sri v2.16b, v2.16b, #5 \n" /* RRRRRRRR */ \
+ "sri v1.16b, v1.16b, #5 \n" /* GGGGGGGG */ \
+ "sri v0.16b, v0.16b, #5 \n" /* BBBBBBBB */
// RGB555TOARGB is same as ARGB1555TOARGB but ignores alpha.
-#define RGB555TOARGB \
- "ushr v2.8h, v0.8h, #10 \n" /* R xxxRRRRR */ \
- "shl v2.8h, v2.8h, #3 \n" /* R RRRRR000 upper 5 */ \
- "xtn v3.8b, v2.8h \n" /* RRRRR000 */ \
- \
- "xtn v2.8b, v0.8h \n" /* B xxxBBBBB */ \
- "shrn2 v2.16b,v0.8h, #5 \n" /* G xxxGGGGG */ \
- \
- "ushr v1.16b, v3.16b, #5 \n" /* R 00000RRR lower 3 */ \
- "shl v0.16b, v2.16b, #3 \n" /* B,G BBBBB000 upper 5 */ \
- "ushr v2.16b, v0.16b, #5 \n" /* B,G 00000BBB lower 3 */ \
- \
- "orr v0.16b, v0.16b, v2.16b \n" /* B,G */ \
- "orr v2.16b, v1.16b, v3.16b \n" /* R */ \
- "dup v1.2D, v0.D[1] \n" /* G */
+#define RGB555TOARGB \
+ /* Input: xRRRRRGGGGGBBBBB */ \
+ "uzp1 v29.16b, v0.16b, v3.16b \n" /* xxxBBBBB */ \
+ "shrn v2.8b, v0.8h, #7 \n" /* RRRRRxxx */ \
+ "shrn v1.8b, v0.8h, #2 \n" /* GGGGGxxx */ \
+ "shl v0.16b, v29.16b, #3 \n" /* BBBBB000 */ \
+ "shrn2 v2.16b, v3.8h, #7 \n" /* RRRRRxxx */ \
+ "shrn2 v1.16b, v3.8h, #2 \n" /* GGGGGxxx */ \
+ \
+ "sri v0.16b, v0.16b, #5 \n" /* BBBBBBBB */ \
+ "sri v2.16b, v2.16b, #5 \n" /* RRRRRRRR */ \
+ "sri v1.16b, v1.16b, #5 \n" /* GGGGGGGG */
void ARGB1555ToARGBRow_NEON(const uint8_t* src_argb1555,
uint8_t* dst_argb,
int width) {
asm volatile(
- "movi v3.8b, #255 \n" // Alpha
- "1: \n"
- "ld1 {v0.16b}, [%0], #16 \n" // load 8 ARGB1555 pixels.
+ "1: \n"
+ "ldp q0, q4, [%0], #32 \n" // load 16 ARGB1555 pixels
"prfm pldl1keep, [%0, 448] \n"
- "subs %w2, %w2, #8 \n" // 8 processed per loop.
+ "subs %w2, %w2, #16 \n" // 16 processed per loop
ARGB1555TOARGB
- "st4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%1], #32 \n" // store 8 ARGB
+ "st4 {v0.16b,v1.16b,v2.16b,v3.16b}, [%1] \n" // store 16 ARGB
+ "add %1, %1, #64 \n"
"b.gt 1b \n"
: "+r"(src_argb1555), // %0
"+r"(dst_argb), // %1
"+r"(width) // %2
:
- : "cc", "memory", "v0", "v1", "v2", "v3" // Clobber List
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v29" // Clobber List
);
}
-// Convert v0.8h to b = v0.8b g = v1.8b r = v2.8b
-// clobbers v3
-#define ARGB4444TOARGB \
- "shrn v1.8b, v0.8h, #8 \n" /* v1(l) AR */ \
- "xtn2 v1.16b, v0.8h \n" /* v1(h) GB */ \
- "shl v2.16b, v1.16b, #4 \n" /* B,R BBBB0000 */ \
- "ushr v3.16b, v1.16b, #4 \n" /* G,A 0000GGGG */ \
- "ushr v0.16b, v2.16b, #4 \n" /* B,R 0000BBBB */ \
- "shl v1.16b, v3.16b, #4 \n" /* G,A GGGG0000 */ \
- "orr v2.16b, v0.16b, v2.16b \n" /* B,R BBBBBBBB */ \
- "orr v3.16b, v1.16b, v3.16b \n" /* G,A GGGGGGGG */ \
- "dup v0.2D, v2.D[1] \n" \
- "dup v1.2D, v3.D[1] \n"
+#define ARGB4444TOARGB \
+ /* Input: v1.8h = AAAARRRR_GGGGBBBB */ \
+ "shl v0.16b, v1.16b, #4 \n" /* RRRR0000_BBBB0000 */ \
+ "sri v1.16b, v1.16b, #4 \n" /* AAAAAAAA_GGGGGGGG */ \
+ "sri v0.16b, v0.16b, #4 \n" /* RRRRRRRR_BBBBBBBB */
+
+#define ARGB4444TORGB \
+ /* Input: v0.8h = xxxxRRRRGGGGBBBB */ \
+ "uzp1 v1.16b, v0.16b, v3.16b \n" /* GGGGBBBB */ \
+ "shrn v2.8b, v0.8h, #4 \n" /* RRRRxxxx */ \
+ "shl v0.16b, v1.16b, #4 \n" /* BBBB0000 */ \
+ "shrn2 v2.16b, v3.8h, #4 \n" /* RRRRxxxx */ \
+ "sri v1.16b, v1.16b, #4 \n" /* GGGGGGGG */ \
+ "sri v2.16b, v2.16b, #4 \n" /* RRRRRRRR */ \
+ "sri v0.16b, v0.16b, #4 \n" /* BBBBBBBB */
void ARGB4444ToARGBRow_NEON(const uint8_t* src_argb4444,
uint8_t* dst_argb,
int width) {
asm volatile(
- "1: \n"
- "ld1 {v0.16b}, [%0], #16 \n" // load 8 ARGB4444 pixels.
+ "1: \n"
+ "ld1 {v1.16b}, [%0], #16 \n" // load 8 ARGB4444 pixels.
"subs %w2, %w2, #8 \n" // 8 processed per loop.
"prfm pldl1keep, [%0, 448] \n" ARGB4444TOARGB
- "st4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%1], #32 \n" // store 8 ARGB
+ "st2 {v0.16b, v1.16b}, [%1], #32 \n" // store 8 ARGB.
"b.gt 1b \n"
: "+r"(src_argb4444), // %0
"+r"(dst_argb), // %1
@@ -1700,11 +2180,64 @@
);
}
+static const int16_t kAR30Row_BoxShifts[] = {0, -6, 0, -6, 0, -6, 0, -6};
+
+static const uint8_t kABGRToAR30Row_BoxIndices[] = {
+ 2, 2, 1, 1, 6, 6, 5, 5, 10, 10, 9, 9, 14, 14, 13, 13,
+ 0, 0, 3, 3, 4, 4, 7, 7, 8, 8, 11, 11, 12, 12, 15, 15};
+static const uint8_t kARGBToAR30Row_BoxIndices[] = {
+ 0, 0, 1, 1, 4, 4, 5, 5, 8, 8, 9, 9, 12, 12, 13, 13,
+ 2, 2, 3, 3, 6, 6, 7, 7, 10, 10, 11, 11, 14, 14, 15, 15};
+
+// ARGB or ABGR as input, reordering based on TBL indices parameter.
+static void ABCDToAR30Row_NEON(const uint8_t* src_abcd,
+ uint8_t* dst_ar30,
+ int width,
+ const uint8_t* indices) {
+ asm volatile(
+ "movi v2.4s, #0xf, msl 16 \n" // 0xfffff
+ "ldr q3, [%[kAR30Row_BoxShifts]] \n"
+ "ldp q4, q5, [%[indices]] \n"
+ "1: \n"
+ "ldp q0, q20, [%[src]], #32 \n"
+ "subs %w[width], %w[width], #8 \n"
+ "tbl v1.16b, {v0.16b}, v5.16b \n"
+ "tbl v21.16b, {v20.16b}, v5.16b \n"
+ "tbl v0.16b, {v0.16b}, v4.16b \n"
+ "tbl v20.16b, {v20.16b}, v4.16b \n"
+ "ushl v0.8h, v0.8h, v3.8h \n"
+ "ushl v20.8h, v20.8h, v3.8h \n"
+ "ushl v1.8h, v1.8h, v3.8h \n"
+ "ushl v21.8h, v21.8h, v3.8h \n"
+ "ushr v0.4s, v0.4s, #6 \n"
+ "ushr v20.4s, v20.4s, #6 \n"
+ "shl v1.4s, v1.4s, #14 \n"
+ "shl v21.4s, v21.4s, #14 \n"
+ "bif v0.16b, v1.16b, v2.16b \n"
+ "bif v20.16b, v21.16b, v2.16b \n"
+ "stp q0, q20, [%[dst]], #32 \n"
+ "b.gt 1b \n"
+ : [src] "+r"(src_abcd), // %[src]
+ [dst] "+r"(dst_ar30), // %[dst]
+ [width] "+r"(width) // %[width]
+ : [kAR30Row_BoxShifts] "r"(kAR30Row_BoxShifts), // %[kAR30Row_BoxShifts]
+ [indices] "r"(indices) // %[indices]
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v20", "v21");
+}
+
+void ABGRToAR30Row_NEON(const uint8_t* src_abgr, uint8_t* dst_ar30, int width) {
+ ABCDToAR30Row_NEON(src_abgr, dst_ar30, width, kABGRToAR30Row_BoxIndices);
+}
+
+void ARGBToAR30Row_NEON(const uint8_t* src_argb, uint8_t* dst_ar30, int width) {
+ ABCDToAR30Row_NEON(src_argb, dst_ar30, width, kARGBToAR30Row_BoxIndices);
+}
+
void ARGBToRGB24Row_NEON(const uint8_t* src_argb,
uint8_t* dst_rgb24,
int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"ld4 {v0.16b,v1.16b,v2.16b,v3.16b}, [%0], #64 \n" // load 16 ARGB
"subs %w2, %w2, #16 \n" // 16 pixels per loop.
"prfm pldl1keep, [%0, 448] \n"
@@ -1720,12 +2253,12 @@
void ARGBToRAWRow_NEON(const uint8_t* src_argb, uint8_t* dst_raw, int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"ld4 {v1.8b,v2.8b,v3.8b,v4.8b}, [%0], #32 \n" // load b g r a
"subs %w2, %w2, #8 \n" // 8 processed per loop.
- "orr v4.8b, v2.8b, v2.8b \n" // mov g
+ "mov v4.8b, v2.8b \n" // mov g
"prfm pldl1keep, [%0, 448] \n"
- "orr v5.8b, v1.8b, v1.8b \n" // mov b
+ "mov v5.8b, v1.8b \n" // mov b
"st3 {v3.8b,v4.8b,v5.8b}, [%1], #24 \n" // store r g b
"b.gt 1b \n"
: "+r"(src_argb), // %0
@@ -1738,7 +2271,7 @@
void YUY2ToYRow_NEON(const uint8_t* src_yuy2, uint8_t* dst_y, int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"ld2 {v0.16b,v1.16b}, [%0], #32 \n" // load 16 pixels of YUY2.
"subs %w2, %w2, #16 \n" // 16 processed per loop.
"prfm pldl1keep, [%0, 448] \n"
@@ -1754,7 +2287,7 @@
void UYVYToYRow_NEON(const uint8_t* src_uyvy, uint8_t* dst_y, int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"ld2 {v0.16b,v1.16b}, [%0], #32 \n" // load 16 pixels of UYVY.
"subs %w2, %w2, #16 \n" // 16 processed per loop.
"prfm pldl1keep, [%0, 448] \n"
@@ -1773,7 +2306,7 @@
uint8_t* dst_v,
int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // load 16 YUY2
"subs %w3, %w3, #16 \n" // 16 pixels = 8 UVs.
"prfm pldl1keep, [%0, 448] \n"
@@ -1794,7 +2327,7 @@
uint8_t* dst_v,
int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // load 16 UYVY
"subs %w3, %w3, #16 \n" // 16 pixels = 8 UVs.
"prfm pldl1keep, [%0, 448] \n"
@@ -1817,7 +2350,7 @@
int width) {
const uint8_t* src_yuy2b = src_yuy2 + stride_yuy2;
asm volatile(
- "1: \n"
+ "1: \n"
"ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // load 16 pixels
"subs %w4, %w4, #16 \n" // 16 pixels = 8 UVs.
"ld4 {v4.8b,v5.8b,v6.8b,v7.8b}, [%1], #32 \n" // load next row
@@ -1845,7 +2378,7 @@
int width) {
const uint8_t* src_uyvyb = src_uyvy + stride_uyvy;
asm volatile(
- "1: \n"
+ "1: \n"
"ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // load 16 pixels
"subs %w4, %w4, #16 \n" // 16 pixels = 8 UVs.
"ld4 {v4.8b,v5.8b,v6.8b,v7.8b}, [%1], #32 \n" // load next row
@@ -1872,7 +2405,7 @@
int width) {
const uint8_t* src_yuy2b = src_yuy2 + stride_yuy2;
asm volatile(
- "1: \n"
+ "1: \n"
"ld2 {v0.16b,v1.16b}, [%0], #32 \n" // load 16 pixels
"subs %w3, %w3, #16 \n" // 16 pixels = 8 UVs.
"ld2 {v2.16b,v3.16b}, [%1], #32 \n" // load next row
@@ -1896,7 +2429,7 @@
int width) {
asm volatile(
"ld1 {v2.16b}, [%3] \n" // shuffler
- "1: \n"
+ "1: \n"
"ld1 {v0.16b}, [%0], #16 \n" // load 4 pixels.
"subs %w2, %w2, #4 \n" // 4 processed per loop
"prfm pldl1keep, [%0, 448] \n"
@@ -1917,10 +2450,10 @@
uint8_t* dst_yuy2,
int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"ld2 {v0.8b, v1.8b}, [%0], #16 \n" // load 16 Ys
"subs %w4, %w4, #16 \n" // 16 pixels
- "orr v2.8b, v1.8b, v1.8b \n"
+ "mov v2.8b, v1.8b \n"
"prfm pldl1keep, [%0, 448] \n"
"ld1 {v1.8b}, [%1], #8 \n" // load 8 Us
"ld1 {v3.8b}, [%2], #8 \n" // load 8 Vs
@@ -1941,13 +2474,13 @@
uint8_t* dst_uyvy,
int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"ld2 {v1.8b,v2.8b}, [%0], #16 \n" // load 16 Ys
- "orr v3.8b, v2.8b, v2.8b \n"
+ "subs %w4, %w4, #16 \n" // 16 pixels
+ "mov v3.8b, v2.8b \n"
"prfm pldl1keep, [%0, 448] \n"
"ld1 {v0.8b}, [%1], #8 \n" // load 8 Us
"ld1 {v2.8b}, [%2], #8 \n" // load 8 Vs
- "subs %w4, %w4, #16 \n" // 16 pixels
"st4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%3], #32 \n" // Store 16 pixels.
"b.gt 1b \n"
: "+r"(src_y), // %0
@@ -1963,7 +2496,7 @@
uint8_t* dst_rgb565,
int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"ld4 {v16.8b,v17.8b,v18.8b,v19.8b}, [%0], #32 \n" // load 8
// pixels
"subs %w2, %w2, #8 \n" // 8 processed per loop.
@@ -1983,7 +2516,7 @@
int width) {
asm volatile(
"dup v1.4s, %w3 \n" // dither4
- "1: \n"
+ "1: \n"
"ld4 {v16.8b,v17.8b,v18.8b,v19.8b}, [%0], #32 \n" // load 8 ARGB
"subs %w2, %w2, #8 \n" // 8 processed per loop.
"uqadd v16.8b, v16.8b, v1.8b \n"
@@ -2003,27 +2536,24 @@
uint8_t* dst_argb1555,
int width) {
asm volatile(
- "1: \n"
- "ld4 {v16.8b,v17.8b,v18.8b,v19.8b}, [%0], #32 \n" // load 8
- // pixels
+ "1: \n"
+ "ld2 {v16.8h,v17.8h}, [%0], #32 \n" // load 8 pixels
"subs %w2, %w2, #8 \n" // 8 processed per loop.
"prfm pldl1keep, [%0, 448] \n" ARGBTOARGB1555
- "st1 {v0.16b}, [%1], #16 \n" // store 8 pixels
+ "st1 {v17.16b}, [%1], #16 \n" // store 8 pixels
"b.gt 1b \n"
: "+r"(src_argb), // %0
"+r"(dst_argb1555), // %1
"+r"(width) // %2
:
- : "cc", "memory", "v0", "v16", "v17", "v18", "v19");
+ : "cc", "memory", "v1", "v2", "v16", "v17");
}
void ARGBToARGB4444Row_NEON(const uint8_t* src_argb,
uint8_t* dst_argb4444,
int width) {
asm volatile(
- "movi v23.16b, #0x0f \n" // bits to clear with
- // vbic.
- "1: \n"
+ "1: \n"
"ld4 {v16.8b,v17.8b,v18.8b,v19.8b}, [%0], #32 \n" // load 8
// pixels
"subs %w2, %w2, #8 \n" // 8 processed per loop.
@@ -2034,20 +2564,20 @@
"+r"(dst_argb4444), // %1
"+r"(width) // %2
:
- : "cc", "memory", "v0", "v1", "v16", "v17", "v18", "v19", "v23");
+ : "cc", "memory", "v0", "v1", "v16", "v17", "v18", "v19");
}
-#if LIBYUV_USE_ST2
+#if defined(LIBYUV_USE_ST2)
void ARGBToAR64Row_NEON(const uint8_t* src_argb,
uint16_t* dst_ar64,
int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"ldp q0, q2, [%0], #32 \n" // load 8 pixels
+ "subs %w2, %w2, #8 \n" // 8 processed per loop.
"mov v1.16b, v0.16b \n"
"prfm pldl1keep, [%0, 448] \n"
"mov v3.16b, v2.16b \n"
- "subs %w2, %w2, #8 \n" // 8 processed per loop.
"st2 {v0.16b, v1.16b}, [%1], #32 \n" // store 4 pixels
"st2 {v2.16b, v3.16b}, [%1], #32 \n" // store 4 pixels
"b.gt 1b \n"
@@ -2066,14 +2596,14 @@
int width) {
asm volatile(
"ldr q4, [%3] \n" // shuffler
- "1: \n"
+ "1: \n"
"ldp q0, q2, [%0], #32 \n" // load 8 pixels
+ "subs %w2, %w2, #8 \n" // 8 processed per loop.
"tbl v0.16b, {v0.16b}, v4.16b \n"
"tbl v2.16b, {v2.16b}, v4.16b \n"
"prfm pldl1keep, [%0, 448] \n"
"mov v1.16b, v0.16b \n"
"mov v3.16b, v2.16b \n"
- "subs %w2, %w2, #8 \n" // 8 processed per loop.
"st2 {v0.16b, v1.16b}, [%1], #32 \n" // store 4 pixels
"st2 {v2.16b, v3.16b}, [%1], #32 \n" // store 4 pixels
"b.gt 1b \n"
@@ -2088,7 +2618,7 @@
uint16_t* dst_ar64,
int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"ldp q0, q1, [%0], #32 \n" // load 8 ARGB pixels
"subs %w2, %w2, #8 \n" // 8 processed per loop.
"zip1 v2.16b, v0.16b, v0.16b \n"
@@ -2114,7 +2644,7 @@
int width) {
asm volatile(
"ldp q6, q7, [%3] \n" // 2 shufflers
- "1: \n"
+ "1: \n"
"ldp q0, q1, [%0], #32 \n" // load 8 pixels
"subs %w2, %w2, #8 \n" // 8 processed per loop.
"tbl v2.16b, {v0.16b}, v6.16b \n" // ARGB to AB64
@@ -2140,13 +2670,13 @@
int width) {
asm volatile(
"ldr q4, [%3] \n" // shuffler
- "1: \n"
+ "1: \n"
"ldp q0, q1, [%0], #32 \n" // load 4 pixels
"ldp q2, q3, [%0], #32 \n" // load 4 pixels
+ "subs %w2, %w2, #8 \n" // 8 processed per loop.
"tbl v0.16b, {v0.16b, v1.16b}, v4.16b \n"
"prfm pldl1keep, [%0, 448] \n"
"tbl v2.16b, {v2.16b, v3.16b}, v4.16b \n"
- "subs %w2, %w2, #8 \n" // 8 processed per loop.
"stp q0, q2, [%1], #32 \n" // store 8 pixels
"b.gt 1b \n"
: "+r"(src_ar64), // %0
@@ -2164,13 +2694,13 @@
int width) {
asm volatile(
"ldr q4, [%3] \n" // shuffler
- "1: \n"
+ "1: \n"
"ldp q0, q1, [%0], #32 \n" // load 4 pixels
"ldp q2, q3, [%0], #32 \n" // load 4 pixels
+ "subs %w2, %w2, #8 \n" // 8 processed per loop.
"tbl v0.16b, {v0.16b, v1.16b}, v4.16b \n"
"prfm pldl1keep, [%0, 448] \n"
"tbl v2.16b, {v2.16b, v3.16b}, v4.16b \n"
- "subs %w2, %w2, #8 \n" // 8 processed per loop.
"stp q0, q2, [%1], #32 \n" // store 8 pixels
"b.gt 1b \n"
: "+r"(src_ab64), // %0
@@ -2184,7 +2714,7 @@
uint8_t* dst_a,
int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"ld4 {v0.16b,v1.16b,v2.16b,v3.16b}, [%0], #64 \n" // load 16
"prfm pldl1keep, [%0, 448] \n"
"subs %w2, %w2, #16 \n" // 16 processed per loop
@@ -2198,17 +2728,19 @@
);
}
+// Coefficients expressed as negatives to allow 128
struct RgbUVConstants {
- uint8_t kRGBToU[4];
- uint8_t kRGBToV[4];
+ int8_t kRGBToU[4];
+ int8_t kRGBToV[4];
};
// 8x1 pixels.
-void ARGBToUV444MatrixRow_NEON(const uint8_t* src_argb,
- uint8_t* dst_u,
- uint8_t* dst_v,
- int width,
- const struct RgbUVConstants* rgbuvconstants) {
+static void ARGBToUV444MatrixRow_NEON(
+ const uint8_t* src_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct RgbUVConstants* rgbuvconstants) {
asm volatile(
"ldr d0, [%4] \n" // load rgbuvconstants
"dup v24.16b, v0.b[0] \n" // UB 0.875 coefficient
@@ -2216,9 +2748,10 @@
"dup v26.16b, v0.b[2] \n" // UR -0.2969 coefficient
"dup v27.16b, v0.b[4] \n" // VB -0.1406 coefficient
"dup v28.16b, v0.b[5] \n" // VG -0.7344 coefficient
- "movi v29.16b, #0x80 \n" // 128.5
+ "neg v24.16b, v24.16b \n"
+ "movi v29.8h, #0x80, lsl #8 \n" // 128.0
- "1: \n"
+ "1: \n"
"ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // load 8 ARGB
"subs %w3, %w3, #8 \n" // 8 processed per loop.
"umull v4.8h, v0.8b, v24.8b \n" // B
@@ -2230,8 +2763,8 @@
"umlsl v3.8h, v1.8b, v28.8b \n" // G
"umlsl v3.8h, v0.8b, v27.8b \n" // B
- "addhn v0.8b, v4.8h, v29.8h \n" // +128 -> unsigned
- "addhn v1.8b, v3.8h, v29.8h \n" // +128 -> unsigned
+ "addhn v0.8b, v4.8h, v29.8h \n" // signed -> unsigned
+ "addhn v1.8b, v3.8h, v29.8h \n"
"st1 {v0.8b}, [%1], #8 \n" // store 8 pixels U.
"st1 {v1.8b}, [%2], #8 \n" // store 8 pixels V.
@@ -2245,51 +2778,104 @@
"v27", "v28", "v29");
}
-// RGB to bt601 coefficients
+static void ARGBToUV444MatrixRow_NEON_I8MM(
+ const uint8_t* src_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const struct RgbUVConstants* rgbuvconstants) {
+ asm volatile(
+ "ld2r {v16.4s, v17.4s}, [%[rgbuvconstants]] \n"
+ "movi v29.8h, #0x80, lsl #8 \n" // 128.0
+ "1: \n"
+ "ldp q0, q1, [%[src]], #32 \n"
+ "subs %w[width], %w[width], #8 \n" // 8 processed per loop.
+ "movi v2.4s, #0 \n"
+ "movi v3.4s, #0 \n"
+ "movi v4.4s, #0 \n"
+ "movi v5.4s, #0 \n"
+ "usdot v2.4s, v0.16b, v16.16b \n"
+ "usdot v3.4s, v1.16b, v16.16b \n"
+ "usdot v4.4s, v0.16b, v17.16b \n"
+ "usdot v5.4s, v1.16b, v17.16b \n"
+ "prfm pldl1keep, [%[src], 448] \n"
+ "uzp1 v0.8h, v2.8h, v3.8h \n"
+ "uzp1 v1.8h, v4.8h, v5.8h \n"
+ "subhn v0.8b, v29.8h, v0.8h \n" // -signed -> unsigned
+ "subhn v1.8b, v29.8h, v1.8h \n"
+ "str d0, [%[dst_u]], #8 \n" // store 8 pixels U.
+ "str d1, [%[dst_v]], #8 \n" // store 8 pixels V.
+ "b.gt 1b \n"
+ : [src] "+r"(src_argb), // %[src]
+ [dst_u] "+r"(dst_u), // %[dst_u]
+ [dst_v] "+r"(dst_v), // %[dst_v]
+ [width] "+r"(width) // %[width]
+ : [rgbuvconstants] "r"(rgbuvconstants) // %[rgbuvconstants]
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v16", "v17",
+ "v29");
+}
+
+// RGB to BT601 coefficients
// UB 0.875 coefficient = 112
-// UG -0.5781 coefficient = 74
-// UR -0.2969 coefficient = 38
-// VB -0.1406 coefficient = 18
-// VG -0.7344 coefficient = 94
-// VR 0.875 coefficient = 112 (ignored)
+// UG -0.5781 coefficient = -74
+// UR -0.2969 coefficient = -38
+// VB -0.1406 coefficient = -18
+// VG -0.7344 coefficient = -94
+// VR 0.875 coefficient = 112
-static const struct RgbUVConstants kRgb24I601UVConstants = {{112, 74, 38, 0},
- {18, 94, 112, 0}};
-
-// RGB to JPeg coefficients
-// UB coeff 0.500 = 127
-// UG coeff -0.33126 = 84
-// UR coeff -0.16874 = 43
-// VB coeff -0.08131 = 20
-// VG coeff -0.41869 = 107
-// VR coeff 0.500 = 127 (ignored)
-
-static const struct RgbUVConstants kRgb24JPegUVConstants = {{127, 84, 43, 0},
- {20, 107, 127, 0}};
+static const struct RgbUVConstants kARGBI601UVConstants = {{-112, 74, 38, 0},
+ {18, 94, -112, 0}};
void ARGBToUV444Row_NEON(const uint8_t* src_argb,
uint8_t* dst_u,
uint8_t* dst_v,
int width) {
ARGBToUV444MatrixRow_NEON(src_argb, dst_u, dst_v, width,
- &kRgb24I601UVConstants);
+ &kARGBI601UVConstants);
}
+void ARGBToUV444Row_NEON_I8MM(const uint8_t* src_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ ARGBToUV444MatrixRow_NEON_I8MM(src_argb, dst_u, dst_v, width,
+ &kARGBI601UVConstants);
+}
+
+// RGB to JPEG coefficients
+// UB 0.500 coefficient = 128
+// UG -0.33126 coefficient = -85
+// UR -0.16874 coefficient = -43
+// VB -0.08131 coefficient = -21
+// VG -0.41869 coefficient = -107
+// VR 0.500 coefficient = 128
+
+static const struct RgbUVConstants kARGBJPEGUVConstants = {{-128, 85, 43, 0},
+ {21, 107, -128, 0}};
+
void ARGBToUVJ444Row_NEON(const uint8_t* src_argb,
uint8_t* dst_u,
uint8_t* dst_v,
int width) {
ARGBToUV444MatrixRow_NEON(src_argb, dst_u, dst_v, width,
- &kRgb24JPegUVConstants);
+ &kARGBJPEGUVConstants);
+}
+
+void ARGBToUVJ444Row_NEON_I8MM(const uint8_t* src_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ ARGBToUV444MatrixRow_NEON_I8MM(src_argb, dst_u, dst_v, width,
+ &kARGBJPEGUVConstants);
}
#define RGBTOUV_SETUP_REG \
- "movi v20.8h, #56, lsl #0 \n" /* UB/VR coefficient (0.875) / 2 */ \
- "movi v21.8h, #37, lsl #0 \n" /* UG coefficient (-0.5781) / 2 */ \
- "movi v22.8h, #19, lsl #0 \n" /* UR coefficient (-0.2969) / 2 */ \
- "movi v23.8h, #9, lsl #0 \n" /* VB coefficient (-0.1406) / 2 */ \
- "movi v24.8h, #47, lsl #0 \n" /* VG coefficient (-0.7344) / 2 */ \
- "movi v25.16b, #0x80 \n" /* 128.5 (0x8080 in 16-bit) */
+ "movi v20.8h, #112 \n" /* UB/VR coefficient (0.875) */ \
+ "movi v21.8h, #74 \n" /* UG coefficient (-0.5781) */ \
+ "movi v22.8h, #38 \n" /* UR coefficient (-0.2969) */ \
+ "movi v23.8h, #18 \n" /* VB coefficient (-0.1406) */ \
+ "movi v24.8h, #94 \n" /* VG coefficient (-0.7344) */ \
+ "movi v25.8h, #0x80, lsl #8 \n" /* 128.0 (0x8000 in 16-bit) */
// 16x2 pixels -> 8x1. width is number of argb pixels. e.g. 16.
// clang-format off
@@ -2315,8 +2901,9 @@
const uint8_t* src_argb_1 = src_argb + src_stride_argb;
asm volatile (
RGBTOUV_SETUP_REG
- "1: \n"
+ "1: \n"
"ld4 {v0.16b,v1.16b,v2.16b,v3.16b}, [%0], #64 \n" // load 16 pixels.
+ "subs %w4, %w4, #16 \n" // 16 processed per loop.
"uaddlp v0.8h, v0.16b \n" // B 16 bytes -> 8 shorts.
"prfm pldl1keep, [%0, 448] \n"
"uaddlp v1.8h, v1.16b \n" // G 16 bytes -> 8 shorts.
@@ -2328,11 +2915,10 @@
"uadalp v1.8h, v5.16b \n" // G 16 bytes -> 8 shorts.
"uadalp v2.8h, v6.16b \n" // R 16 bytes -> 8 shorts.
- "urshr v0.8h, v0.8h, #1 \n" // 2x average
- "urshr v1.8h, v1.8h, #1 \n"
- "urshr v2.8h, v2.8h, #1 \n"
+ "urshr v0.8h, v0.8h, #2 \n" // average of 4
+ "urshr v1.8h, v1.8h, #2 \n"
+ "urshr v2.8h, v2.8h, #2 \n"
- "subs %w4, %w4, #16 \n" // 16 processed per loop.
RGBTOUV(v0.8h, v1.8h, v2.8h)
"st1 {v0.8b}, [%2], #8 \n" // store 8 pixels U.
"st1 {v1.8b}, [%3], #8 \n" // store 8 pixels V.
@@ -2348,7 +2934,6 @@
);
}
-// TODO(fbarchard): Subsample match Intel code.
void ARGBToUVJRow_NEON(const uint8_t* src_argb,
int src_stride_argb,
uint8_t* dst_u,
@@ -2356,14 +2941,15 @@
int width) {
const uint8_t* src_argb_1 = src_argb + src_stride_argb;
asm volatile (
- "movi v20.8h, #63, lsl #0 \n" // UB/VR coeff (0.500) / 2
- "movi v21.8h, #42, lsl #0 \n" // UG coeff (-0.33126) / 2
- "movi v22.8h, #21, lsl #0 \n" // UR coeff (-0.16874) / 2
- "movi v23.8h, #10, lsl #0 \n" // VB coeff (-0.08131) / 2
- "movi v24.8h, #53, lsl #0 \n" // VG coeff (-0.41869) / 2
- "movi v25.16b, #0x80 \n" // 128.5 (0x8080 in 16-bit)
- "1: \n"
+ "movi v20.8h, #128 \n" // UB/VR coeff (0.500)
+ "movi v21.8h, #85 \n" // UG coeff (-0.33126)
+ "movi v22.8h, #43 \n" // UR coeff (-0.16874)
+ "movi v23.8h, #21 \n" // VB coeff (-0.08131)
+ "movi v24.8h, #107 \n" // VG coeff (-0.41869)
+ "movi v25.8h, #0x80, lsl #8 \n" // 128.0 (0x8000 in 16-bit)
+ "1: \n"
"ld4 {v0.16b,v1.16b,v2.16b,v3.16b}, [%0], #64 \n" // load 16 pixels.
+ "subs %w4, %w4, #16 \n" // 16 processed per loop.
"uaddlp v0.8h, v0.16b \n" // B 16 bytes -> 8 shorts.
"prfm pldl1keep, [%0, 448] \n"
"uaddlp v1.8h, v1.16b \n" // G 16 bytes -> 8 shorts.
@@ -2374,11 +2960,10 @@
"uadalp v1.8h, v5.16b \n" // G 16 bytes -> 8 shorts.
"uadalp v2.8h, v6.16b \n" // R 16 bytes -> 8 shorts.
- "urshr v0.8h, v0.8h, #1 \n" // 2x average
- "urshr v1.8h, v1.8h, #1 \n"
- "urshr v2.8h, v2.8h, #1 \n"
+ "urshr v0.8h, v0.8h, #2 \n" // average of 4
+ "urshr v1.8h, v1.8h, #2 \n"
+ "urshr v2.8h, v2.8h, #2 \n"
- "subs %w4, %w4, #16 \n" // 16 processed per loop.
RGBTOUV(v0.8h, v1.8h, v2.8h)
"st1 {v0.8b}, [%2], #8 \n" // store 8 pixels U.
"st1 {v1.8b}, [%3], #8 \n" // store 8 pixels V.
@@ -2401,14 +2986,15 @@
int width) {
const uint8_t* src_abgr_1 = src_abgr + src_stride_abgr;
asm volatile (
- "movi v20.8h, #63, lsl #0 \n" // UB/VR coeff (0.500) / 2
- "movi v21.8h, #42, lsl #0 \n" // UG coeff (-0.33126) / 2
- "movi v22.8h, #21, lsl #0 \n" // UR coeff (-0.16874) / 2
- "movi v23.8h, #10, lsl #0 \n" // VB coeff (-0.08131) / 2
- "movi v24.8h, #53, lsl #0 \n" // VG coeff (-0.41869) / 2
- "movi v25.16b, #0x80 \n" // 128.5 (0x8080 in 16-bit)
- "1: \n"
+ "movi v20.8h, #128 \n" // UB/VR coeff (0.500)
+ "movi v21.8h, #85 \n" // UG coeff (-0.33126)
+ "movi v22.8h, #43 \n" // UR coeff (-0.16874)
+ "movi v23.8h, #21 \n" // VB coeff (-0.08131)
+ "movi v24.8h, #107 \n" // VG coeff (-0.41869)
+ "movi v25.8h, #0x80, lsl #8 \n" // 128.0 (0x8000 in 16-bit)
+ "1: \n"
"ld4 {v0.16b,v1.16b,v2.16b,v3.16b}, [%0], #64 \n" // load 16 pixels.
+ "subs %w4, %w4, #16 \n" // 16 processed per loop.
"uaddlp v0.8h, v0.16b \n" // R 16 bytes -> 8 shorts.
"prfm pldl1keep, [%0, 448] \n"
"uaddlp v1.8h, v1.16b \n" // G 16 bytes -> 8 shorts.
@@ -2419,11 +3005,10 @@
"uadalp v1.8h, v5.16b \n" // G 16 bytes -> 8 shorts.
"uadalp v2.8h, v6.16b \n" // B 16 bytes -> 8 shorts.
- "urshr v0.8h, v0.8h, #1 \n" // 2x average
- "urshr v1.8h, v1.8h, #1 \n"
- "urshr v2.8h, v2.8h, #1 \n"
+ "urshr v0.8h, v0.8h, #2 \n" // average of 4
+ "urshr v1.8h, v1.8h, #2 \n"
+ "urshr v2.8h, v2.8h, #2 \n"
- "subs %w4, %w4, #16 \n" // 16 processed per loop.
RGBTOUV(v2.8h, v1.8h, v0.8h)
"st1 {v0.8b}, [%2], #8 \n" // store 8 pixels U.
"st1 {v1.8b}, [%3], #8 \n" // store 8 pixels V.
@@ -2446,14 +3031,15 @@
int width) {
const uint8_t* src_rgb24_1 = src_rgb24 + src_stride_rgb24;
asm volatile (
- "movi v20.8h, #63, lsl #0 \n" // UB/VR coeff (0.500) / 2
- "movi v21.8h, #42, lsl #0 \n" // UG coeff (-0.33126) / 2
- "movi v22.8h, #21, lsl #0 \n" // UR coeff (-0.16874) / 2
- "movi v23.8h, #10, lsl #0 \n" // VB coeff (-0.08131) / 2
- "movi v24.8h, #53, lsl #0 \n" // VG coeff (-0.41869) / 2
- "movi v25.16b, #0x80 \n" // 128.5 (0x8080 in 16-bit)
- "1: \n"
+ "movi v20.8h, #128 \n" // UB/VR coeff (0.500)
+ "movi v21.8h, #85 \n" // UG coeff (-0.33126)
+ "movi v22.8h, #43 \n" // UR coeff (-0.16874)
+ "movi v23.8h, #21 \n" // VB coeff (-0.08131)
+ "movi v24.8h, #107 \n" // VG coeff (-0.41869)
+ "movi v25.8h, #0x80, lsl #8 \n" // 128.0 (0x8000 in 16-bit)
+ "1: \n"
"ld3 {v0.16b,v1.16b,v2.16b}, [%0], #48 \n" // load 16 pixels.
+ "subs %w4, %w4, #16 \n" // 16 processed per loop.
"uaddlp v0.8h, v0.16b \n" // B 16 bytes -> 8 shorts.
"prfm pldl1keep, [%0, 448] \n"
"uaddlp v1.8h, v1.16b \n" // G 16 bytes -> 8 shorts.
@@ -2464,11 +3050,10 @@
"uadalp v1.8h, v5.16b \n" // G 16 bytes -> 8 shorts.
"uadalp v2.8h, v6.16b \n" // R 16 bytes -> 8 shorts.
- "urshr v0.8h, v0.8h, #1 \n" // 2x average
- "urshr v1.8h, v1.8h, #1 \n"
- "urshr v2.8h, v2.8h, #1 \n"
+ "urshr v0.8h, v0.8h, #2 \n" // average of 4
+ "urshr v1.8h, v1.8h, #2 \n"
+ "urshr v2.8h, v2.8h, #2 \n"
- "subs %w4, %w4, #16 \n" // 16 processed per loop.
RGBTOUV(v0.8h, v1.8h, v2.8h)
"st1 {v0.8b}, [%2], #8 \n" // store 8 pixels U.
"st1 {v1.8b}, [%3], #8 \n" // store 8 pixels V.
@@ -2491,14 +3076,15 @@
int width) {
const uint8_t* src_raw_1 = src_raw + src_stride_raw;
asm volatile (
- "movi v20.8h, #63, lsl #0 \n" // UB/VR coeff (0.500) / 2
- "movi v21.8h, #42, lsl #0 \n" // UG coeff (-0.33126) / 2
- "movi v22.8h, #21, lsl #0 \n" // UR coeff (-0.16874) / 2
- "movi v23.8h, #10, lsl #0 \n" // VB coeff (-0.08131) / 2
- "movi v24.8h, #53, lsl #0 \n" // VG coeff (-0.41869) / 2
- "movi v25.16b, #0x80 \n" // 128.5 (0x8080 in 16-bit)
- "1: \n"
+ "movi v20.8h, #128 \n" // UB/VR coeff (0.500)
+ "movi v21.8h, #85 \n" // UG coeff (-0.33126)
+ "movi v22.8h, #43 \n" // UR coeff (-0.16874)
+ "movi v23.8h, #21 \n" // VB coeff (-0.08131)
+ "movi v24.8h, #107 \n" // VG coeff (-0.41869)
+ "movi v25.8h, #0x80, lsl #8 \n" // 128.0 (0x8000 in 16-bit)
+ "1: \n"
"ld3 {v0.16b,v1.16b,v2.16b}, [%0], #48 \n" // load 16 pixels.
+ "subs %w4, %w4, #16 \n" // 16 processed per loop.
"uaddlp v0.8h, v0.16b \n" // B 16 bytes -> 8 shorts.
"prfm pldl1keep, [%0, 448] \n"
"uaddlp v1.8h, v1.16b \n" // G 16 bytes -> 8 shorts.
@@ -2509,11 +3095,10 @@
"uadalp v1.8h, v5.16b \n" // G 16 bytes -> 8 shorts.
"uadalp v2.8h, v6.16b \n" // R 16 bytes -> 8 shorts.
- "urshr v0.8h, v0.8h, #1 \n" // 2x average
- "urshr v1.8h, v1.8h, #1 \n"
- "urshr v2.8h, v2.8h, #1 \n"
+ "urshr v0.8h, v0.8h, #2 \n" // average of 4
+ "urshr v1.8h, v1.8h, #2 \n"
+ "urshr v2.8h, v2.8h, #2 \n"
- "subs %w4, %w4, #16 \n" // 16 processed per loop.
RGBTOUV(v2.8h, v1.8h, v0.8h)
"st1 {v0.8b}, [%2], #8 \n" // store 8 pixels U.
"st1 {v1.8b}, [%3], #8 \n" // store 8 pixels V.
@@ -2537,8 +3122,9 @@
const uint8_t* src_bgra_1 = src_bgra + src_stride_bgra;
asm volatile (
RGBTOUV_SETUP_REG
- "1: \n"
+ "1: \n"
"ld4 {v0.16b,v1.16b,v2.16b,v3.16b}, [%0], #64 \n" // load 16 pixels.
+ "subs %w4, %w4, #16 \n" // 16 processed per loop.
"uaddlp v0.8h, v3.16b \n" // B 16 bytes -> 8 shorts.
"prfm pldl1keep, [%0, 448] \n"
"uaddlp v3.8h, v2.16b \n" // G 16 bytes -> 8 shorts.
@@ -2549,11 +3135,10 @@
"uadalp v3.8h, v6.16b \n" // G 16 bytes -> 8 shorts.
"uadalp v2.8h, v5.16b \n" // R 16 bytes -> 8 shorts.
- "urshr v0.8h, v0.8h, #1 \n" // 2x average
- "urshr v1.8h, v3.8h, #1 \n"
- "urshr v2.8h, v2.8h, #1 \n"
+ "urshr v0.8h, v0.8h, #2 \n" // average of 4
+ "urshr v1.8h, v3.8h, #2 \n"
+ "urshr v2.8h, v2.8h, #2 \n"
- "subs %w4, %w4, #16 \n" // 16 processed per loop.
RGBTOUV(v0.8h, v1.8h, v2.8h)
"st1 {v0.8b}, [%2], #8 \n" // store 8 pixels U.
"st1 {v1.8b}, [%3], #8 \n" // store 8 pixels V.
@@ -2577,8 +3162,9 @@
const uint8_t* src_abgr_1 = src_abgr + src_stride_abgr;
asm volatile (
RGBTOUV_SETUP_REG
- "1: \n"
+ "1: \n"
"ld4 {v0.16b,v1.16b,v2.16b,v3.16b}, [%0], #64 \n" // load 16 pixels.
+ "subs %w4, %w4, #16 \n" // 16 processed per loop.
"uaddlp v3.8h, v2.16b \n" // B 16 bytes -> 8 shorts.
"prfm pldl1keep, [%0, 448] \n"
"uaddlp v2.8h, v1.16b \n" // G 16 bytes -> 8 shorts.
@@ -2589,11 +3175,10 @@
"uadalp v2.8h, v5.16b \n" // G 16 bytes -> 8 shorts.
"uadalp v1.8h, v4.16b \n" // R 16 bytes -> 8 shorts.
- "urshr v0.8h, v3.8h, #1 \n" // 2x average
- "urshr v2.8h, v2.8h, #1 \n"
- "urshr v1.8h, v1.8h, #1 \n"
+ "urshr v0.8h, v3.8h, #2 \n" // average of 4
+ "urshr v2.8h, v2.8h, #2 \n"
+ "urshr v1.8h, v1.8h, #2 \n"
- "subs %w4, %w4, #16 \n" // 16 processed per loop.
RGBTOUV(v0.8h, v2.8h, v1.8h)
"st1 {v0.8b}, [%2], #8 \n" // store 8 pixels U.
"st1 {v1.8b}, [%3], #8 \n" // store 8 pixels V.
@@ -2617,8 +3202,9 @@
const uint8_t* src_rgba_1 = src_rgba + src_stride_rgba;
asm volatile (
RGBTOUV_SETUP_REG
- "1: \n"
+ "1: \n"
"ld4 {v0.16b,v1.16b,v2.16b,v3.16b}, [%0], #64 \n" // load 16 pixels.
+ "subs %w4, %w4, #16 \n" // 16 processed per loop.
"uaddlp v0.8h, v1.16b \n" // B 16 bytes -> 8 shorts.
"prfm pldl1keep, [%0, 448] \n"
"uaddlp v1.8h, v2.16b \n" // G 16 bytes -> 8 shorts.
@@ -2629,11 +3215,10 @@
"uadalp v1.8h, v6.16b \n" // G 16 bytes -> 8 shorts.
"uadalp v2.8h, v7.16b \n" // R 16 bytes -> 8 shorts.
- "urshr v0.8h, v0.8h, #1 \n" // 2x average
- "urshr v1.8h, v1.8h, #1 \n"
- "urshr v2.8h, v2.8h, #1 \n"
+ "urshr v0.8h, v0.8h, #2 \n" // average of 4
+ "urshr v1.8h, v1.8h, #2 \n"
+ "urshr v2.8h, v2.8h, #2 \n"
- "subs %w4, %w4, #16 \n" // 16 processed per loop.
RGBTOUV(v0.8h, v1.8h, v2.8h)
"st1 {v0.8b}, [%2], #8 \n" // store 8 pixels U.
"st1 {v1.8b}, [%3], #8 \n" // store 8 pixels V.
@@ -2657,8 +3242,9 @@
const uint8_t* src_rgb24_1 = src_rgb24 + src_stride_rgb24;
asm volatile (
RGBTOUV_SETUP_REG
- "1: \n"
+ "1: \n"
"ld3 {v0.16b,v1.16b,v2.16b}, [%0], #48 \n" // load 16 pixels.
+ "subs %w4, %w4, #16 \n" // 16 processed per loop.
"uaddlp v0.8h, v0.16b \n" // B 16 bytes -> 8 shorts.
"prfm pldl1keep, [%0, 448] \n"
"uaddlp v1.8h, v1.16b \n" // G 16 bytes -> 8 shorts.
@@ -2669,11 +3255,10 @@
"uadalp v1.8h, v5.16b \n" // G 16 bytes -> 8 shorts.
"uadalp v2.8h, v6.16b \n" // R 16 bytes -> 8 shorts.
- "urshr v0.8h, v0.8h, #1 \n" // 2x average
- "urshr v1.8h, v1.8h, #1 \n"
- "urshr v2.8h, v2.8h, #1 \n"
+ "urshr v0.8h, v0.8h, #2 \n" // average of 4
+ "urshr v1.8h, v1.8h, #2 \n"
+ "urshr v2.8h, v2.8h, #2 \n"
- "subs %w4, %w4, #16 \n" // 16 processed per loop.
RGBTOUV(v0.8h, v1.8h, v2.8h)
"st1 {v0.8b}, [%2], #8 \n" // store 8 pixels U.
"st1 {v1.8b}, [%3], #8 \n" // store 8 pixels V.
@@ -2697,8 +3282,9 @@
const uint8_t* src_raw_1 = src_raw + src_stride_raw;
asm volatile (
RGBTOUV_SETUP_REG
- "1: \n"
+ "1: \n"
"ld3 {v0.16b,v1.16b,v2.16b}, [%0], #48 \n" // load 16 RAW pixels.
+ "subs %w4, %w4, #16 \n" // 16 processed per loop.
"uaddlp v2.8h, v2.16b \n" // B 16 bytes -> 8 shorts.
"prfm pldl1keep, [%0, 448] \n"
"uaddlp v1.8h, v1.16b \n" // G 16 bytes -> 8 shorts.
@@ -2709,11 +3295,10 @@
"uadalp v1.8h, v5.16b \n" // G 16 bytes -> 8 shorts.
"uadalp v0.8h, v4.16b \n" // R 16 bytes -> 8 shorts.
- "urshr v2.8h, v2.8h, #1 \n" // 2x average
- "urshr v1.8h, v1.8h, #1 \n"
- "urshr v0.8h, v0.8h, #1 \n"
+ "urshr v2.8h, v2.8h, #2 \n" // average of 4
+ "urshr v1.8h, v1.8h, #2 \n"
+ "urshr v0.8h, v0.8h, #2 \n"
- "subs %w4, %w4, #16 \n" // 16 processed per loop.
RGBTOUV(v2.8h, v1.8h, v0.8h)
"st1 {v0.8b}, [%2], #8 \n" // store 8 pixels U.
"st1 {v1.8b}, [%3], #8 \n" // store 8 pixels V.
@@ -2738,40 +3323,26 @@
const uint8_t* src_rgb565_1 = src_rgb565 + src_stride_rgb565;
asm volatile(
RGBTOUV_SETUP_REG
- "1: \n"
- "ld1 {v0.16b}, [%0], #16 \n" // load 8 RGB565 pixels.
- RGB565TOARGB
- "uaddlp v16.4h, v0.8b \n" // B 8 bytes -> 4 shorts.
- "prfm pldl1keep, [%0, 448] \n"
- "uaddlp v17.4h, v1.8b \n" // G 8 bytes -> 4 shorts.
- "uaddlp v18.4h, v2.8b \n" // R 8 bytes -> 4 shorts.
- "ld1 {v0.16b}, [%0], #16 \n" // next 8 RGB565 pixels.
- RGB565TOARGB
- "uaddlp v26.4h, v0.8b \n" // B 8 bytes -> 4 shorts.
- "uaddlp v27.4h, v1.8b \n" // G 8 bytes -> 4 shorts.
- "uaddlp v28.4h, v2.8b \n" // R 8 bytes -> 4 shorts.
-
- "ld1 {v0.16b}, [%1], #16 \n" // load 8 RGB565 pixels.
- RGB565TOARGB
- "uadalp v16.4h, v0.8b \n" // B 8 bytes -> 4 shorts.
- "prfm pldl1keep, [%1, 448] \n"
- "uadalp v17.4h, v1.8b \n" // G 8 bytes -> 4 shorts.
- "uadalp v18.4h, v2.8b \n" // R 8 bytes -> 4 shorts.
- "ld1 {v0.16b}, [%1], #16 \n" // next 8 RGB565 pixels.
- RGB565TOARGB
- "uadalp v26.4h, v0.8b \n" // B 8 bytes -> 4 shorts.
- "uadalp v27.4h, v1.8b \n" // G 8 bytes -> 4 shorts.
- "uadalp v28.4h, v2.8b \n" // R 8 bytes -> 4 shorts.
-
- "ins v16.D[1], v26.D[0] \n"
- "ins v17.D[1], v27.D[0] \n"
- "ins v18.D[1], v28.D[0] \n"
-
- "urshr v0.8h, v16.8h, #1 \n" // 2x average
- "urshr v1.8h, v17.8h, #1 \n"
- "urshr v2.8h, v18.8h, #1 \n"
-
+ "1: \n"
+ "ldp q0, q4, [%0], #32 \n" // load 16 RGB565 pixels.
"subs %w4, %w4, #16 \n" // 16 processed per loop.
+ RGB565TOARGB
+ "uaddlp v16.8h, v0.16b \n" // B 16 bytes -> 8 shorts.
+ "prfm pldl1keep, [%0, 448] \n"
+ "uaddlp v17.8h, v1.16b \n" // G 16 bytes -> 8 shorts.
+ "uaddlp v18.8h, v2.16b \n" // R 16 bytes -> 8 shorts.
+
+ "ldp q0, q4, [%1], #32 \n" // load 16 RGB565 pixels.
+ RGB565TOARGB
+ "uadalp v16.8h, v0.16b \n" // B 16 bytes -> 8 shorts.
+ "prfm pldl1keep, [%1, 448] \n"
+ "uadalp v17.8h, v1.16b \n" // G 16 bytes -> 8 shorts.
+ "uadalp v18.8h, v2.16b \n" // R 16 bytes -> 8 shorts.
+
+ "urshr v0.8h, v16.8h, #2 \n" // average of 4
+ "urshr v1.8h, v17.8h, #2 \n"
+ "urshr v2.8h, v18.8h, #2 \n"
+
RGBTOUV(v0.8h, v1.8h, v2.8h)
"st1 {v0.8b}, [%2], #8 \n" // store 8 pixels U.
"st1 {v1.8b}, [%3], #8 \n" // store 8 pixels V.
@@ -2796,40 +3367,26 @@
const uint8_t* src_argb1555_1 = src_argb1555 + src_stride_argb1555;
asm volatile(
RGBTOUV_SETUP_REG
- "1: \n"
- "ld1 {v0.16b}, [%0], #16 \n" // load 8 ARGB1555 pixels.
- RGB555TOARGB
- "uaddlp v16.4h, v0.8b \n" // B 8 bytes -> 4 shorts.
- "prfm pldl1keep, [%0, 448] \n"
- "uaddlp v17.4h, v1.8b \n" // G 8 bytes -> 4 shorts.
- "uaddlp v18.4h, v2.8b \n" // R 8 bytes -> 4 shorts.
- "ld1 {v0.16b}, [%0], #16 \n" // next 8 ARGB1555 pixels.
- RGB555TOARGB
- "uaddlp v26.4h, v0.8b \n" // B 8 bytes -> 4 shorts.
- "uaddlp v27.4h, v1.8b \n" // G 8 bytes -> 4 shorts.
- "uaddlp v28.4h, v2.8b \n" // R 8 bytes -> 4 shorts.
-
- "ld1 {v0.16b}, [%1], #16 \n" // load 8 ARGB1555 pixels.
- RGB555TOARGB
- "uadalp v16.4h, v0.8b \n" // B 8 bytes -> 4 shorts.
- "prfm pldl1keep, [%1, 448] \n"
- "uadalp v17.4h, v1.8b \n" // G 8 bytes -> 4 shorts.
- "uadalp v18.4h, v2.8b \n" // R 8 bytes -> 4 shorts.
- "ld1 {v0.16b}, [%1], #16 \n" // next 8 ARGB1555 pixels.
- RGB555TOARGB
- "uadalp v26.4h, v0.8b \n" // B 8 bytes -> 4 shorts.
- "uadalp v27.4h, v1.8b \n" // G 8 bytes -> 4 shorts.
- "uadalp v28.4h, v2.8b \n" // R 8 bytes -> 4 shorts.
-
- "ins v16.D[1], v26.D[0] \n"
- "ins v17.D[1], v27.D[0] \n"
- "ins v18.D[1], v28.D[0] \n"
-
- "urshr v0.8h, v16.8h, #1 \n" // 2x average
- "urshr v1.8h, v17.8h, #1 \n"
- "urshr v2.8h, v18.8h, #1 \n"
-
+ "1: \n"
+ "ldp q0, q3, [%0], #32 \n" // load 16 ARGB1555 pixels.
"subs %w4, %w4, #16 \n" // 16 processed per loop.
+ RGB555TOARGB
+ "uaddlp v16.8h, v0.16b \n" // B 16 bytes -> 8 shorts.
+ "prfm pldl1keep, [%0, 448] \n"
+ "uaddlp v17.8h, v1.16b \n" // G 16 bytes -> 8 shorts.
+ "uaddlp v18.8h, v2.16b \n" // R 16 bytes -> 8 shorts.
+
+ "ldp q0, q3, [%1], #32 \n" // load 16 ARGB1555 pixels.
+ RGB555TOARGB
+ "uadalp v16.8h, v0.16b \n" // B 16 bytes -> 8 shorts.
+ "prfm pldl1keep, [%1, 448] \n"
+ "uadalp v17.8h, v1.16b \n" // G 16 bytes -> 8 shorts.
+ "uadalp v18.8h, v2.16b \n" // R 16 bytes -> 8 shorts.
+
+ "urshr v0.8h, v16.8h, #2 \n" // average of 4
+ "urshr v1.8h, v17.8h, #2 \n"
+ "urshr v2.8h, v18.8h, #2 \n"
+
RGBTOUV(v0.8h, v1.8h, v2.8h)
"st1 {v0.8b}, [%2], #8 \n" // store 8 pixels U.
"st1 {v1.8b}, [%3], #8 \n" // store 8 pixels V.
@@ -2842,7 +3399,7 @@
:
: "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v16", "v17",
"v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27",
- "v28");
+ "v28", "v29");
}
// 16x2 pixels -> 8x1. width is number of argb pixels. e.g. 16.
@@ -2854,40 +3411,26 @@
const uint8_t* src_argb4444_1 = src_argb4444 + src_stride_argb4444;
asm volatile(
RGBTOUV_SETUP_REG // sets v20-v25
- "1: \n"
- "ld1 {v0.16b}, [%0], #16 \n" // load 8 ARGB4444 pixels.
- ARGB4444TOARGB
- "uaddlp v16.4h, v0.8b \n" // B 8 bytes -> 4 shorts.
- "prfm pldl1keep, [%0, 448] \n"
- "uaddlp v17.4h, v1.8b \n" // G 8 bytes -> 4 shorts.
- "uaddlp v18.4h, v2.8b \n" // R 8 bytes -> 4 shorts.
- "ld1 {v0.16b}, [%0], #16 \n" // next 8 ARGB4444 pixels.
- ARGB4444TOARGB
- "uaddlp v26.4h, v0.8b \n" // B 8 bytes -> 4 shorts.
- "uaddlp v27.4h, v1.8b \n" // G 8 bytes -> 4 shorts.
- "uaddlp v28.4h, v2.8b \n" // R 8 bytes -> 4 shorts.
-
- "ld1 {v0.16b}, [%1], #16 \n" // load 8 ARGB4444 pixels.
- ARGB4444TOARGB
- "uadalp v16.4h, v0.8b \n" // B 8 bytes -> 4 shorts.
- "prfm pldl1keep, [%1, 448] \n"
- "uadalp v17.4h, v1.8b \n" // G 8 bytes -> 4 shorts.
- "uadalp v18.4h, v2.8b \n" // R 8 bytes -> 4 shorts.
- "ld1 {v0.16b}, [%1], #16 \n" // next 8 ARGB4444 pixels.
- ARGB4444TOARGB
- "uadalp v26.4h, v0.8b \n" // B 8 bytes -> 4 shorts.
- "uadalp v27.4h, v1.8b \n" // G 8 bytes -> 4 shorts.
- "uadalp v28.4h, v2.8b \n" // R 8 bytes -> 4 shorts.
-
- "ins v16.D[1], v26.D[0] \n"
- "ins v17.D[1], v27.D[0] \n"
- "ins v18.D[1], v28.D[0] \n"
-
- "urshr v0.8h, v16.8h, #1 \n" // 2x average
- "urshr v1.8h, v17.8h, #1 \n"
- "urshr v2.8h, v18.8h, #1 \n"
-
+ "1: \n"
+ "ldp q0, q3, [%0], #32 \n" // load 16 ARGB4444 pixels.
"subs %w4, %w4, #16 \n" // 16 processed per loop.
+ ARGB4444TORGB
+ "uaddlp v16.8h, v0.16b \n" // B 16 bytes -> 8 shorts.
+ "prfm pldl1keep, [%0, 448] \n"
+ "uaddlp v17.8h, v1.16b \n" // G 16 bytes -> 8 shorts.
+ "uaddlp v18.8h, v2.16b \n" // R 16 bytes -> 8 shorts.
+
+ "ldp q0, q3, [%1], #32 \n" // load 16 ARGB4444 pixels.
+ ARGB4444TORGB
+ "uadalp v16.8h, v0.16b \n" // B 16 bytes -> 8 shorts.
+ "prfm pldl1keep, [%1, 448] \n"
+ "uadalp v17.8h, v1.16b \n" // G 16 bytes -> 8 shorts.
+ "uadalp v18.8h, v2.16b \n" // R 16 bytes -> 8 shorts.
+
+ "urshr v0.8h, v16.8h, #2 \n" // average of 4
+ "urshr v1.8h, v17.8h, #2 \n"
+ "urshr v2.8h, v18.8h, #2 \n"
+
RGBTOUV(v0.8h, v1.8h, v2.8h)
"st1 {v0.8b}, [%2], #8 \n" // store 8 pixels U.
"st1 {v1.8b}, [%3], #8 \n" // store 8 pixels V.
@@ -2907,21 +3450,26 @@
void RGB565ToYRow_NEON(const uint8_t* src_rgb565, uint8_t* dst_y, int width) {
asm volatile(
- "movi v24.8b, #25 \n" // B * 0.1016 coefficient
- "movi v25.8b, #129 \n" // G * 0.5078 coefficient
- "movi v26.8b, #66 \n" // R * 0.2578 coefficient
- "movi v27.8b, #16 \n" // Add 16 constant
- "1: \n"
- "ld1 {v0.16b}, [%0], #16 \n" // load 8 RGB565 pixels.
- "subs %w2, %w2, #8 \n" // 8 processed per loop.
+ "movi v24.16b, #25 \n" // B * 0.1016 coefficient
+ "movi v25.16b, #129 \n" // G * 0.5078 coefficient
+ "movi v26.16b, #66 \n" // R * 0.2578 coefficient
+ "movi v27.16b, #16 \n" // Add 16 constant
+ "1: \n"
+ "ldp q0, q4, [%0], #32 \n" // load 16 RGB565 pixels.
+ "subs %w2, %w2, #16 \n" // 16 processed per loop.
RGB565TOARGB
"umull v3.8h, v0.8b, v24.8b \n" // B
+ "umull2 v4.8h, v0.16b, v24.16b \n" // B
"prfm pldl1keep, [%0, 448] \n"
"umlal v3.8h, v1.8b, v25.8b \n" // G
+ "umlal2 v4.8h, v1.16b, v25.16b \n" // G
"umlal v3.8h, v2.8b, v26.8b \n" // R
+ "umlal2 v4.8h, v2.16b, v26.16b \n" // R
"uqrshrn v0.8b, v3.8h, #8 \n" // 16 bit to 8 bit Y
+ "uqrshrn v1.8b, v4.8h, #8 \n" // 16 bit to 8 bit Y
"uqadd v0.8b, v0.8b, v27.8b \n"
- "st1 {v0.8b}, [%1], #8 \n" // store 8 pixels Y.
+ "uqadd v1.8b, v1.8b, v27.8b \n"
+ "stp d0, d1, [%1], #16 \n" // store 8 pixels Y.
"b.gt 1b \n"
: "+r"(src_rgb565), // %0
"+r"(dst_y), // %1
@@ -2935,48 +3483,59 @@
uint8_t* dst_y,
int width) {
asm volatile(
- "movi v4.8b, #25 \n" // B * 0.1016 coefficient
- "movi v5.8b, #129 \n" // G * 0.5078 coefficient
- "movi v6.8b, #66 \n" // R * 0.2578 coefficient
- "movi v7.8b, #16 \n" // Add 16 constant
- "1: \n"
- "ld1 {v0.16b}, [%0], #16 \n" // load 8 ARGB1555 pixels.
- "subs %w2, %w2, #8 \n" // 8 processed per loop.
- ARGB1555TOARGB
- "umull v3.8h, v0.8b, v4.8b \n" // B
+ "movi v4.16b, #25 \n" // B * 0.1016 coefficient
+ "movi v5.16b, #129 \n" // G * 0.5078 coefficient
+ "movi v6.16b, #66 \n" // R * 0.2578 coefficient
+ "movi v7.16b, #16 \n" // Add 16 constant
+ "1: \n"
+ "ldp q0, q3, [%0], #32 \n" // load 16 ARGB1555
+ // pixels.
+ "subs %w2, %w2, #16 \n" // 16 processed per loop.
+ RGB555TOARGB
+ "umull v16.8h, v0.8b, v4.8b \n" // B
+ "umull2 v17.8h, v0.16b, v4.16b \n" // B
"prfm pldl1keep, [%0, 448] \n"
- "umlal v3.8h, v1.8b, v5.8b \n" // G
- "umlal v3.8h, v2.8b, v6.8b \n" // R
- "uqrshrn v0.8b, v3.8h, #8 \n" // 16 bit to 8 bit Y
- "uqadd v0.8b, v0.8b, v7.8b \n"
- "st1 {v0.8b}, [%1], #8 \n" // store 8 pixels Y.
+ "umlal v16.8h, v1.8b, v5.8b \n" // G
+ "umlal2 v17.8h, v1.16b, v5.16b \n" // G
+ "umlal v16.8h, v2.8b, v6.8b \n" // R
+ "umlal2 v17.8h, v2.16b, v6.16b \n" // R
+ "uqrshrn v0.8b, v16.8h, #8 \n" // 16 bit to 8 bit Y
+ "uqrshrn2 v0.16b, v17.8h, #8 \n" // 16 bit to 8 bit Y
+ "uqadd v0.16b, v0.16b, v7.16b \n"
+ "str q0, [%1], #16 \n" // store pixels Y.
"b.gt 1b \n"
: "+r"(src_argb1555), // %0
"+r"(dst_y), // %1
"+r"(width) // %2
:
- : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7");
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16",
+ "v17", "v29");
}
void ARGB4444ToYRow_NEON(const uint8_t* src_argb4444,
uint8_t* dst_y,
int width) {
asm volatile(
- "movi v24.8b, #25 \n" // B * 0.1016 coefficient
- "movi v25.8b, #129 \n" // G * 0.5078 coefficient
- "movi v26.8b, #66 \n" // R * 0.2578 coefficient
- "movi v27.8b, #16 \n" // Add 16 constant
- "1: \n"
- "ld1 {v0.16b}, [%0], #16 \n" // load 8 ARGB4444 pixels.
- "subs %w2, %w2, #8 \n" // 8 processed per loop.
- ARGB4444TOARGB
- "umull v3.8h, v0.8b, v24.8b \n" // B
+ "movi v24.16b, #25 \n" // B * 0.1016 coefficient
+ "movi v25.16b, #129 \n" // G * 0.5078 coefficient
+ "movi v26.16b, #66 \n" // R * 0.2578 coefficient
+ "movi v27.16b, #16 \n" // Add 16 constant
+ "1: \n"
+ "ldp q0, q3, [%0], #32 \n" // load 16 ARGB4444
+ // pixels.
+ "subs %w2, %w2, #16 \n" // 16 processed per loop.
+ ARGB4444TORGB
+ "umull v16.8h, v0.8b, v24.8b \n" // B
+ "umull2 v17.8h, v0.16b, v24.16b \n" // B
"prfm pldl1keep, [%0, 448] \n"
- "umlal v3.8h, v1.8b, v25.8b \n" // G
- "umlal v3.8h, v2.8b, v26.8b \n" // R
- "uqrshrn v0.8b, v3.8h, #8 \n" // 16 bit to 8 bit Y
- "uqadd v0.8b, v0.8b, v27.8b \n"
- "st1 {v0.8b}, [%1], #8 \n" // store 8 pixels Y.
+ "umlal v16.8h, v1.8b, v25.8b \n" // G
+ "umlal2 v17.8h, v1.16b, v25.16b \n" // G
+ "umlal v16.8h, v2.8b, v26.8b \n" // R
+ "umlal2 v17.8h, v2.16b, v26.16b \n" // R
+ "uqrshrn v0.8b, v16.8h, #8 \n" // 16 bit to 8 bit Y
+ "uqrshrn2 v0.16b, v17.8h, #8 \n" // 16 bit to 8 bit Y
+ "uqadd v0.16b, v0.16b, v27.16b \n"
+ "str q0, [%1], #16 \n" // store 8 pixels Y.
"b.gt 1b \n"
: "+r"(src_argb4444), // %0
"+r"(dst_y), // %1
@@ -2991,17 +3550,17 @@
};
// ARGB expects first 3 values to contain RGB and 4th value is ignored.
-void ARGBToYMatrixRow_NEON(const uint8_t* src_argb,
- uint8_t* dst_y,
- int width,
- const struct RgbConstants* rgbconstants) {
+static void ARGBToYMatrixRow_NEON(const uint8_t* src_argb,
+ uint8_t* dst_y,
+ int width,
+ const struct RgbConstants* rgbconstants) {
asm volatile(
"ldr d0, [%3] \n" // load rgbconstants
"dup v6.16b, v0.b[0] \n"
"dup v7.16b, v0.b[1] \n"
"dup v16.16b, v0.b[2] \n"
"dup v17.8h, v0.h[2] \n"
- "1: \n"
+ "1: \n"
"ld4 {v2.16b,v3.16b,v4.16b,v5.16b}, [%0], #64 \n" // load 16
// pixels.
"subs %w2, %w2, #16 \n" // 16 processed per loop.
@@ -3024,14 +3583,52 @@
"v17");
}
+static void ARGBToYMatrixRow_NEON_DotProd(
+ const uint8_t* src_argb,
+ uint8_t* dst_y,
+ int width,
+ const struct RgbConstants* rgbconstants) {
+ asm volatile(
+ "ldr d0, [%3] \n" // load rgbconstants
+ "dup v16.4s, v0.s[0] \n"
+ "dup v17.8h, v0.h[2] \n"
+ "1: \n"
+ "ld1 {v4.16b, v5.16b, v6.16b, v7.16b}, [%0], #64 \n" // load 16
+ // pixels.
+ "subs %w2, %w2, #16 \n" // 16 processed per loop.
+ "movi v0.16b, #0 \n"
+ "movi v1.16b, #0 \n"
+ "movi v2.16b, #0 \n"
+ "movi v3.16b, #0 \n"
+ "udot v0.4s, v4.16b, v16.16b \n"
+ "udot v1.4s, v5.16b, v16.16b \n"
+ "udot v2.4s, v6.16b, v16.16b \n"
+ "udot v3.4s, v7.16b, v16.16b \n"
+ "uzp1 v0.8h, v0.8h, v1.8h \n"
+ "uzp1 v1.8h, v2.8h, v3.8h \n"
+ "addhn v0.8b, v0.8h, v17.8h \n"
+ "addhn v1.8b, v1.8h, v17.8h \n"
+ "st1 {v0.8b, v1.8b}, [%1], #16 \n" // store 16 pixels Y.
+ "b.gt 1b \n"
+ : "+r"(src_argb), // %0
+ "+r"(dst_y), // %1
+ "+r"(width) // %2
+ : "r"(rgbconstants) // %3
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16",
+ "v17");
+}
+
// RGB to JPeg coefficients
// B * 0.1140 coefficient = 29
// G * 0.5870 coefficient = 150
// R * 0.2990 coefficient = 77
-// Add 0.5 = 0x80
-static const struct RgbConstants kRgb24JPEGConstants = {{29, 150, 77, 0}, 128};
+// Add 0.5
+static const struct RgbConstants kRgb24JPEGConstants = {{29, 150, 77, 0},
+ 0x0080};
+static const struct RgbConstants kRgb24JPEGDotProdConstants = {{0, 29, 150, 77},
+ 0x0080};
-static const struct RgbConstants kRawJPEGConstants = {{77, 150, 29, 0}, 128};
+static const struct RgbConstants kRawJPEGConstants = {{77, 150, 29, 0}, 0x0080};
// RGB to BT.601 coefficients
// B * 0.1016 coefficient = 25
@@ -3041,8 +3638,12 @@
static const struct RgbConstants kRgb24I601Constants = {{25, 129, 66, 0},
0x1080};
+static const struct RgbConstants kRgb24I601DotProdConstants = {{0, 25, 129, 66},
+ 0x1080};
static const struct RgbConstants kRawI601Constants = {{66, 129, 25, 0}, 0x1080};
+static const struct RgbConstants kRawI601DotProdConstants = {{0, 66, 129, 25},
+ 0x1080};
void ARGBToYRow_NEON(const uint8_t* src_argb, uint8_t* dst_y, int width) {
ARGBToYMatrixRow_NEON(src_argb, dst_y, width, &kRgb24I601Constants);
@@ -3060,19 +3661,43 @@
ARGBToYMatrixRow_NEON(src_abgr, dst_yj, width, &kRawJPEGConstants);
}
+void ARGBToYRow_NEON_DotProd(const uint8_t* src_argb,
+ uint8_t* dst_y,
+ int width) {
+ ARGBToYMatrixRow_NEON_DotProd(src_argb, dst_y, width, &kRgb24I601Constants);
+}
+
+void ARGBToYJRow_NEON_DotProd(const uint8_t* src_argb,
+ uint8_t* dst_yj,
+ int width) {
+ ARGBToYMatrixRow_NEON_DotProd(src_argb, dst_yj, width, &kRgb24JPEGConstants);
+}
+
+void ABGRToYRow_NEON_DotProd(const uint8_t* src_abgr,
+ uint8_t* dst_y,
+ int width) {
+ ARGBToYMatrixRow_NEON_DotProd(src_abgr, dst_y, width, &kRawI601Constants);
+}
+
+void ABGRToYJRow_NEON_DotProd(const uint8_t* src_abgr,
+ uint8_t* dst_yj,
+ int width) {
+ ARGBToYMatrixRow_NEON_DotProd(src_abgr, dst_yj, width, &kRawJPEGConstants);
+}
+
// RGBA expects first value to be A and ignored, then 3 values to contain RGB.
// Same code as ARGB, except the LD4
-void RGBAToYMatrixRow_NEON(const uint8_t* src_rgba,
- uint8_t* dst_y,
- int width,
- const struct RgbConstants* rgbconstants) {
+static void RGBAToYMatrixRow_NEON(const uint8_t* src_rgba,
+ uint8_t* dst_y,
+ int width,
+ const struct RgbConstants* rgbconstants) {
asm volatile(
"ldr d0, [%3] \n" // load rgbconstants
"dup v6.16b, v0.b[0] \n"
"dup v7.16b, v0.b[1] \n"
"dup v16.16b, v0.b[2] \n"
"dup v17.8h, v0.h[2] \n"
- "1: \n"
+ "1: \n"
"ld4 {v1.16b,v2.16b,v3.16b,v4.16b}, [%0], #64 \n" // load 16
// pixels.
"subs %w2, %w2, #16 \n" // 16 processed per loop.
@@ -3107,17 +3732,44 @@
RGBAToYMatrixRow_NEON(src_bgra, dst_y, width, &kRawI601Constants);
}
-void RGBToYMatrixRow_NEON(const uint8_t* src_rgb,
- uint8_t* dst_y,
- int width,
- const struct RgbConstants* rgbconstants) {
+void RGBAToYRow_NEON_DotProd(const uint8_t* src_rgba,
+ uint8_t* dst_y,
+ int width) {
+ // No need for a separate implementation for RGBA inputs, just permute the
+ // RGB constants.
+ ARGBToYMatrixRow_NEON_DotProd(src_rgba, dst_y, width,
+ &kRgb24I601DotProdConstants);
+}
+
+void RGBAToYJRow_NEON_DotProd(const uint8_t* src_rgba,
+ uint8_t* dst_yj,
+ int width) {
+ // No need for a separate implementation for RGBA inputs, just permute the
+ // RGB constants.
+ ARGBToYMatrixRow_NEON_DotProd(src_rgba, dst_yj, width,
+ &kRgb24JPEGDotProdConstants);
+}
+
+void BGRAToYRow_NEON_DotProd(const uint8_t* src_bgra,
+ uint8_t* dst_y,
+ int width) {
+ // No need for a separate implementation for RGBA inputs, just permute the
+ // RGB constants.
+ ARGBToYMatrixRow_NEON_DotProd(src_bgra, dst_y, width,
+ &kRawI601DotProdConstants);
+}
+
+static void RGBToYMatrixRow_NEON(const uint8_t* src_rgb,
+ uint8_t* dst_y,
+ int width,
+ const struct RgbConstants* rgbconstants) {
asm volatile(
"ldr d0, [%3] \n" // load rgbconstants
"dup v5.16b, v0.b[0] \n"
"dup v6.16b, v0.b[1] \n"
"dup v7.16b, v0.b[2] \n"
"dup v16.8h, v0.h[2] \n"
- "1: \n"
+ "1: \n"
"ld3 {v2.16b,v3.16b,v4.16b}, [%0], #48 \n" // load 16 pixels.
"subs %w2, %w2, #16 \n" // 16 processed per loop.
"umull v0.8h, v2.8b, v5.8b \n" // B
@@ -3172,7 +3824,7 @@
"dup v5.16b, %w4 \n"
"dup v4.16b, %w5 \n"
// General purpose row blend.
- "1: \n"
+ "1: \n"
"ld1 {v0.16b}, [%1], #16 \n"
"ld1 {v1.16b}, [%2], #16 \n"
"subs %w3, %w3, #16 \n"
@@ -3189,7 +3841,7 @@
"b 99f \n"
// Blend 50 / 50.
- "50: \n"
+ "50: \n"
"ld1 {v0.16b}, [%1], #16 \n"
"ld1 {v1.16b}, [%2], #16 \n"
"subs %w3, %w3, #16 \n"
@@ -3201,14 +3853,14 @@
"b 99f \n"
// Blend 100 / 0 - Copy row unchanged.
- "100: \n"
+ "100: \n"
"ld1 {v0.16b}, [%1], #16 \n"
"subs %w3, %w3, #16 \n"
"prfm pldl1keep, [%1, 448] \n"
"st1 {v0.16b}, [%0], #16 \n"
"b.gt 100b \n"
- "99: \n"
+ "99: \n"
: "+r"(dst_ptr), // %0
"+r"(src_ptr), // %1
"+r"(src_ptr1), // %2
@@ -3238,7 +3890,7 @@
"dup v5.8h, %w4 \n"
"dup v4.8h, %w5 \n"
// General purpose row blend.
- "1: \n"
+ "1: \n"
"ld1 {v0.8h}, [%1], #16 \n"
"ld1 {v1.8h}, [%2], #16 \n"
"subs %w3, %w3, #8 \n"
@@ -3255,7 +3907,7 @@
"b 99f \n"
// Blend 50 / 50.
- "50: \n"
+ "50: \n"
"ld1 {v0.8h}, [%1], #16 \n"
"ld1 {v1.8h}, [%2], #16 \n"
"subs %w3, %w3, #8 \n"
@@ -3267,14 +3919,14 @@
"b 99f \n"
// Blend 100 / 0 - Copy row unchanged.
- "100: \n"
+ "100: \n"
"ld1 {v0.8h}, [%1], #16 \n"
"subs %w3, %w3, #8 \n"
"prfm pldl1keep, [%1, 448] \n"
"st1 {v0.8h}, [%0], #16 \n"
"b.gt 100b \n"
- "99: \n"
+ "99: \n"
: "+r"(dst_ptr), // %0
"+r"(src_ptr), // %1
"+r"(src_ptr1), // %2
@@ -3311,7 +3963,7 @@
"dup v5.8h, %w4 \n"
"dup v4.8h, %w5 \n"
// General purpose row blend.
- "1: \n"
+ "1: \n"
"ld1 {v0.8h}, [%1], #16 \n"
"ld1 {v1.8h}, [%2], #16 \n"
"subs %w3, %w3, #8 \n"
@@ -3330,7 +3982,7 @@
"b 99f \n"
// Blend 50 / 50.
- "50: \n"
+ "50: \n"
"ld1 {v0.8h}, [%1], #16 \n"
"ld1 {v1.8h}, [%2], #16 \n"
"subs %w3, %w3, #8 \n"
@@ -3344,7 +3996,7 @@
"b 99f \n"
// Blend 100 / 0 - Copy row unchanged.
- "100: \n"
+ "100: \n"
"ldr q0, [%1], #16 \n"
"ushl v0.8h, v0.8h, v2.8h \n" // shr = v2 is negative
"prfm pldl1keep, [%1, 448] \n"
@@ -3353,7 +4005,7 @@
"str d0, [%0], #8 \n" // store 8 pixels
"b.gt 100b \n"
- "99: \n"
+ "99: \n"
: "+r"(dst_ptr), // %0
"+r"(src_ptr), // %1
"+r"(src_ptr1), // %2
@@ -3373,7 +4025,7 @@
"subs %w3, %w3, #8 \n"
"b.lt 89f \n"
// Blend 8 pixels.
- "8: \n"
+ "8: \n"
"ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // load 8 ARGB0
"ld4 {v4.8b,v5.8b,v6.8b,v7.8b}, [%1], #32 \n" // load 8 ARGB1
"subs %w3, %w3, #8 \n" // 8 processed per loop.
@@ -3396,12 +4048,12 @@
// pixels
"b.ge 8b \n"
- "89: \n"
+ "89: \n"
"adds %w3, %w3, #8-1 \n"
"b.lt 99f \n"
// Blend 1 pixels.
- "1: \n"
+ "1: \n"
"ld4 {v0.b,v1.b,v2.b,v3.b}[0], [%0], #4 \n" // load 1 pixel
// ARGB0.
"ld4 {v4.b,v5.b,v6.b,v7.b}[0], [%1], #4 \n" // load 1 pixel
@@ -3425,7 +4077,7 @@
"st4 {v0.b,v1.b,v2.b,v3.b}[0], [%2], #4 \n" // store 1 pixel.
"b.ge 1b \n"
- "99: \n"
+ "99: \n"
: "+r"(src_argb), // %0
"+r"(src_argb1), // %1
@@ -3444,7 +4096,7 @@
"movi v7.8h, #0x00ff \n" // 255 for rounding up
// Attenuate 8 pixels.
- "1: \n"
+ "1: \n"
"ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // load 8 ARGB
"subs %w2, %w2, #8 \n" // 8 processed per loop.
"umull v4.8h, v0.8b, v3.8b \n" // b * a
@@ -3477,7 +4129,7 @@
"dup v6.8h, %w4 \n" // interval add
// 8 pixel loop.
- "1: \n"
+ "1: \n"
"ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0] \n" // load 8 ARGB.
"subs %w1, %w1, #8 \n" // 8 processed per loop.
"uxtl v0.8h, v0.8b \n" // b (0 .. 255)
@@ -3507,35 +4159,36 @@
}
// Shade 8 pixels at a time by specified value.
-// NOTE vqrdmulh.s16 q10, q10, d0[0] must use a scaler register from 0 to 8.
-// Rounding in vqrdmulh does +1 to high if high bit of low s16 is set.
+// sqrdmulh is a rounding instruction, so +1 if high bit of low half of
+// multiply result is set.
void ARGBShadeRow_NEON(const uint8_t* src_argb,
uint8_t* dst_argb,
int width,
uint32_t value) {
asm volatile(
"dup v0.4s, %w3 \n" // duplicate scale value.
- "zip1 v0.8b, v0.8b, v0.8b \n" // v0.8b aarrggbb.
+ "zip1 v0.16b, v0.16b, v0.16b \n" // v0.16b
+ // aarrggbbaarrggbb.
"ushr v0.8h, v0.8h, #1 \n" // scale / 2.
// 8 pixel loop.
- "1: \n"
- "ld4 {v4.8b,v5.8b,v6.8b,v7.8b}, [%0], #32 \n" // load 8 ARGB
+ "1: \n"
+ "ld1 {v4.8b,v5.8b,v6.8b,v7.8b}, [%0], #32 \n" // load 8 ARGB
"subs %w2, %w2, #8 \n" // 8 processed per loop.
- "uxtl v4.8h, v4.8b \n" // b (0 .. 255)
+ "uxtl v4.8h, v4.8b \n"
"prfm pldl1keep, [%0, 448] \n"
"uxtl v5.8h, v5.8b \n"
"uxtl v6.8h, v6.8b \n"
"uxtl v7.8h, v7.8b \n"
- "sqrdmulh v4.8h, v4.8h, v0.h[0] \n" // b * scale * 2
- "sqrdmulh v5.8h, v5.8h, v0.h[1] \n" // g
- "sqrdmulh v6.8h, v6.8h, v0.h[2] \n" // r
- "sqrdmulh v7.8h, v7.8h, v0.h[3] \n" // a
+ "sqrdmulh v4.8h, v4.8h, v0.8h \n" // argb * scale * 2
+ "sqrdmulh v5.8h, v5.8h, v0.8h \n"
+ "sqrdmulh v6.8h, v6.8h, v0.8h \n"
+ "sqrdmulh v7.8h, v7.8h, v0.8h \n"
"uqxtn v4.8b, v4.8h \n"
"uqxtn v5.8b, v5.8h \n"
"uqxtn v6.8b, v6.8h \n"
"uqxtn v7.8b, v7.8h \n"
- "st4 {v4.8b,v5.8b,v6.8b,v7.8b}, [%1], #32 \n" // store 8 ARGB
+ "st1 {v4.8b,v5.8b,v6.8b,v7.8b}, [%1], #32 \n" // store 8 ARGB
"b.gt 1b \n"
: "+r"(src_argb), // %0
"+r"(dst_argb), // %1
@@ -3552,7 +4205,7 @@
"movi v24.8b, #29 \n" // B * 0.1140 coefficient
"movi v25.8b, #150 \n" // G * 0.5870 coefficient
"movi v26.8b, #77 \n" // R * 0.2990 coefficient
- "1: \n"
+ "1: \n"
"ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // load 8 ARGB
"subs %w2, %w2, #8 \n" // 8 processed per loop.
"umull v4.8h, v0.8b, v24.8b \n" // B
@@ -3560,8 +4213,8 @@
"umlal v4.8h, v1.8b, v25.8b \n" // G
"umlal v4.8h, v2.8b, v26.8b \n" // R
"uqrshrn v0.8b, v4.8h, #8 \n" // 16 bit to 8 bit B
- "orr v1.8b, v0.8b, v0.8b \n" // G
- "orr v2.8b, v0.8b, v0.8b \n" // R
+ "mov v1.8b, v0.8b \n" // G
+ "mov v2.8b, v0.8b \n" // R
"st4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%1], #32 \n" // store 8 pixels.
"b.gt 1b \n"
: "+r"(src_argb), // %0
@@ -3571,6 +4224,38 @@
: "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v24", "v25", "v26");
}
+static const uvec8 kARGBGrayRowCoeffs = {29, 150, 77, 0};
+static const uvec8 kARGBGrayRowIndices = {0, 0, 0, 19, 2, 2, 2, 23,
+ 4, 4, 4, 27, 6, 6, 6, 31};
+
+void ARGBGrayRow_NEON_DotProd(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ int width) {
+ asm volatile(
+ "ld1r {v24.4s}, [%[coeffs]] \n"
+ "ldr q25, [%[indices]] \n"
+ "1: \n"
+ "ldp q1, q3, [%[src]], #32 \n" // load 8 ARGB
+ "subs %w[width], %w[width], #8 \n" // 8 processed per loop
+ "movi v0.4s, #0 \n"
+ "movi v2.4s, #0 \n"
+ "udot v0.4s, v1.16b, v24.16b \n"
+ "udot v2.4s, v3.16b, v24.16b \n"
+ "prfm pldl1keep, [%[src], 448] \n"
+ "uqrshrn v0.8b, v0.8h, #8 \n"
+ "uqrshrn v2.8b, v2.8h, #8 \n"
+ "tbl v0.16b, {v0.16b, v1.16b}, v25.16b \n" // merge in alpha
+ "tbl v1.16b, {v2.16b, v3.16b}, v25.16b \n"
+ "stp q0, q1, [%[dst]], #32 \n" // store 8 pixels
+ "b.gt 1b \n"
+ : [src] "+r"(src_argb), // %[src]
+ [dst] "+r"(dst_argb), // %[dst]
+ [width] "+r"(width) // %[width]
+ : [coeffs] "r"(&kARGBGrayRowCoeffs), // %[coeffs]
+ [indices] "r"(&kARGBGrayRowIndices) // %[indices]
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v24", "v25");
+}
+
// Convert 8 ARGB pixels (32 bytes) to 8 Sepia ARGB pixels.
// b = (r * 35 + g * 68 + b * 17) >> 7
// g = (r * 45 + g * 88 + b * 22) >> 7
@@ -3587,7 +4272,7 @@
"movi v28.8b, #24 \n" // BB coefficient
"movi v29.8b, #98 \n" // BG coefficient
"movi v30.8b, #50 \n" // BR coefficient
- "1: \n"
+ "1: \n"
"ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0] \n" // load 8 ARGB pixels.
"subs %w1, %w1, #8 \n" // 8 processed per loop.
"umull v4.8h, v0.8b, v20.8b \n" // B to Sepia B
@@ -3612,6 +4297,47 @@
"v21", "v22", "v24", "v25", "v26", "v28", "v29", "v30");
}
+static const uvec8 kARGBSepiaRowCoeffs = {17, 68, 35, 0, 22, 88,
+ 45, 0, 24, 98, 50, 0};
+static const uvec8 kARGBSepiaRowAlphaIndices = {3, 7, 11, 15, 19, 23, 27, 31};
+
+void ARGBSepiaRow_NEON_DotProd(uint8_t* dst_argb, int width) {
+ asm volatile(
+ "ld3r {v20.4s, v21.4s, v22.4s}, [%[coeffs]] \n"
+ "ldr d23, [%[indices]] \n"
+ "1: \n"
+ "ldp q0, q1, [%[dst]] \n"
+ "subs %w1, %w1, #8 \n"
+ "movi v2.4s, #0 \n"
+ "movi v3.4s, #0 \n"
+ "movi v4.4s, #0 \n"
+ "movi v5.4s, #0 \n"
+ "movi v6.4s, #0 \n"
+ "movi v7.4s, #0 \n"
+ "udot v2.4s, v0.16b, v20.16b \n"
+ "udot v3.4s, v1.16b, v20.16b \n"
+ "udot v4.4s, v0.16b, v21.16b \n"
+ "udot v5.4s, v1.16b, v21.16b \n"
+ "udot v6.4s, v0.16b, v22.16b \n"
+ "udot v7.4s, v1.16b, v22.16b \n"
+ "prfm pldl1keep, [%[dst], 448] \n"
+ "uzp1 v6.8h, v6.8h, v7.8h \n"
+ "uzp1 v5.8h, v4.8h, v5.8h \n"
+ "uzp1 v4.8h, v2.8h, v3.8h \n"
+ "tbl v3.16b, {v0.16b, v1.16b}, v23.16b \n"
+ "uqshrn v0.8b, v4.8h, #7 \n"
+ "uqshrn v1.8b, v5.8h, #7 \n"
+ "uqshrn v2.8b, v6.8h, #7 \n"
+ "st4 {v0.8b, v1.8b, v2.8b, v3.8b}, [%[dst]], #32 \n"
+ "b.gt 1b \n"
+ : [dst] "+r"(dst_argb), // %[dst]
+ [width] "+r"(width) // %[width]
+ : [coeffs] "r"(&kARGBSepiaRowCoeffs), // %[coeffs]
+ [indices] "r"(&kARGBSepiaRowAlphaIndices) // %[indices]
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v20",
+ "v21", "v22", "v24", "v25", "v26", "v28", "v29", "v30");
+}
+
// Tranform 8 ARGB pixels (32 bytes) with color matrix.
// TODO(fbarchard): Was same as Sepia except matrix is provided. This function
// needs to saturate. Consider doing a non-saturating version.
@@ -3624,7 +4350,7 @@
"sxtl v0.8h, v2.8b \n" // B,G coefficients s16.
"sxtl2 v1.8h, v2.16b \n" // R,A coefficients s16.
- "1: \n"
+ "1: \n"
"ld4 {v16.8b,v17.8b,v18.8b,v19.8b}, [%0], #32 \n" // load 8 ARGB
"subs %w2, %w2, #8 \n" // 8 processed per loop.
"uxtl v16.8h, v16.8b \n" // b (0 .. 255) 16 bit
@@ -3674,7 +4400,62 @@
"v17", "v18", "v19", "v22", "v23", "v24", "v25");
}
-// TODO(fbarchard): fix vqshrun in ARGBMultiplyRow_NEON and reenable.
+void ARGBColorMatrixRow_NEON_I8MM(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ const int8_t* matrix_argb,
+ int width) {
+ asm volatile(
+ "ld1 {v31.16b}, [%[matrix_argb]] \n"
+
+ "1: \n"
+ "ld1 {v0.16b, v1.16b}, [%[src_argb]], #32 \n"
+ "subs %w2, %w2, #8 \n" // 8 processed per loop.
+
+ "movi v16.4s, #0 \n"
+ "movi v17.4s, #0 \n"
+ "movi v18.4s, #0 \n"
+ "movi v19.4s, #0 \n"
+ "movi v20.4s, #0 \n"
+ "movi v21.4s, #0 \n"
+ "movi v22.4s, #0 \n"
+ "movi v23.4s, #0 \n"
+
+ "prfm pldl1keep, [%[src_argb], 448] \n"
+
+ "sudot v16.4s, v31.16b, v0.4b[0] \n"
+ "sudot v17.4s, v31.16b, v0.4b[1] \n"
+ "sudot v18.4s, v31.16b, v0.4b[2] \n"
+ "sudot v19.4s, v31.16b, v0.4b[3] \n"
+ "sudot v20.4s, v31.16b, v1.4b[0] \n"
+ "sudot v21.4s, v31.16b, v1.4b[1] \n"
+ "sudot v22.4s, v31.16b, v1.4b[2] \n"
+ "sudot v23.4s, v31.16b, v1.4b[3] \n"
+
+ "shrn v16.4h, v16.4s, #6 \n"
+ "shrn v18.4h, v18.4s, #6 \n"
+ "shrn v20.4h, v20.4s, #6 \n"
+ "shrn v22.4h, v22.4s, #6 \n"
+ "shrn2 v16.8h, v17.4s, #6 \n"
+ "shrn2 v18.8h, v19.4s, #6 \n"
+ "shrn2 v20.8h, v21.4s, #6 \n"
+ "shrn2 v22.8h, v23.4s, #6 \n"
+
+ "uqxtn v16.8b, v16.8h \n"
+ "uqxtn v18.8b, v18.8h \n"
+ "uqxtn v20.8b, v20.8h \n"
+ "uqxtn v22.8b, v22.8h \n"
+
+ "stp d16, d18, [%[dst_argb]], #16 \n"
+ "stp d20, d22, [%[dst_argb]], #16 \n"
+ "b.gt 1b \n"
+ : [src_argb] "+r"(src_argb), // %[src_argb]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width) // %[width]
+ : [matrix_argb] "r"(matrix_argb) // %[matrix_argb]
+ : "cc", "memory", "v0", "v1", "v16", "v17", "v18", "v19", "v20", "v21",
+ "v22", "v23", "v31");
+}
+
// Multiply 2 rows of ARGB pixels together, 8 pixels at a time.
void ARGBMultiplyRow_NEON(const uint8_t* src_argb,
const uint8_t* src_argb1,
@@ -3682,9 +4463,9 @@
int width) {
asm volatile(
// 8 pixel loop.
- "1: \n"
- "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // load 8 ARGB
- "ld4 {v4.8b,v5.8b,v6.8b,v7.8b}, [%1], #32 \n" // load 8 more
+ "1: \n"
+ "ld1 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // load 8 ARGB
+ "ld1 {v4.8b,v5.8b,v6.8b,v7.8b}, [%1], #32 \n" // load 8 more
"subs %w3, %w3, #8 \n" // 8 processed per loop.
"umull v0.8h, v0.8b, v4.8b \n" // multiply B
"prfm pldl1keep, [%0, 448] \n"
@@ -3696,7 +4477,7 @@
"rshrn v1.8b, v1.8h, #8 \n" // 16 bit to 8 bit G
"rshrn v2.8b, v2.8h, #8 \n" // 16 bit to 8 bit R
"rshrn v3.8b, v3.8h, #8 \n" // 16 bit to 8 bit A
- "st4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%2], #32 \n" // store 8 ARGB
+ "st1 {v0.8b,v1.8b,v2.8b,v3.8b}, [%2], #32 \n" // store 8 ARGB
"b.gt 1b \n"
: "+r"(src_argb), // %0
"+r"(src_argb1), // %1
@@ -3713,17 +4494,15 @@
int width) {
asm volatile(
// 8 pixel loop.
- "1: \n"
- "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // load 8 ARGB
- "ld4 {v4.8b,v5.8b,v6.8b,v7.8b}, [%1], #32 \n" // load 8 more
+ "1: \n"
+ "ldp q0, q1, [%0], #32 \n" // load 8 ARGB
+ "ldp q4, q5, [%1], #32 \n" // load 8 more
"subs %w3, %w3, #8 \n" // 8 processed per loop.
- "uqadd v0.8b, v0.8b, v4.8b \n"
"prfm pldl1keep, [%0, 448] \n"
- "uqadd v1.8b, v1.8b, v5.8b \n"
"prfm pldl1keep, [%1, 448] \n"
- "uqadd v2.8b, v2.8b, v6.8b \n"
- "uqadd v3.8b, v3.8b, v7.8b \n"
- "st4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%2], #32 \n" // store 8 ARGB
+ "uqadd v0.16b, v0.16b, v4.16b \n"
+ "uqadd v1.16b, v1.16b, v5.16b \n"
+ "stp q0, q1, [%2], #32 \n" // store 8 ARGB
"b.gt 1b \n"
: "+r"(src_argb), // %0
"+r"(src_argb1), // %1
@@ -3740,17 +4519,15 @@
int width) {
asm volatile(
// 8 pixel loop.
- "1: \n"
- "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // load 8 ARGB
- "ld4 {v4.8b,v5.8b,v6.8b,v7.8b}, [%1], #32 \n" // load 8 more
+ "1: \n"
+ "ldp q0, q1, [%0], #32 \n" // load 8 ARGB
+ "ldp q4, q5, [%1], #32 \n" // load 8 more
"subs %w3, %w3, #8 \n" // 8 processed per loop.
- "uqsub v0.8b, v0.8b, v4.8b \n"
"prfm pldl1keep, [%0, 448] \n"
- "uqsub v1.8b, v1.8b, v5.8b \n"
"prfm pldl1keep, [%1, 448] \n"
- "uqsub v2.8b, v2.8b, v6.8b \n"
- "uqsub v3.8b, v3.8b, v7.8b \n"
- "st4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%2], #32 \n" // store 8 ARGB
+ "uqsub v0.16b, v0.16b, v4.16b \n"
+ "uqsub v1.16b, v1.16b, v5.16b \n"
+ "stp q0, q1, [%2], #32 \n" // store 8 ARGB
"b.gt 1b \n"
: "+r"(src_argb), // %0
"+r"(src_argb1), // %1
@@ -3772,15 +4549,15 @@
asm volatile(
"movi v3.8b, #255 \n" // alpha
// 8 pixel loop.
- "1: \n"
+ "1: \n"
"ld1 {v0.8b}, [%0], #8 \n" // load 8 sobelx.
"ld1 {v1.8b}, [%1], #8 \n" // load 8 sobely.
"subs %w3, %w3, #8 \n" // 8 processed per loop.
"uqadd v0.8b, v0.8b, v1.8b \n" // add
"prfm pldl1keep, [%0, 448] \n"
- "orr v1.8b, v0.8b, v0.8b \n"
+ "mov v1.8b, v0.8b \n"
"prfm pldl1keep, [%1, 448] \n"
- "orr v2.8b, v0.8b, v0.8b \n"
+ "mov v2.8b, v0.8b \n"
"st4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%2], #32 \n" // store 8 ARGB
"b.gt 1b \n"
: "+r"(src_sobelx), // %0
@@ -3798,7 +4575,7 @@
int width) {
asm volatile(
// 16 pixel loop.
- "1: \n"
+ "1: \n"
"ld1 {v0.16b}, [%0], #16 \n" // load 16 sobelx.
"ld1 {v1.16b}, [%1], #16 \n" // load 16 sobely.
"subs %w3, %w3, #16 \n" // 16 processed per loop.
@@ -3827,7 +4604,7 @@
asm volatile(
"movi v3.8b, #255 \n" // alpha
// 8 pixel loop.
- "1: \n"
+ "1: \n"
"ld1 {v2.8b}, [%0], #8 \n" // load 8 sobelx.
"ld1 {v0.8b}, [%1], #8 \n" // load 8 sobely.
"subs %w3, %w3, #8 \n" // 8 processed per loop.
@@ -3854,9 +4631,10 @@
uint8_t* dst_sobelx,
int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"ld1 {v0.8b}, [%0],%5 \n" // top
"ld1 {v1.8b}, [%0],%6 \n"
+ "subs %w4, %w4, #8 \n" // 8 pixels
"usubl v0.8h, v0.8b, v1.8b \n"
"prfm pldl1keep, [%0, 448] \n"
"ld1 {v2.8b}, [%1],%5 \n" // center * 2
@@ -3867,7 +4645,6 @@
"add v0.8h, v0.8h, v1.8h \n"
"ld1 {v2.8b}, [%2],%5 \n" // bottom
"ld1 {v3.8b}, [%2],%6 \n"
- "subs %w4, %w4, #8 \n" // 8 pixels
"prfm pldl1keep, [%2, 448] \n"
"usubl v1.8h, v2.8b, v3.8b \n"
"add v0.8h, v0.8h, v1.8h \n"
@@ -3895,9 +4672,10 @@
uint8_t* dst_sobely,
int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"ld1 {v0.8b}, [%0],%4 \n" // left
"ld1 {v1.8b}, [%1],%4 \n"
+ "subs %w3, %w3, #8 \n" // 8 pixels
"usubl v0.8h, v0.8b, v1.8b \n"
"ld1 {v2.8b}, [%0],%4 \n" // center * 2
"ld1 {v3.8b}, [%1],%4 \n"
@@ -3906,7 +4684,6 @@
"add v0.8h, v0.8h, v1.8h \n"
"ld1 {v2.8b}, [%0],%5 \n" // right
"ld1 {v3.8b}, [%1],%5 \n"
- "subs %w3, %w3, #8 \n" // 8 pixels
"usubl v1.8h, v2.8b, v3.8b \n"
"prfm pldl1keep, [%0, 448] \n"
"add v0.8h, v0.8h, v1.8h \n"
@@ -3925,55 +4702,38 @@
);
}
-// Caveat - rounds float to half float whereas scaling version truncates.
-void HalfFloat1Row_NEON(const uint16_t* src,
- uint16_t* dst,
- float /*unused*/,
- int width) {
- asm volatile(
- "1: \n"
- "ld1 {v1.16b}, [%0], #16 \n" // load 8 shorts
- "subs %w2, %w2, #8 \n" // 8 pixels per loop
- "uxtl v2.4s, v1.4h \n" // 8 int's
- "prfm pldl1keep, [%0, 448] \n"
- "uxtl2 v3.4s, v1.8h \n"
- "scvtf v2.4s, v2.4s \n" // 8 floats
- "scvtf v3.4s, v3.4s \n"
- "fcvtn v1.4h, v2.4s \n" // 8 half floats
- "fcvtn2 v1.8h, v3.4s \n"
- "st1 {v1.16b}, [%1], #16 \n" // store 8 shorts
- "b.gt 1b \n"
- : "+r"(src), // %0
- "+r"(dst), // %1
- "+r"(width) // %2
- :
- : "cc", "memory", "v1", "v2", "v3");
-}
-
void HalfFloatRow_NEON(const uint16_t* src,
uint16_t* dst,
float scale,
int width) {
asm volatile(
- "1: \n"
- "ld1 {v1.16b}, [%0], #16 \n" // load 8 shorts
- "subs %w2, %w2, #8 \n" // 8 pixels per loop
- "uxtl v2.4s, v1.4h \n" // 8 int's
+ "1: \n"
+ "ldp q0, q1, [%0], #32 \n" // load 16 shorts
+ "subs %w2, %w2, #16 \n" // 16 pixels per loop
+ "uxtl v2.4s, v0.4h \n"
+ "uxtl v4.4s, v1.4h \n"
+ "uxtl2 v3.4s, v0.8h \n"
+ "uxtl2 v5.4s, v1.8h \n"
"prfm pldl1keep, [%0, 448] \n"
- "uxtl2 v3.4s, v1.8h \n"
- "scvtf v2.4s, v2.4s \n" // 8 floats
+ "scvtf v2.4s, v2.4s \n"
+ "scvtf v4.4s, v4.4s \n"
"scvtf v3.4s, v3.4s \n"
+ "scvtf v5.4s, v5.4s \n"
"fmul v2.4s, v2.4s, %3.s[0] \n" // adjust exponent
+ "fmul v4.4s, v4.4s, %3.s[0] \n"
"fmul v3.4s, v3.4s, %3.s[0] \n"
- "uqshrn v1.4h, v2.4s, #13 \n" // isolate halffloat
- "uqshrn2 v1.8h, v3.4s, #13 \n"
- "st1 {v1.16b}, [%1], #16 \n" // store 8 shorts
+ "fmul v5.4s, v5.4s, %3.s[0] \n"
+ "uqshrn v0.4h, v2.4s, #13 \n" // isolate halffloat
+ "uqshrn v1.4h, v4.4s, #13 \n"
+ "uqshrn2 v0.8h, v3.4s, #13 \n"
+ "uqshrn2 v1.8h, v5.4s, #13 \n"
+ "stp q0, q1, [%1], #32 \n" // store 16 fp16
"b.gt 1b \n"
: "+r"(src), // %0
"+r"(dst), // %1
"+r"(width) // %2
: "w"(scale * 1.9259299444e-34f) // %3
- : "cc", "memory", "v1", "v2", "v3");
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5");
}
void ByteToFloatRow_NEON(const uint8_t* src,
@@ -3981,7 +4741,7 @@
float scale,
int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"ld1 {v1.8b}, [%0], #8 \n" // load 8 bytes
"subs %w2, %w2, #8 \n" // 8 pixels per loop
"uxtl v1.8h, v1.8b \n" // 8 shorts
@@ -4006,7 +4766,7 @@
float* dst,
int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"ld1 {v1.8h}, [%0], #16 \n" // load 8 halffloats
"subs %w2, %w2, #8 \n" // 8 floats per loop
"prfm pldl1keep, [%0, 448] \n"
@@ -4030,7 +4790,7 @@
asm volatile(
"cmp %w2, #8 \n" // Is there 8 rows?
"b.lo 2f \n"
- "1: \n"
+ "1: \n"
"ld1 {v0.h}[0], [%0], %3 \n" // load 8 halffloats
"ld1 {v0.h}[1], [%0], %3 \n"
"ld1 {v0.h}[2], [%0], %3 \n"
@@ -4047,13 +4807,13 @@
"b.gt 1b \n"
"cmp %w2, #1 \n" // Is there 1 value?
"b.lo 3f \n"
- "2: \n"
+ "2: \n"
"ld1 {v1.h}[0], [%0], %3 \n" // load 1 halffloats
"subs %w2, %w2, #1 \n" // 1 floats per loop
"fcvtl v2.4s, v1.4h \n" // 1 floats
"str s2, [%1], #4 \n" // store 1 floats
"b.gt 2b \n"
- "3: \n"
+ "3: \n"
: "+r"(src), // %0
"+r"(dst), // %1
"+r"(width) // %2
@@ -4066,7 +4826,7 @@
uint16_t* dst, // fp16
int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"ldp q2, q3, [%0], #32 \n" // load 8 floats
"subs %w2, %w2, #8 \n" // 8 floats per loop
"prfm pldl1keep, [%0, 448] \n"
@@ -4090,7 +4850,7 @@
"movi v5.4s, #0 \n" // max
"movi v6.4s, #0 \n"
- "1: \n"
+ "1: \n"
"ld1 {v1.4s, v2.4s}, [%0], #32 \n" // load 8 samples
"subs %w2, %w2, #8 \n" // 8 processed per loop
"fmul v3.4s, v1.4s, %4.s[0] \n" // scale
@@ -4120,7 +4880,7 @@
"movi v5.4s, #0 \n" // max
"movi v6.4s, #0 \n" // max
- "1: \n"
+ "1: \n"
"ld1 {v1.4s, v2.4s}, [%0], #32 \n" // load 8 samples
"subs %w2, %w2, #8 \n" // 8 processed per loop
"fmul v3.4s, v1.4s, %4.s[0] \n" // scale
@@ -4144,10 +4904,10 @@
void ScaleSamples_NEON(const float* src, float* dst, float scale, int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"ld1 {v1.4s, v2.4s}, [%0], #32 \n" // load 8 samples
- "prfm pldl1keep, [%0, 448] \n"
"subs %w2, %w2, #8 \n" // 8 processed per loop
+ "prfm pldl1keep, [%0, 448] \n"
"fmul v1.4s, v1.4s, %3.s[0] \n" // scale
"fmul v2.4s, v2.4s, %3.s[0] \n" // scale
"st1 {v1.4s, v2.4s}, [%1], #32 \n" // store 8 samples
@@ -4171,9 +4931,10 @@
"movi v6.8h, #4 \n" // constant 4
"movi v7.8h, #6 \n" // constant 6
- "1: \n"
+ "1: \n"
"ld1 {v1.8h}, [%0], #16 \n" // load 8 samples, 5 rows
"ld1 {v2.8h}, [%4], #16 \n"
+ "subs %w6, %w6, #8 \n" // 8 processed per loop
"uaddl v0.4s, v1.4h, v2.4h \n" // * 1
"prfm pldl1keep, [%0, 448] \n"
"uaddl2 v1.4s, v1.8h, v2.8h \n" // * 1
@@ -4189,7 +4950,6 @@
"umlal v0.4s, v2.4h, v6.4h \n" // * 4
"prfm pldl1keep, [%3, 448] \n"
"umlal2 v1.4s, v2.8h, v6.8h \n" // * 4
- "subs %w6, %w6, #8 \n" // 8 processed per loop
"st1 {v0.4s,v1.4s}, [%5], #32 \n" // store 8 samples
"prfm pldl1keep, [%4, 448] \n"
"b.gt 1b \n"
@@ -4213,8 +4973,9 @@
"movi v6.4s, #4 \n" // constant 4
"movi v7.4s, #6 \n" // constant 6
- "1: \n"
+ "1: \n"
"ld1 {v0.4s,v1.4s,v2.4s}, [%0], %6 \n" // load 12 source samples
+ "subs %w5, %w5, #8 \n" // 8 processed per loop
"add v0.4s, v0.4s, v1.4s \n" // * 1
"add v1.4s, v1.4s, v2.4s \n" // * 1
"ld1 {v2.4s,v3.4s}, [%2], #32 \n"
@@ -4227,7 +4988,6 @@
"prfm pldl1keep, [%0, 448] \n"
"mla v0.4s, v2.4s, v6.4s \n" // * 4
"mla v1.4s, v3.4s, v6.4s \n" // * 4
- "subs %w5, %w5, #8 \n" // 8 processed per loop
"uqrshrn v0.4h, v0.4s, #8 \n" // round and pack
"uqrshrn2 v0.8h, v1.4s, #8 \n"
"st1 {v0.8h}, [%4], #16 \n" // store 8 samples
@@ -4255,9 +5015,10 @@
asm volatile(
"ld2r {v6.4s, v7.4s}, [%7] \n" // constants 4 and 6
- "1: \n"
+ "1: \n"
"ld1 {v0.4s, v1.4s}, [%0], #32 \n" // load 8 samples, 5 rows
"ld1 {v2.4s, v3.4s}, [%1], #32 \n"
+ "subs %w6, %w6, #8 \n" // 8 processed per loop
"fmla v0.4s, v2.4s, v6.4s \n" // * 4
"ld1 {v4.4s, v5.4s}, [%2], #32 \n"
"fmla v1.4s, v3.4s, v6.4s \n"
@@ -4274,7 +5035,6 @@
"prfm pldl1keep, [%3, 448] \n"
"fadd v1.4s, v1.4s, v5.4s \n"
"prfm pldl1keep, [%4, 448] \n"
- "subs %w6, %w6, #8 \n" // 8 processed per loop
"st1 {v0.4s, v1.4s}, [%5], #32 \n" // store 8 samples
"b.gt 1b \n"
: "+r"(src0), // %0
@@ -4293,9 +5053,10 @@
asm volatile(
"ld3r {v6.4s, v7.4s, v8.4s}, [%3] \n" // constants 4, 6, 1/256
- "1: \n"
+ "1: \n"
"ld1 {v0.4s, v1.4s, v2.4s}, [%0], %4 \n" // load 12 samples, 5
// rows
+ "subs %w2, %w2, #8 \n" // 8 processed per loop
"fadd v0.4s, v0.4s, v1.4s \n" // * 1
"ld1 {v4.4s, v5.4s}, [%0], %5 \n"
"fadd v1.4s, v1.4s, v2.4s \n"
@@ -4310,7 +5071,6 @@
"prfm pldl1keep, [%0, 448] \n"
"fmul v0.4s, v0.4s, v8.4s \n" // / 256
"fmul v1.4s, v1.4s, v8.4s \n"
- "subs %w2, %w2, #8 \n" // 8 processed per loop
"st1 {v0.4s, v1.4s}, [%1], #32 \n" // store 8 samples
"b.gt 1b \n"
: "+r"(src), // %0
@@ -4323,21 +5083,21 @@
: "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8");
}
-#if LIBYUV_USE_ST3
+#if defined(LIBYUV_USE_ST3)
// Convert biplanar NV21 to packed YUV24
void NV21ToYUV24Row_NEON(const uint8_t* src_y,
const uint8_t* src_vu,
uint8_t* dst_yuv24,
int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"ld1 {v2.16b}, [%0], #16 \n" // load 16 Y values
"ld2 {v0.8b, v1.8b}, [%1], #16 \n" // load 8 VU values
+ "subs %w3, %w3, #16 \n" // 16 pixels per loop
"zip1 v0.16b, v0.16b, v0.16b \n" // replicate V values
"prfm pldl1keep, [%0, 448] \n"
"zip1 v1.16b, v1.16b, v1.16b \n" // replicate U values
"prfm pldl1keep, [%1, 448] \n"
- "subs %w3, %w3, #16 \n" // 16 pixels per loop
"st3 {v0.16b,v1.16b,v2.16b}, [%2], #48 \n" // store 16 YUV pixels
"b.gt 1b \n"
: "+r"(src_y), // %0
@@ -4362,15 +5122,15 @@
int width) {
asm volatile(
"ld1 {v5.16b,v6.16b,v7.16b}, [%4] \n" // 3 shuffler constants
- "1: \n"
+ "1: \n"
"ld1 {v0.16b}, [%0], #16 \n" // load 16 Y values
"ld1 {v1.16b}, [%1], #16 \n" // load 8 VU values
+ "subs %w3, %w3, #16 \n" // 16 pixels per loop
"tbl v2.16b, {v0.16b,v1.16b}, v5.16b \n" // weave into YUV24
"prfm pldl1keep, [%0, 448] \n"
"tbl v3.16b, {v0.16b,v1.16b}, v6.16b \n"
"prfm pldl1keep, [%1, 448] \n"
"tbl v4.16b, {v0.16b,v1.16b}, v7.16b \n"
- "subs %w3, %w3, #16 \n" // 16 pixels per loop
"st1 {v2.16b,v3.16b,v4.16b}, [%2], #48 \n" // store 16 YUV pixels
"b.gt 1b \n"
: "+r"(src_y), // %0
@@ -4392,8 +5152,9 @@
const uint8_t* src_ayuv_1 = src_ayuv + src_stride_ayuv;
asm volatile(
- "1: \n"
+ "1: \n"
"ld4 {v0.16b,v1.16b,v2.16b,v3.16b}, [%0], #64 \n" // load 16 ayuv
+ "subs %w3, %w3, #16 \n" // 16 processed per loop.
"uaddlp v0.8h, v0.16b \n" // V 16 bytes -> 8 shorts.
"prfm pldl1keep, [%0, 448] \n"
"uaddlp v1.8h, v1.16b \n" // U 16 bytes -> 8 shorts.
@@ -4403,7 +5164,6 @@
"prfm pldl1keep, [%1, 448] \n"
"uqrshrn v3.8b, v0.8h, #2 \n" // 2x2 average
"uqrshrn v2.8b, v1.8h, #2 \n"
- "subs %w3, %w3, #16 \n" // 16 processed per loop.
"st2 {v2.8b,v3.8b}, [%2], #16 \n" // store 8 pixels UV.
"b.gt 1b \n"
: "+r"(src_ayuv), // %0
@@ -4421,8 +5181,9 @@
const uint8_t* src_ayuv_1 = src_ayuv + src_stride_ayuv;
asm volatile(
- "1: \n"
+ "1: \n"
"ld4 {v0.16b,v1.16b,v2.16b,v3.16b}, [%0], #64 \n" // load 16 ayuv
+ "subs %w3, %w3, #16 \n" // 16 processed per loop.
"uaddlp v0.8h, v0.16b \n" // V 16 bytes -> 8 shorts.
"prfm pldl1keep, [%0, 448] \n"
"uaddlp v1.8h, v1.16b \n" // U 16 bytes -> 8 shorts.
@@ -4432,7 +5193,6 @@
"prfm pldl1keep, [%1, 448] \n"
"uqrshrn v0.8b, v0.8h, #2 \n" // 2x2 average
"uqrshrn v1.8b, v1.8h, #2 \n"
- "subs %w3, %w3, #16 \n" // 16 processed per loop.
"st2 {v0.8b,v1.8b}, [%2], #16 \n" // store 8 pixels VU.
"b.gt 1b \n"
: "+r"(src_ayuv), // %0
@@ -4446,7 +5206,7 @@
// Copy row of AYUV Y's into Y
void AYUVToYRow_NEON(const uint8_t* src_ayuv, uint8_t* dst_y, int width) {
asm volatile(
- "1: \n"
+ "1: \n"
"ld4 {v0.16b,v1.16b,v2.16b,v3.16b}, [%0], #64 \n" // load 16
"subs %w2, %w2, #16 \n" // 16 pixels per loop
"prfm pldl1keep, [%0, 448] \n"
@@ -4459,28 +5219,23 @@
: "cc", "memory", "v0", "v1", "v2", "v3");
}
-// Shuffle table for swapping UV bytes.
-static const uvec8 kShuffleSwapUV = {1u, 0u, 3u, 2u, 5u, 4u, 7u, 6u,
- 9u, 8u, 11u, 10u, 13u, 12u, 15u, 14u};
-
// Convert UV plane of NV12 to VU of NV21.
void SwapUVRow_NEON(const uint8_t* src_uv, uint8_t* dst_vu, int width) {
asm volatile(
- "ld1 {v2.16b}, [%3] \n" // shuffler
- "1: \n"
+ "1: \n"
"ld1 {v0.16b}, [%0], 16 \n" // load 16 UV values
"ld1 {v1.16b}, [%0], 16 \n"
"subs %w2, %w2, #16 \n" // 16 pixels per loop
- "tbl v0.16b, {v0.16b}, v2.16b \n"
+ "rev16 v0.16b, v0.16b \n"
"prfm pldl1keep, [%0, 448] \n"
- "tbl v1.16b, {v1.16b}, v2.16b \n"
+ "rev16 v1.16b, v1.16b \n"
"stp q0, q1, [%1], 32 \n" // store 16 VU pixels
"b.gt 1b \n"
- : "+r"(src_uv), // %0
- "+r"(dst_vu), // %1
- "+r"(width) // %2
- : "r"(&kShuffleSwapUV) // %3
- : "cc", "memory", "v0", "v1", "v2");
+ : "+r"(src_uv), // %0
+ "+r"(dst_vu), // %1
+ "+r"(width) // %2
+ :
+ : "cc", "memory", "v0", "v1");
}
void HalfMergeUVRow_NEON(const uint8_t* src_u,
@@ -4492,11 +5247,12 @@
const uint8_t* src_u_1 = src_u + src_stride_u;
const uint8_t* src_v_1 = src_v + src_stride_v;
asm volatile(
- "1: \n"
+ "1: \n"
"ld1 {v0.16b}, [%0], #16 \n" // load 16 U values
"ld1 {v1.16b}, [%2], #16 \n" // load 16 V values
"ld1 {v2.16b}, [%1], #16 \n"
"ld1 {v3.16b}, [%3], #16 \n"
+ "subs %w5, %w5, #16 \n" // 16 src pixels per loop
"uaddlp v0.8h, v0.16b \n" // half size
"prfm pldl1keep, [%0, 448] \n"
"uaddlp v1.8h, v1.16b \n"
@@ -4507,7 +5263,6 @@
"prfm pldl1keep, [%3, 448] \n"
"uqrshrn v0.8b, v0.8h, #2 \n"
"uqrshrn v1.8b, v1.8h, #2 \n"
- "subs %w5, %w5, #16 \n" // 16 src pixels per loop
"st2 {v0.8b, v1.8b}, [%4], #16 \n" // store 8 UV pixels
"b.gt 1b \n"
: "+r"(src_u), // %0
@@ -4528,7 +5283,7 @@
int shift = depth - 16; // Negative for right shift.
asm volatile(
"dup v2.8h, %w4 \n"
- "1: \n"
+ "1: \n"
"ld2 {v0.8h, v1.8h}, [%0], #32 \n" // load 8 UV
"subs %w3, %w3, #8 \n" // 8 src pixels per loop
"ushl v0.8h, v0.8h, v2.8h \n"
@@ -4551,13 +5306,13 @@
int width) {
asm volatile(
"dup v2.8h, %w3 \n"
- "1: \n"
+ "1: \n"
"ldp q0, q1, [%0], #32 \n"
+ "subs %w2, %w2, #16 \n" // 16 src pixels per loop
"mul v0.8h, v0.8h, v2.8h \n"
"prfm pldl1keep, [%0, 448] \n"
"mul v1.8h, v1.8h, v2.8h \n"
"stp q0, q1, [%1], #32 \n" // store 16 pixels
- "subs %w2, %w2, #16 \n" // 16 src pixels per loop
"b.gt 1b \n"
: "+r"(src_y), // %0
"+r"(dst_y), // %1
@@ -4572,19 +5327,17 @@
int width) {
asm volatile(
"dup v4.8h, %w3 \n"
- "1: \n"
+ "1: \n"
"ldp q2, q3, [%0], #32 \n"
+ "subs %w2, %w2, #16 \n" // 16 src pixels per loop
"umull v0.4s, v2.4h, v4.4h \n"
"umull2 v1.4s, v2.8h, v4.8h \n"
"umull v2.4s, v3.4h, v4.4h \n"
"umull2 v3.4s, v3.8h, v4.8h \n"
"prfm pldl1keep, [%0, 448] \n"
- "shrn v0.4h, v0.4s, #16 \n"
- "shrn2 v0.8h, v1.4s, #16 \n"
- "shrn v1.4h, v2.4s, #16 \n"
- "shrn2 v1.8h, v3.4s, #16 \n"
+ "uzp2 v0.8h, v0.8h, v1.8h \n"
+ "uzp2 v1.8h, v2.8h, v3.8h \n"
"stp q0, q1, [%1], #32 \n" // store 16 pixels
- "subs %w2, %w2, #16 \n" // 16 src pixels per loop
"b.gt 1b \n"
: "+r"(src_y), // %0
"+r"(dst_y), // %1
@@ -4602,17 +5355,19 @@
uint8_t* dst_y,
int scale,
int width) {
- int shift = 15 - __builtin_clz((int32_t)scale); // Negative shl is shr
+ // 15 - clz(scale), + 8 to shift result into the high half of the lane to
+ // saturate, then we can just use UZP2 to narrow rather than a pair of
+ // saturating narrow instructions.
+ int shift = 23 - __builtin_clz((int32_t)scale);
asm volatile(
"dup v2.8h, %w3 \n"
- "1: \n"
+ "1: \n"
"ldp q0, q1, [%0], #32 \n"
- "ushl v0.8h, v0.8h, v2.8h \n" // shr = v2 is negative
- "ushl v1.8h, v1.8h, v2.8h \n"
- "prfm pldl1keep, [%0, 448] \n"
- "uqxtn v0.8b, v0.8h \n"
- "uqxtn2 v0.16b, v1.8h \n"
"subs %w2, %w2, #16 \n" // 16 src pixels per loop
+ "uqshl v0.8h, v0.8h, v2.8h \n"
+ "uqshl v1.8h, v1.8h, v2.8h \n"
+ "prfm pldl1keep, [%0, 448] \n"
+ "uzp2 v0.16b, v0.16b, v1.16b \n"
"str q0, [%1], #16 \n" // store 16 pixels
"b.gt 1b \n"
: "+r"(src_y), // %0
@@ -4622,6 +5377,40 @@
: "cc", "memory", "v0", "v1", "v2");
}
+// Use scale to convert J420 to I420
+// scale parameter is 8.8 fixed point but limited to 0 to 255
+// Function is based on DivideRow, but adds a bias
+// Does not clamp
+void Convert8To8Row_NEON(const uint8_t* src_y,
+ uint8_t* dst_y,
+ int scale,
+ int bias,
+ int width) {
+ asm volatile(
+ "dup v4.16b, %w3 \n" // scale
+ "dup v5.16b, %w4 \n" // bias
+ "1: \n"
+ "ldp q2, q3, [%0], #32 \n"
+ "subs %w2, %w2, #32 \n" // 32 pixels per loop
+ "umull v0.8h, v2.8b, v4.8b \n"
+ "umull2 v1.8h, v2.16b, v4.16b \n"
+ "umull v2.8h, v3.8b, v4.8b \n"
+ "umull2 v3.8h, v3.16b, v4.16b \n"
+ "prfm pldl1keep, [%0, 448] \n"
+ "uzp2 v0.16b, v0.16b, v1.16b \n"
+ "uzp2 v1.16b, v2.16b, v3.16b \n"
+ "add v0.16b, v0.16b, v5.16b \n" // add bias (16)
+ "add v1.16b, v1.16b, v5.16b \n"
+ "stp q0, q1, [%1], #32 \n" // store 32 pixels
+ "b.gt 1b \n"
+ : "+r"(src_y), // %0
+ "+r"(dst_y), // %1
+ "+r"(width) // %2
+ : "r"(scale), // %3
+ "r"(bias) // %4
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5");
+}
+
#endif // !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__)
#ifdef __cplusplus
diff --git a/source/row_rvv.cc b/source/row_rvv.cc
index 0bf2bef..07606d7 100644
--- a/source/row_rvv.cc
+++ b/source/row_rvv.cc
@@ -28,6 +28,20 @@
extern "C" {
#endif
+#ifdef LIBYUV_RVV_HAS_VXRM_ARG
+// Fill YUV -> RGB conversion constants into vectors
+#define YUVTORGB_SETUP(yuvconst, ub, vr, ug, vg, yg, bb, bg, br) \
+ { \
+ ub = yuvconst->kUVCoeff[0]; \
+ vr = yuvconst->kUVCoeff[1]; \
+ ug = yuvconst->kUVCoeff[2]; \
+ vg = yuvconst->kUVCoeff[3]; \
+ yg = yuvconst->kRGBCoeffBias[0]; \
+ bb = yuvconst->kRGBCoeffBias[1] + 32; \
+ bg = yuvconst->kRGBCoeffBias[2] - 32; \
+ br = yuvconst->kRGBCoeffBias[3] + 32; \
+ }
+#else
// Fill YUV -> RGB conversion constants into vectors
// NOTE: To match behavior on other platforms, vxrm (fixed-point rounding mode
// register) is set to round-to-nearest-up mode(0).
@@ -43,7 +57,7 @@
bg = yuvconst->kRGBCoeffBias[2] - 32; \
br = yuvconst->kRGBCoeffBias[3] + 32; \
}
-
+#endif
// Read [2*VLEN/8] Y, [VLEN/8] U and [VLEN/8] V from 422
#define READYUV422(vl, w, src_y, src_u, src_v, v_u, v_v, v_y_16) \
{ \
@@ -95,6 +109,15 @@
v_r_16 = __riscv_vssubu_vx_u16m4(v_tmp2, br, vl); \
}
+#ifdef LIBYUV_RVV_HAS_VXRM_ARG
+// Convert from fixed point RGB To 8 bit RGB
+#define RGBTORGB8(vl, v_g_16, v_b_16, v_r_16, v_g, v_b, v_r) \
+ { \
+ v_g = __riscv_vnclipu_wx_u8m2(v_g_16, 6, __RISCV_VXRM_RNU, vl); \
+ v_b = __riscv_vnclipu_wx_u8m2(v_b_16, 6, __RISCV_VXRM_RNU, vl); \
+ v_r = __riscv_vnclipu_wx_u8m2(v_r_16, 6, __RISCV_VXRM_RNU, vl); \
+ }
+#else
// Convert from fixed point RGB To 8 bit RGB
#define RGBTORGB8(vl, v_g_16, v_b_16, v_r_16, v_g, v_b, v_r) \
{ \
@@ -102,7 +125,53 @@
v_b = __riscv_vnclipu_wx_u8m2(v_b_16, 6, vl); \
v_r = __riscv_vnclipu_wx_u8m2(v_r_16, 6, vl); \
}
+#endif
+#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE
+// Read [2*VLEN/8] Y from src_y; Read [VLEN/8] U and [VLEN/8] V from src_uv
+#define READNV12(vl, w, src_y, src_uv, v_u, v_v, v_y_16) \
+ { \
+ vuint8m1x2_t v_tmp; \
+ vuint8m1_t v_tmp0, v_tmp1; \
+ vuint8m2_t v_y; \
+ vuint16m2_t v_u_16, v_v_16; \
+ vl = __riscv_vsetvl_e8m1((w + 1) / 2); \
+ v_tmp = __riscv_vlseg2e8_v_u8m1x2(src_uv, vl); \
+ v_tmp0 = __riscv_vget_v_u8m1x2_u8m1(v_tmp, 0); \
+ v_tmp1 = __riscv_vget_v_u8m1x2_u8m1(v_tmp, 1); \
+ v_u_16 = __riscv_vwaddu_vx_u16m2(v_tmp0, 0, vl); \
+ v_v_16 = __riscv_vwaddu_vx_u16m2(v_tmp1, 0, vl); \
+ v_v_16 = __riscv_vmul_vx_u16m2(v_v_16, 0x0101, vl); \
+ v_u_16 = __riscv_vmul_vx_u16m2(v_u_16, 0x0101, vl); \
+ v_v = __riscv_vreinterpret_v_u16m2_u8m2(v_v_16); \
+ v_u = __riscv_vreinterpret_v_u16m2_u8m2(v_u_16); \
+ vl = __riscv_vsetvl_e8m2(w); \
+ v_y = __riscv_vle8_v_u8m2(src_y, vl); \
+ v_y_16 = __riscv_vwaddu_vx_u16m4(v_y, 0, vl); \
+ }
+
+// Read 2*[VLEN/8] Y from src_y; Read [VLEN/8] U and [VLEN/8] V from src_vu
+#define READNV21(vl, w, src_y, src_vu, v_u, v_v, v_y_16) \
+ { \
+ vuint8m1x2_t v_tmp; \
+ vuint8m1_t v_tmp0, v_tmp1; \
+ vuint8m2_t v_y; \
+ vuint16m2_t v_u_16, v_v_16; \
+ vl = __riscv_vsetvl_e8m1((w + 1) / 2); \
+ v_tmp = __riscv_vlseg2e8_v_u8m1x2(src_vu, vl); \
+ v_tmp0 = __riscv_vget_v_u8m1x2_u8m1(v_tmp, 0); \
+ v_tmp1 = __riscv_vget_v_u8m1x2_u8m1(v_tmp, 1); \
+ v_u_16 = __riscv_vwaddu_vx_u16m2(v_tmp1, 0, vl); \
+ v_v_16 = __riscv_vwaddu_vx_u16m2(v_tmp0, 0, vl); \
+ v_v_16 = __riscv_vmul_vx_u16m2(v_v_16, 0x0101, vl); \
+ v_u_16 = __riscv_vmul_vx_u16m2(v_u_16, 0x0101, vl); \
+ v_v = __riscv_vreinterpret_v_u16m2_u8m2(v_v_16); \
+ v_u = __riscv_vreinterpret_v_u16m2_u8m2(v_u_16); \
+ vl = __riscv_vsetvl_e8m2(w); \
+ v_y = __riscv_vle8_v_u8m2(src_y, vl); \
+ v_y_16 = __riscv_vwaddu_vx_u16m4(v_y, 0, vl); \
+ }
+#else
// Read [2*VLEN/8] Y from src_y; Read [VLEN/8] U and [VLEN/8] V from src_uv
#define READNV12(vl, w, src_y, src_uv, v_u, v_v, v_y_16) \
{ \
@@ -140,6 +209,7 @@
v_y = __riscv_vle8_v_u8m2(src_y, vl); \
v_y_16 = __riscv_vwaddu_vx_u16m4(v_y, 0, vl); \
}
+#endif
#ifdef HAS_ARGBTOAR64ROW_RVV
void ARGBToAR64Row_RVV(const uint8_t* src_argb, uint16_t* dst_ar64, int width) {
@@ -160,6 +230,34 @@
#endif
#ifdef HAS_ARGBTOAB64ROW_RVV
+#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE
+void ARGBToAB64Row_RVV(const uint8_t* src_argb, uint16_t* dst_ab64, int width) {
+ size_t avl = (size_t)width;
+ do {
+ vuint16m2x4_t v_dst_ab64;
+ vuint16m2_t v_b_16, v_g_16, v_r_16, v_a_16;
+ size_t vl = __riscv_vsetvl_e8m1(avl);
+ vuint8m1x4_t v_src_argb = __riscv_vlseg4e8_v_u8m1x4(src_argb, vl);
+ vuint8m1_t v_b = __riscv_vget_v_u8m1x4_u8m1(v_src_argb, 0);
+ vuint8m1_t v_g = __riscv_vget_v_u8m1x4_u8m1(v_src_argb, 1);
+ vuint8m1_t v_r = __riscv_vget_v_u8m1x4_u8m1(v_src_argb, 2);
+ vuint8m1_t v_a = __riscv_vget_v_u8m1x4_u8m1(v_src_argb, 3);
+ v_b_16 = __riscv_vwaddu_vx_u16m2(v_b, 0, vl);
+ v_g_16 = __riscv_vwaddu_vx_u16m2(v_g, 0, vl);
+ v_r_16 = __riscv_vwaddu_vx_u16m2(v_r, 0, vl);
+ v_a_16 = __riscv_vwaddu_vx_u16m2(v_a, 0, vl);
+ v_b_16 = __riscv_vmul_vx_u16m2(v_b_16, 0x0101, vl);
+ v_g_16 = __riscv_vmul_vx_u16m2(v_g_16, 0x0101, vl);
+ v_r_16 = __riscv_vmul_vx_u16m2(v_r_16, 0x0101, vl);
+ v_a_16 = __riscv_vmul_vx_u16m2(v_a_16, 0x0101, vl);
+ v_dst_ab64 = __riscv_vcreate_v_u16m2x4(v_r_16, v_g_16, v_b_16, v_a_16);
+ __riscv_vsseg4e16_v_u16m2x4(dst_ab64, v_dst_ab64, vl);
+ avl -= vl;
+ src_argb += 4 * vl;
+ dst_ab64 += 4 * vl;
+ } while (avl > 0);
+}
+#else
void ARGBToAB64Row_RVV(const uint8_t* src_argb, uint16_t* dst_ab64, int width) {
size_t avl = (size_t)width;
do {
@@ -182,6 +280,7 @@
} while (avl > 0);
}
#endif
+#endif
#ifdef HAS_AR64TOARGBROW_RVV
void AR64ToARGBRow_RVV(const uint16_t* src_ar64, uint8_t* dst_argb, int width) {
@@ -201,6 +300,26 @@
#endif
#ifdef HAS_AR64TOAB64ROW_RVV
+#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE
+void AR64ToAB64Row_RVV(const uint16_t* src_ar64,
+ uint16_t* dst_ab64,
+ int width) {
+ size_t w = (size_t)width;
+ do {
+ size_t vl = __riscv_vsetvl_e16m2(w);
+ vuint16m2x4_t v_argb16 = __riscv_vlseg4e16_v_u16m2x4(src_ar64, vl);
+ vuint16m2_t v_b = __riscv_vget_v_u16m2x4_u16m2(v_argb16, 0);
+ vuint16m2_t v_g = __riscv_vget_v_u16m2x4_u16m2(v_argb16, 1);
+ vuint16m2_t v_r = __riscv_vget_v_u16m2x4_u16m2(v_argb16, 2);
+ vuint16m2_t v_a = __riscv_vget_v_u16m2x4_u16m2(v_argb16, 3);
+ vuint16m2x4_t v_dst_abgr = __riscv_vcreate_v_u16m2x4(v_r, v_g, v_b, v_a);
+ __riscv_vsseg4e16_v_u16m2x4(dst_ab64, v_dst_abgr, vl);
+ w -= vl;
+ src_ar64 += vl * 4;
+ dst_ab64 += vl * 4;
+ } while (w > 0);
+}
+#else
void AR64ToAB64Row_RVV(const uint16_t* src_ar64,
uint16_t* dst_ab64,
int width) {
@@ -216,8 +335,31 @@
} while (w > 0);
}
#endif
+#endif
#ifdef HAS_AB64TOARGBROW_RVV
+#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE
+void AB64ToARGBRow_RVV(const uint16_t* src_ab64, uint8_t* dst_argb, int width) {
+ size_t avl = (size_t)width;
+ do {
+ size_t vl = __riscv_vsetvl_e16m2(avl);
+ vuint16m2x4_t v_abgr16 = __riscv_vlseg4e16_v_u16m2x4(src_ab64, vl);
+ vuint16m2_t v_r_16 = __riscv_vget_v_u16m2x4_u16m2(v_abgr16, 0);
+ vuint16m2_t v_g_16 = __riscv_vget_v_u16m2x4_u16m2(v_abgr16, 1);
+ vuint16m2_t v_b_16 = __riscv_vget_v_u16m2x4_u16m2(v_abgr16, 2);
+ vuint16m2_t v_a_16 = __riscv_vget_v_u16m2x4_u16m2(v_abgr16, 3);
+ vuint8m1_t v_b = __riscv_vnsrl_wx_u8m1(v_b_16, 8, vl);
+ vuint8m1_t v_g = __riscv_vnsrl_wx_u8m1(v_g_16, 8, vl);
+ vuint8m1_t v_r = __riscv_vnsrl_wx_u8m1(v_r_16, 8, vl);
+ vuint8m1_t v_a = __riscv_vnsrl_wx_u8m1(v_a_16, 8, vl);
+ vuint8m1x4_t v_dst_argb = __riscv_vcreate_v_u8m1x4(v_b, v_g, v_r, v_a);
+ __riscv_vsseg4e8_v_u8m1x4(dst_argb, v_dst_argb, vl);
+ avl -= vl;
+ src_ab64 += 4 * vl;
+ dst_argb += 4 * vl;
+ } while (avl > 0);
+}
+#else
void AB64ToARGBRow_RVV(const uint16_t* src_ab64, uint8_t* dst_argb, int width) {
size_t avl = (size_t)width;
do {
@@ -236,8 +378,28 @@
} while (avl > 0);
}
#endif
+#endif
#ifdef HAS_RAWTOARGBROW_RVV
+#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE
+void RAWToARGBRow_RVV(const uint8_t* src_raw, uint8_t* dst_argb, int width) {
+ size_t w = (size_t)width;
+ size_t vl = __riscv_vsetvl_e8m2(w);
+ vuint8m2_t v_a = __riscv_vmv_v_x_u8m2(255u, vl);
+ do {
+ vuint8m2x3_t v_bgr = __riscv_vlseg3e8_v_u8m2x3(src_raw, vl);
+ vuint8m2_t v_r = __riscv_vget_v_u8m2x3_u8m2(v_bgr, 0);
+ vuint8m2_t v_g = __riscv_vget_v_u8m2x3_u8m2(v_bgr, 1);
+ vuint8m2_t v_b = __riscv_vget_v_u8m2x3_u8m2(v_bgr, 2);
+ vuint8m2x4_t v_dst_argb = __riscv_vcreate_v_u8m2x4(v_b, v_g, v_r, v_a);
+ __riscv_vsseg4e8_v_u8m2x4(dst_argb, v_dst_argb, vl);
+ w -= vl;
+ src_raw += vl * 3;
+ dst_argb += vl * 4;
+ vl = __riscv_vsetvl_e8m2(w);
+ } while (w > 0);
+}
+#else
void RAWToARGBRow_RVV(const uint8_t* src_raw, uint8_t* dst_argb, int width) {
size_t w = (size_t)width;
size_t vl = __riscv_vsetvl_e8m2(w);
@@ -253,8 +415,28 @@
} while (w > 0);
}
#endif
+#endif
#ifdef HAS_RAWTORGBAROW_RVV
+#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE
+void RAWToRGBARow_RVV(const uint8_t* src_raw, uint8_t* dst_rgba, int width) {
+ size_t w = (size_t)width;
+ size_t vl = __riscv_vsetvl_e8m2(w);
+ vuint8m2_t v_a = __riscv_vmv_v_x_u8m2(255u, vl);
+ do {
+ vuint8m2x3_t v_bgr = __riscv_vlseg3e8_v_u8m2x3(src_raw, vl);
+ vuint8m2_t v_r = __riscv_vget_v_u8m2x3_u8m2(v_bgr, 0);
+ vuint8m2_t v_g = __riscv_vget_v_u8m2x3_u8m2(v_bgr, 1);
+ vuint8m2_t v_b = __riscv_vget_v_u8m2x3_u8m2(v_bgr, 2);
+ vuint8m2x4_t v_dst_rgba = __riscv_vcreate_v_u8m2x4(v_a, v_b, v_g, v_r);
+ __riscv_vsseg4e8_v_u8m2x4(dst_rgba, v_dst_rgba, vl);
+ w -= vl;
+ src_raw += vl * 3;
+ dst_rgba += vl * 4;
+ vl = __riscv_vsetvl_e8m2(w);
+ } while (w > 0);
+}
+#else
void RAWToRGBARow_RVV(const uint8_t* src_raw, uint8_t* dst_rgba, int width) {
size_t w = (size_t)width;
size_t vl = __riscv_vsetvl_e8m2(w);
@@ -270,8 +452,26 @@
} while (w > 0);
}
#endif
+#endif
#ifdef HAS_RAWTORGB24ROW_RVV
+#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE
+void RAWToRGB24Row_RVV(const uint8_t* src_raw, uint8_t* dst_rgb24, int width) {
+ size_t w = (size_t)width;
+ do {
+ size_t vl = __riscv_vsetvl_e8m2(w);
+ vuint8m2x3_t v_bgr = __riscv_vlseg3e8_v_u8m2x3(src_raw, vl);
+ vuint8m2_t v_r = __riscv_vget_v_u8m2x3_u8m2(v_bgr, 0);
+ vuint8m2_t v_g = __riscv_vget_v_u8m2x3_u8m2(v_bgr, 1);
+ vuint8m2_t v_b = __riscv_vget_v_u8m2x3_u8m2(v_bgr, 2);
+ vuint8m2x3_t v_dst_rgb = __riscv_vcreate_v_u8m2x3(v_b, v_g, v_r);
+ __riscv_vsseg3e8_v_u8m2x3(dst_rgb24, v_dst_rgb, vl);
+ w -= vl;
+ src_raw += vl * 3;
+ dst_rgb24 += vl * 3;
+ } while (w > 0);
+}
+#else
void RAWToRGB24Row_RVV(const uint8_t* src_raw, uint8_t* dst_rgb24, int width) {
size_t w = (size_t)width;
do {
@@ -285,8 +485,26 @@
} while (w > 0);
}
#endif
+#endif
#ifdef HAS_ARGBTORAWROW_RVV
+#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE
+void ARGBToRAWRow_RVV(const uint8_t* src_argb, uint8_t* dst_raw, int width) {
+ size_t w = (size_t)width;
+ do {
+ size_t vl = __riscv_vsetvl_e8m2(w);
+ vuint8m2x4_t v_src_argb = __riscv_vlseg4e8_v_u8m2x4(src_argb, vl);
+ vuint8m2_t v_b = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 0);
+ vuint8m2_t v_g = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 1);
+ vuint8m2_t v_r = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 2);
+ vuint8m2x3_t v_dst_bgr = __riscv_vcreate_v_u8m2x3(v_r, v_g, v_b);
+ __riscv_vsseg3e8_v_u8m2x3(dst_raw, v_dst_bgr, vl);
+ w -= vl;
+ src_argb += vl * 4;
+ dst_raw += vl * 3;
+ } while (w > 0);
+}
+#else
void ARGBToRAWRow_RVV(const uint8_t* src_argb, uint8_t* dst_raw, int width) {
size_t w = (size_t)width;
do {
@@ -300,8 +518,28 @@
} while (w > 0);
}
#endif
+#endif
#ifdef HAS_ARGBTORGB24ROW_RVV
+#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE
+void ARGBToRGB24Row_RVV(const uint8_t* src_argb,
+ uint8_t* dst_rgb24,
+ int width) {
+ size_t w = (size_t)width;
+ do {
+ size_t vl = __riscv_vsetvl_e8m2(w);
+ vuint8m2x4_t v_src_argb = __riscv_vlseg4e8_v_u8m2x4(src_argb, vl);
+ vuint8m2_t v_b = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 0);
+ vuint8m2_t v_g = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 1);
+ vuint8m2_t v_r = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 2);
+ vuint8m2x3_t v_dst_rgb = __riscv_vcreate_v_u8m2x3(v_b, v_g, v_r);
+ __riscv_vsseg3e8_v_u8m2x3(dst_rgb24, v_dst_rgb, vl);
+ w -= vl;
+ src_argb += vl * 4;
+ dst_rgb24 += vl * 3;
+ } while (w > 0);
+}
+#else
void ARGBToRGB24Row_RVV(const uint8_t* src_argb,
uint8_t* dst_rgb24,
int width) {
@@ -317,8 +555,27 @@
} while (w > 0);
}
#endif
+#endif
#ifdef HAS_ARGBTOABGRROW_RVV
+#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE
+void ARGBToABGRRow_RVV(const uint8_t* src_argb, uint8_t* dst_abgr, int width) {
+ size_t w = (size_t)width;
+ do {
+ size_t vl = __riscv_vsetvl_e8m2(w);
+ vuint8m2x4_t v_src_argb = __riscv_vlseg4e8_v_u8m2x4(src_argb, vl);
+ vuint8m2_t v_b = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 0);
+ vuint8m2_t v_g = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 1);
+ vuint8m2_t v_r = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 2);
+ vuint8m2_t v_a = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 3);
+ vuint8m2x4_t v_dst_abgr = __riscv_vcreate_v_u8m2x4(v_r, v_g, v_b, v_a);
+ __riscv_vsseg4e8_v_u8m2x4(dst_abgr, v_dst_abgr, vl);
+ w -= vl;
+ src_argb += vl * 4;
+ dst_abgr += vl * 4;
+ } while (w > 0);
+}
+#else
void ARGBToABGRRow_RVV(const uint8_t* src_argb, uint8_t* dst_abgr, int width) {
size_t w = (size_t)width;
do {
@@ -332,8 +589,27 @@
} while (w > 0);
}
#endif
+#endif
#ifdef HAS_ARGBTOBGRAROW_RVV
+#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE
+void ARGBToBGRARow_RVV(const uint8_t* src_argb, uint8_t* dst_bgra, int width) {
+ size_t w = (size_t)width;
+ do {
+ size_t vl = __riscv_vsetvl_e8m2(w);
+ vuint8m2x4_t v_src_argb = __riscv_vlseg4e8_v_u8m2x4(src_argb, vl);
+ vuint8m2_t v_b = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 0);
+ vuint8m2_t v_g = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 1);
+ vuint8m2_t v_r = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 2);
+ vuint8m2_t v_a = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 3);
+ vuint8m2x4_t v_dst_bgra = __riscv_vcreate_v_u8m2x4(v_a, v_r, v_g, v_b);
+ __riscv_vsseg4e8_v_u8m2x4(dst_bgra, v_dst_bgra, vl);
+ w -= vl;
+ src_argb += vl * 4;
+ dst_bgra += vl * 4;
+ } while (w > 0);
+}
+#else
void ARGBToBGRARow_RVV(const uint8_t* src_argb, uint8_t* dst_bgra, int width) {
size_t w = (size_t)width;
do {
@@ -347,8 +623,27 @@
} while (w > 0);
}
#endif
+#endif
#ifdef HAS_ARGBTORGBAROW_RVV
+#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE
+void ARGBToRGBARow_RVV(const uint8_t* src_argb, uint8_t* dst_rgba, int width) {
+ size_t w = (size_t)width;
+ do {
+ size_t vl = __riscv_vsetvl_e8m2(w);
+ vuint8m2x4_t v_src_argb = __riscv_vlseg4e8_v_u8m2x4(src_argb, vl);
+ vuint8m2_t v_b = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 0);
+ vuint8m2_t v_g = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 1);
+ vuint8m2_t v_r = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 2);
+ vuint8m2_t v_a = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 3);
+ vuint8m2x4_t v_dst_rgba = __riscv_vcreate_v_u8m2x4(v_a, v_b, v_g, v_r);
+ __riscv_vsseg4e8_v_u8m2x4(dst_rgba, v_dst_rgba, vl);
+ w -= vl;
+ src_argb += vl * 4;
+ dst_rgba += vl * 4;
+ } while (w > 0);
+}
+#else
void ARGBToRGBARow_RVV(const uint8_t* src_argb, uint8_t* dst_rgba, int width) {
size_t w = (size_t)width;
do {
@@ -362,8 +657,27 @@
} while (w > 0);
}
#endif
+#endif
#ifdef HAS_RGBATOARGBROW_RVV
+#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE
+void RGBAToARGBRow_RVV(const uint8_t* src_rgba, uint8_t* dst_argb, int width) {
+ size_t w = (size_t)width;
+ do {
+ size_t vl = __riscv_vsetvl_e8m2(w);
+ vuint8m2x4_t v_src_rgba = __riscv_vlseg4e8_v_u8m2x4(src_rgba, vl);
+ vuint8m2_t v_a = __riscv_vget_v_u8m2x4_u8m2(v_src_rgba, 0);
+ vuint8m2_t v_b = __riscv_vget_v_u8m2x4_u8m2(v_src_rgba, 1);
+ vuint8m2_t v_g = __riscv_vget_v_u8m2x4_u8m2(v_src_rgba, 2);
+ vuint8m2_t v_r = __riscv_vget_v_u8m2x4_u8m2(v_src_rgba, 3);
+ vuint8m2x4_t v_dst_argb = __riscv_vcreate_v_u8m2x4(v_b, v_g, v_r, v_a);
+ __riscv_vsseg4e8_v_u8m2x4(dst_argb, v_dst_argb, vl);
+ w -= vl;
+ src_rgba += vl * 4;
+ dst_argb += vl * 4;
+ } while (w > 0);
+}
+#else
void RGBAToARGBRow_RVV(const uint8_t* src_rgba, uint8_t* dst_argb, int width) {
size_t w = (size_t)width;
do {
@@ -377,8 +691,30 @@
} while (w > 0);
}
#endif
+#endif
#ifdef HAS_RGB24TOARGBROW_RVV
+#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE
+void RGB24ToARGBRow_RVV(const uint8_t* src_rgb24,
+ uint8_t* dst_argb,
+ int width) {
+ size_t w = (size_t)width;
+ size_t vl = __riscv_vsetvl_e8m2(w);
+ vuint8m2_t v_a = __riscv_vmv_v_x_u8m2(255u, vl);
+ do {
+ vuint8m2x3_t v_src_rgb = __riscv_vlseg3e8_v_u8m2x3(src_rgb24, vl);
+ vuint8m2_t v_b = __riscv_vget_v_u8m2x3_u8m2(v_src_rgb, 0);
+ vuint8m2_t v_g = __riscv_vget_v_u8m2x3_u8m2(v_src_rgb, 1);
+ vuint8m2_t v_r = __riscv_vget_v_u8m2x3_u8m2(v_src_rgb, 2);
+ vuint8m2x4_t v_dst_argb = __riscv_vcreate_v_u8m2x4(v_b, v_g, v_r, v_a);
+ __riscv_vsseg4e8_v_u8m2x4(dst_argb, v_dst_argb, vl);
+ w -= vl;
+ src_rgb24 += vl * 3;
+ dst_argb += vl * 4;
+ vl = __riscv_vsetvl_e8m2(w);
+ } while (w > 0);
+}
+#else
void RGB24ToARGBRow_RVV(const uint8_t* src_rgb24,
uint8_t* dst_argb,
int width) {
@@ -396,8 +732,41 @@
} while (w > 0);
}
#endif
+#endif
#ifdef HAS_I444TOARGBROW_RVV
+#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE
+void I444ToARGBRow_RVV(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ size_t w = (size_t)width;
+ size_t vl = __riscv_vsetvl_e8m2(w);
+ uint8_t ub, vr, ug, vg;
+ int16_t yg, bb, bg, br;
+ vuint8m2_t v_u, v_v;
+ vuint8m2_t v_b, v_g, v_r, v_a;
+ vuint16m4_t v_y_16, v_g_16, v_b_16, v_r_16;
+ vuint8m2x4_t v_dst_argb;
+ YUVTORGB_SETUP(yuvconstants, ub, vr, ug, vg, yg, bb, bg, br);
+ v_a = __riscv_vmv_v_x_u8m2(255u, vl);
+ do {
+ READYUV444(vl, w, src_y, src_u, src_v, v_u, v_v, v_y_16);
+ YUVTORGB(vl, v_u, v_v, ub, vr, ug, vg, yg, bb, bg, br, v_y_16, v_g_16,
+ v_b_16, v_r_16);
+ RGBTORGB8(vl, v_g_16, v_b_16, v_r_16, v_g, v_b, v_r);
+ v_dst_argb = __riscv_vcreate_v_u8m2x4(v_b, v_g, v_r, v_a);
+ __riscv_vsseg4e8_v_u8m2x4(dst_argb, v_dst_argb, vl);
+ w -= vl;
+ src_y += vl;
+ src_u += vl;
+ src_v += vl;
+ dst_argb += vl * 4;
+ } while (w > 0);
+}
+#else
void I444ToARGBRow_RVV(const uint8_t* src_y,
const uint8_t* src_u,
const uint8_t* src_v,
@@ -427,8 +796,43 @@
} while (w > 0);
}
#endif
+#endif
#ifdef HAS_I444ALPHATOARGBROW_RVV
+#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE
+void I444AlphaToARGBRow_RVV(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ const uint8_t* src_a,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ size_t vl;
+ size_t w = (size_t)width;
+ uint8_t ub, vr, ug, vg;
+ int16_t yg, bb, bg, br;
+ vuint8m2_t v_u, v_v;
+ vuint8m2_t v_b, v_g, v_r, v_a;
+ vuint16m4_t v_y_16, v_g_16, v_b_16, v_r_16;
+ YUVTORGB_SETUP(yuvconstants, ub, vr, ug, vg, yg, bb, bg, br);
+ do {
+ vuint8m2x4_t v_dst_argb;
+ READYUV444(vl, w, src_y, src_u, src_v, v_u, v_v, v_y_16);
+ v_a = __riscv_vle8_v_u8m2(src_a, vl);
+ YUVTORGB(vl, v_u, v_v, ub, vr, ug, vg, yg, bb, bg, br, v_y_16, v_g_16,
+ v_b_16, v_r_16);
+ RGBTORGB8(vl, v_g_16, v_b_16, v_r_16, v_g, v_b, v_r);
+ v_dst_argb = __riscv_vcreate_v_u8m2x4(v_b, v_g, v_r, v_a);
+ __riscv_vsseg4e8_v_u8m2x4(dst_argb, v_dst_argb, vl);
+ w -= vl;
+ src_y += vl;
+ src_a += vl;
+ src_u += vl;
+ src_v += vl;
+ dst_argb += vl * 4;
+ } while (w > 0);
+}
+#else
void I444AlphaToARGBRow_RVV(const uint8_t* src_y,
const uint8_t* src_u,
const uint8_t* src_v,
@@ -460,8 +864,40 @@
} while (w > 0);
}
#endif
+#endif
#ifdef HAS_I444TORGB24ROW_RVV
+#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE
+void I444ToRGB24Row_RVV(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ size_t vl;
+ size_t w = (size_t)width;
+ uint8_t ub, vr, ug, vg;
+ int16_t yg, bb, bg, br;
+ vuint8m2_t v_u, v_v;
+ vuint8m2_t v_b, v_g, v_r;
+ vuint16m4_t v_y_16, v_g_16, v_b_16, v_r_16;
+ YUVTORGB_SETUP(yuvconstants, ub, vr, ug, vg, yg, bb, bg, br);
+ do {
+ vuint8m2x3_t v_dst_rgb;
+ READYUV444(vl, w, src_y, src_u, src_v, v_u, v_v, v_y_16);
+ YUVTORGB(vl, v_u, v_v, ub, vr, ug, vg, yg, bb, bg, br, v_y_16, v_g_16,
+ v_b_16, v_r_16);
+ RGBTORGB8(vl, v_g_16, v_b_16, v_r_16, v_g, v_b, v_r);
+ v_dst_rgb = __riscv_vcreate_v_u8m2x3(v_b, v_g, v_r);
+ __riscv_vsseg3e8_v_u8m2x3(dst_rgb24, v_dst_rgb, vl);
+ w -= vl;
+ src_y += vl;
+ src_u += vl;
+ src_v += vl;
+ dst_rgb24 += vl * 3;
+ } while (w > 0);
+}
+#else
void I444ToRGB24Row_RVV(const uint8_t* src_y,
const uint8_t* src_u,
const uint8_t* src_v,
@@ -490,8 +926,41 @@
} while (w > 0);
}
#endif
+#endif
#ifdef HAS_I422TOARGBROW_RVV
+#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE
+void I422ToARGBRow_RVV(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ size_t w = (size_t)width;
+ size_t vl = __riscv_vsetvl_e8m2(w);
+ uint8_t ub, vr, ug, vg;
+ int16_t yg, bb, bg, br;
+ vuint8m2_t v_u, v_v;
+ vuint8m2_t v_b, v_g, v_r, v_a;
+ vuint16m4_t v_y_16, v_g_16, v_b_16, v_r_16;
+ vuint8m2x4_t v_dst_argb;
+ YUVTORGB_SETUP(yuvconstants, ub, vr, ug, vg, yg, bb, bg, br);
+ v_a = __riscv_vmv_v_x_u8m2(255u, vl);
+ do {
+ READYUV422(vl, w, src_y, src_u, src_v, v_u, v_v, v_y_16);
+ YUVTORGB(vl, v_u, v_v, ub, vr, ug, vg, yg, bb, bg, br, v_y_16, v_g_16,
+ v_b_16, v_r_16);
+ RGBTORGB8(vl, v_g_16, v_b_16, v_r_16, v_g, v_b, v_r);
+ v_dst_argb = __riscv_vcreate_v_u8m2x4(v_b, v_g, v_r, v_a);
+ __riscv_vsseg4e8_v_u8m2x4(dst_argb, v_dst_argb, vl);
+ w -= vl;
+ src_y += vl;
+ src_u += vl / 2;
+ src_v += vl / 2;
+ dst_argb += vl * 4;
+ } while (w > 0);
+}
+#else
void I422ToARGBRow_RVV(const uint8_t* src_y,
const uint8_t* src_u,
const uint8_t* src_v,
@@ -521,8 +990,43 @@
} while (w > 0);
}
#endif
+#endif
#ifdef HAS_I422ALPHATOARGBROW_RVV
+#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE
+void I422AlphaToARGBRow_RVV(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ const uint8_t* src_a,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ size_t vl;
+ size_t w = (size_t)width;
+ uint8_t ub, vr, ug, vg;
+ int16_t yg, bb, bg, br;
+ vuint8m2_t v_u, v_v;
+ vuint8m2_t v_b, v_g, v_r, v_a;
+ vuint16m4_t v_y_16, v_g_16, v_b_16, v_r_16;
+ YUVTORGB_SETUP(yuvconstants, ub, vr, ug, vg, yg, bb, bg, br);
+ do {
+ vuint8m2x4_t v_dst_argb;
+ READYUV422(vl, w, src_y, src_u, src_v, v_u, v_v, v_y_16);
+ v_a = __riscv_vle8_v_u8m2(src_a, vl);
+ YUVTORGB(vl, v_u, v_v, ub, vr, ug, vg, yg, bb, bg, br, v_y_16, v_g_16,
+ v_b_16, v_r_16);
+ RGBTORGB8(vl, v_g_16, v_b_16, v_r_16, v_g, v_b, v_r);
+ v_dst_argb = __riscv_vcreate_v_u8m2x4(v_b, v_g, v_r, v_a);
+ __riscv_vsseg4e8_v_u8m2x4(dst_argb, v_dst_argb, vl);
+ w -= vl;
+ src_y += vl;
+ src_a += vl;
+ src_u += vl / 2;
+ src_v += vl / 2;
+ dst_argb += vl * 4;
+ } while (w > 0);
+}
+#else
void I422AlphaToARGBRow_RVV(const uint8_t* src_y,
const uint8_t* src_u,
const uint8_t* src_v,
@@ -554,8 +1058,41 @@
} while (w > 0);
}
#endif
+#endif
#ifdef HAS_I422TORGBAROW_RVV
+#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE
+void I422ToRGBARow_RVV(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_rgba,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ size_t w = (size_t)width;
+ size_t vl = __riscv_vsetvl_e8m2(w);
+ uint8_t ub, vr, ug, vg;
+ int16_t yg, bb, bg, br;
+ vuint8m2_t v_u, v_v;
+ vuint8m2_t v_b, v_g, v_r, v_a;
+ vuint16m4_t v_y_16, v_g_16, v_b_16, v_r_16;
+ vuint8m2x4_t v_dst_rgba;
+ YUVTORGB_SETUP(yuvconstants, ub, vr, ug, vg, yg, bb, bg, br);
+ v_a = __riscv_vmv_v_x_u8m2(255u, vl);
+ do {
+ READYUV422(vl, w, src_y, src_u, src_v, v_u, v_v, v_y_16);
+ YUVTORGB(vl, v_u, v_v, ub, vr, ug, vg, yg, bb, bg, br, v_y_16, v_g_16,
+ v_b_16, v_r_16);
+ RGBTORGB8(vl, v_g_16, v_b_16, v_r_16, v_g, v_b, v_r);
+ v_dst_rgba = __riscv_vcreate_v_u8m2x4(v_a, v_b, v_g, v_r);
+ __riscv_vsseg4e8_v_u8m2x4(dst_rgba, v_dst_rgba, vl);
+ w -= vl;
+ src_y += vl;
+ src_u += vl / 2;
+ src_v += vl / 2;
+ dst_rgba += vl * 4;
+ } while (w > 0);
+}
+#else
void I422ToRGBARow_RVV(const uint8_t* src_y,
const uint8_t* src_u,
const uint8_t* src_v,
@@ -585,8 +1122,40 @@
} while (w > 0);
}
#endif
+#endif
#ifdef HAS_I422TORGB24ROW_RVV
+#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE
+void I422ToRGB24Row_RVV(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ size_t vl;
+ size_t w = (size_t)width;
+ uint8_t ub, vr, ug, vg;
+ int16_t yg, bb, bg, br;
+ vuint8m2_t v_u, v_v;
+ vuint8m2_t v_b, v_g, v_r;
+ vuint16m4_t v_y_16, v_g_16, v_b_16, v_r_16;
+ vuint8m2x3_t v_dst_rgb;
+ YUVTORGB_SETUP(yuvconstants, ub, vr, ug, vg, yg, bb, bg, br);
+ do {
+ READYUV422(vl, w, src_y, src_u, src_v, v_u, v_v, v_y_16);
+ YUVTORGB(vl, v_u, v_v, ub, vr, ug, vg, yg, bb, bg, br, v_y_16, v_g_16,
+ v_b_16, v_r_16);
+ RGBTORGB8(vl, v_g_16, v_b_16, v_r_16, v_g, v_b, v_r);
+ v_dst_rgb = __riscv_vcreate_v_u8m2x3(v_b, v_g, v_r);
+ __riscv_vsseg3e8_v_u8m2x3(dst_rgb24, v_dst_rgb, vl);
+ w -= vl;
+ src_y += vl;
+ src_u += vl / 2;
+ src_v += vl / 2;
+ dst_rgb24 += vl * 3;
+ } while (w > 0);
+}
+#else
void I422ToRGB24Row_RVV(const uint8_t* src_y,
const uint8_t* src_u,
const uint8_t* src_v,
@@ -615,8 +1184,48 @@
} while (w > 0);
}
#endif
+#endif
#ifdef HAS_I400TOARGBROW_RVV
+#if defined(LIBYUV_RVV_HAS_TUPLE_TYPE) && defined(LIBYUV_RVV_HAS_VXRM_ARG)
+void I400ToARGBRow_RVV(const uint8_t* src_y,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ size_t w = (size_t)width;
+ size_t vl = __riscv_vsetvl_e8m2(w);
+ const bool is_yb_positive = (yuvconstants->kRGBCoeffBias[4] >= 0);
+ vuint8m2_t v_a = __riscv_vmv_v_x_u8m2(255u, vl);
+ vuint16m4_t v_yg = __riscv_vmv_v_x_u16m4(yuvconstants->kRGBCoeffBias[0], vl);
+ vuint8m2x4_t v_dst_argb;
+ vuint16m4_t v_yb;
+ if (is_yb_positive) {
+ v_yb = __riscv_vmv_v_x_u16m4(yuvconstants->kRGBCoeffBias[4] - 32, vl);
+ } else {
+ v_yb = __riscv_vmv_v_x_u16m4(-yuvconstants->kRGBCoeffBias[4] + 32, vl);
+ }
+ do {
+ vuint8m2_t v_y, v_out;
+ vuint16m4_t v_y_16, v_tmp0, v_tmp1, v_tmp2;
+ vl = __riscv_vsetvl_e8m2(w);
+ v_y = __riscv_vle8_v_u8m2(src_y, vl);
+ v_y_16 = __riscv_vwaddu_vx_u16m4(v_y, 0, vl);
+ v_tmp0 = __riscv_vmul_vx_u16m4(v_y_16, 0x0101, vl); // 257 * v_y
+ v_tmp1 = __riscv_vmulhu_vv_u16m4(v_tmp0, v_yg, vl);
+ if (is_yb_positive) {
+ v_tmp2 = __riscv_vsaddu_vv_u16m4(v_tmp1, v_yb, vl);
+ } else {
+ v_tmp2 = __riscv_vssubu_vv_u16m4(v_tmp1, v_yb, vl);
+ }
+ v_out = __riscv_vnclipu_wx_u8m2(v_tmp2, 6, __RISCV_VXRM_RNU, vl);
+ v_dst_argb = __riscv_vcreate_v_u8m2x4(v_out, v_out, v_out, v_a);
+ __riscv_vsseg4e8_v_u8m2x4(dst_argb, v_dst_argb, vl);
+ w -= vl;
+ src_y += vl;
+ dst_argb += vl * 4;
+ } while (w > 0);
+}
+#else
void I400ToARGBRow_RVV(const uint8_t* src_y,
uint8_t* dst_argb,
const struct YuvConstants* yuvconstants,
@@ -656,8 +1265,25 @@
} while (w > 0);
}
#endif
+#endif
#ifdef HAS_J400TOARGBROW_RVV
+#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE
+void J400ToARGBRow_RVV(const uint8_t* src_y, uint8_t* dst_argb, int width) {
+ size_t w = (size_t)width;
+ size_t vl = __riscv_vsetvl_e8m2(w);
+ vuint8m2_t v_a = __riscv_vmv_v_x_u8m2(255u, vl);
+ do {
+ vuint8m2_t v_y = __riscv_vle8_v_u8m2(src_y, vl);
+ vuint8m2x4_t v_dst_argb = __riscv_vcreate_v_u8m2x4(v_y, v_y, v_y, v_a);
+ __riscv_vsseg4e8_v_u8m2x4(dst_argb, v_dst_argb, vl);
+ w -= vl;
+ src_y += vl;
+ dst_argb += vl * 4;
+ vl = __riscv_vsetvl_e8m2(w);
+ } while (w > 0);
+}
+#else
void J400ToARGBRow_RVV(const uint8_t* src_y, uint8_t* dst_argb, int width) {
size_t w = (size_t)width;
size_t vl = __riscv_vsetvl_e8m2(w);
@@ -673,6 +1299,7 @@
} while (w > 0);
}
#endif
+#endif
#ifdef HAS_COPYROW_RVV
void CopyRow_RVV(const uint8_t* src, uint8_t* dst, int width) {
@@ -689,6 +1316,36 @@
#endif
#ifdef HAS_NV12TOARGBROW_RVV
+#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE
+void NV12ToARGBRow_RVV(const uint8_t* src_y,
+ const uint8_t* src_uv,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ size_t w = (size_t)width;
+ size_t vl = __riscv_vsetvl_e8m2(w);
+ uint8_t ub, vr, ug, vg;
+ int16_t yg, bb, bg, br;
+ vuint8m2_t v_u, v_v;
+ vuint8m2_t v_b, v_g, v_r, v_a;
+ vuint16m4_t v_y_16, v_g_16, v_b_16, v_r_16;
+ vuint8m2x4_t v_dst_argb;
+ YUVTORGB_SETUP(yuvconstants, ub, vr, ug, vg, yg, bb, bg, br);
+ v_a = __riscv_vmv_v_x_u8m2(255u, vl);
+ do {
+ READNV12(vl, w, src_y, src_uv, v_u, v_v, v_y_16);
+ YUVTORGB(vl, v_u, v_v, ub, vr, ug, vg, yg, bb, bg, br, v_y_16, v_g_16,
+ v_b_16, v_r_16);
+ RGBTORGB8(vl, v_g_16, v_b_16, v_r_16, v_g, v_b, v_r);
+ v_dst_argb = __riscv_vcreate_v_u8m2x4(v_b, v_g, v_r, v_a);
+ __riscv_vsseg4e8_v_u8m2x4(dst_argb, v_dst_argb, vl);
+ w -= vl;
+ src_y += vl;
+ src_uv += vl;
+ dst_argb += vl * 4;
+ } while (w > 0);
+}
+#else
void NV12ToARGBRow_RVV(const uint8_t* src_y,
const uint8_t* src_uv,
uint8_t* dst_argb,
@@ -716,8 +1373,38 @@
} while (w > 0);
}
#endif
+#endif
#ifdef HAS_NV12TORGB24ROW_RVV
+#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE
+void NV12ToRGB24Row_RVV(const uint8_t* src_y,
+ const uint8_t* src_uv,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ size_t w = (size_t)width;
+ size_t vl = __riscv_vsetvl_e8m2(w);
+ uint8_t ub, vr, ug, vg;
+ int16_t yg, bb, bg, br;
+ vuint8m2_t v_u, v_v;
+ vuint8m2_t v_b, v_g, v_r;
+ vuint8m2x3_t v_dst_rgb;
+ vuint16m4_t v_y_16, v_g_16, v_b_16, v_r_16;
+ YUVTORGB_SETUP(yuvconstants, ub, vr, ug, vg, yg, bb, bg, br);
+ do {
+ READNV12(vl, w, src_y, src_uv, v_u, v_v, v_y_16);
+ YUVTORGB(vl, v_u, v_v, ub, vr, ug, vg, yg, bb, bg, br, v_y_16, v_g_16,
+ v_b_16, v_r_16);
+ RGBTORGB8(vl, v_g_16, v_b_16, v_r_16, v_g, v_b, v_r);
+ v_dst_rgb = __riscv_vcreate_v_u8m2x3(v_b, v_g, v_r);
+ __riscv_vsseg3e8_v_u8m2x3(dst_rgb24, v_dst_rgb, vl);
+ w -= vl;
+ src_y += vl;
+ src_uv += vl;
+ dst_rgb24 += vl * 3;
+ } while (w > 0);
+}
+#else
void NV12ToRGB24Row_RVV(const uint8_t* src_y,
const uint8_t* src_uv,
uint8_t* dst_rgb24,
@@ -744,8 +1431,39 @@
} while (w > 0);
}
#endif
+#endif
#ifdef HAS_NV21TOARGBROW_RVV
+#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE
+void NV21ToARGBRow_RVV(const uint8_t* src_y,
+ const uint8_t* src_vu,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ size_t w = (size_t)width;
+ size_t vl = __riscv_vsetvl_e8m2(w);
+ uint8_t ub, vr, ug, vg;
+ int16_t yg, bb, bg, br;
+ vuint8m2_t v_u, v_v;
+ vuint8m2_t v_b, v_g, v_r, v_a;
+ vuint8m2x4_t v_dst_argb;
+ vuint16m4_t v_y_16, v_g_16, v_b_16, v_r_16;
+ YUVTORGB_SETUP(yuvconstants, ub, vr, ug, vg, yg, bb, bg, br);
+ v_a = __riscv_vmv_v_x_u8m2(255u, vl);
+ do {
+ READNV21(vl, w, src_y, src_vu, v_u, v_v, v_y_16);
+ YUVTORGB(vl, v_u, v_v, ub, vr, ug, vg, yg, bb, bg, br, v_y_16, v_g_16,
+ v_b_16, v_r_16);
+ RGBTORGB8(vl, v_g_16, v_b_16, v_r_16, v_g, v_b, v_r);
+ v_dst_argb = __riscv_vcreate_v_u8m2x4(v_b, v_g, v_r, v_a);
+ __riscv_vsseg4e8_v_u8m2x4(dst_argb, v_dst_argb, vl);
+ w -= vl;
+ src_y += vl;
+ src_vu += vl;
+ dst_argb += vl * 4;
+ } while (w > 0);
+}
+#else
void NV21ToARGBRow_RVV(const uint8_t* src_y,
const uint8_t* src_vu,
uint8_t* dst_argb,
@@ -773,8 +1491,38 @@
} while (w > 0);
}
#endif
+#endif
#ifdef HAS_NV21TORGB24ROW_RVV
+#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE
+void NV21ToRGB24Row_RVV(const uint8_t* src_y,
+ const uint8_t* src_vu,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ size_t w = (size_t)width;
+ size_t vl = __riscv_vsetvl_e8m2(w);
+ uint8_t ub, vr, ug, vg;
+ int16_t yg, bb, bg, br;
+ vuint8m2_t v_u, v_v;
+ vuint8m2_t v_b, v_g, v_r;
+ vuint8m2x3_t v_dst_rgb;
+ vuint16m4_t v_y_16, v_g_16, v_b_16, v_r_16;
+ YUVTORGB_SETUP(yuvconstants, ub, vr, ug, vg, yg, bb, bg, br);
+ do {
+ READNV21(vl, w, src_y, src_vu, v_u, v_v, v_y_16);
+ YUVTORGB(vl, v_u, v_v, ub, vr, ug, vg, yg, bb, bg, br, v_y_16, v_g_16,
+ v_b_16, v_r_16);
+ RGBTORGB8(vl, v_g_16, v_b_16, v_r_16, v_g, v_b, v_r);
+ v_dst_rgb = __riscv_vcreate_v_u8m2x3(v_b, v_g, v_r);
+ __riscv_vsseg3e8_v_u8m2x3(dst_rgb24, v_dst_rgb, vl);
+ w -= vl;
+ src_y += vl;
+ src_vu += vl;
+ dst_rgb24 += vl * 3;
+ } while (w > 0);
+}
+#else
void NV21ToRGB24Row_RVV(const uint8_t* src_y,
const uint8_t* src_vu,
uint8_t* dst_rgb24,
@@ -801,10 +1549,65 @@
} while (w > 0);
}
#endif
+#endif
// Bilinear filter [VLEN/8]x2 -> [VLEN/8]x1
-
#ifdef HAS_INTERPOLATEROW_RVV
+#ifdef LIBYUV_RVV_HAS_VXRM_ARG
+void InterpolateRow_RVV(uint8_t* dst_ptr,
+ const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ int dst_width,
+ int source_y_fraction) {
+ int y1_fraction = source_y_fraction;
+ int y0_fraction = 256 - y1_fraction;
+ const uint8_t* src_ptr1 = src_ptr + src_stride;
+ size_t dst_w = (size_t)dst_width;
+ assert(source_y_fraction >= 0);
+ assert(source_y_fraction < 256);
+ // Blend 100 / 0 - Copy row unchanged.
+ if (y1_fraction == 0) {
+ do {
+ size_t vl = __riscv_vsetvl_e8m8(dst_w);
+ __riscv_vse8_v_u8m8(dst_ptr, __riscv_vle8_v_u8m8(src_ptr, vl), vl);
+ dst_w -= vl;
+ src_ptr += vl;
+ dst_ptr += vl;
+ } while (dst_w > 0);
+ return;
+ }
+ // Blend 50 / 50.
+ if (y1_fraction == 128) {
+ do {
+ size_t vl = __riscv_vsetvl_e8m8(dst_w);
+ vuint8m8_t row0 = __riscv_vle8_v_u8m8(src_ptr, vl);
+ vuint8m8_t row1 = __riscv_vle8_v_u8m8(src_ptr1, vl);
+ vuint8m8_t row_out =
+ __riscv_vaaddu_vv_u8m8(row0, row1, __RISCV_VXRM_RNU, vl);
+ __riscv_vse8_v_u8m8(dst_ptr, row_out, vl);
+ dst_w -= vl;
+ src_ptr += vl;
+ src_ptr1 += vl;
+ dst_ptr += vl;
+ } while (dst_w > 0);
+ return;
+ }
+ // General purpose row blend.
+ do {
+ size_t vl = __riscv_vsetvl_e8m4(dst_w);
+ vuint8m4_t row0 = __riscv_vle8_v_u8m4(src_ptr, vl);
+ vuint16m8_t acc = __riscv_vwmulu_vx_u16m8(row0, y0_fraction, vl);
+ vuint8m4_t row1 = __riscv_vle8_v_u8m4(src_ptr1, vl);
+ acc = __riscv_vwmaccu_vx_u16m8(acc, y1_fraction, row1, vl);
+ __riscv_vse8_v_u8m4(
+ dst_ptr, __riscv_vnclipu_wx_u8m4(acc, 8, __RISCV_VXRM_RNU, vl), vl);
+ dst_w -= vl;
+ src_ptr += vl;
+ src_ptr1 += vl;
+ dst_ptr += vl;
+ } while (dst_w > 0);
+}
+#else
void InterpolateRow_RVV(uint8_t* dst_ptr,
const uint8_t* src_ptr,
ptrdiff_t src_stride,
@@ -862,8 +1665,33 @@
} while (dst_w > 0);
}
#endif
+#endif
#ifdef HAS_SPLITRGBROW_RVV
+#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE
+void SplitRGBRow_RVV(const uint8_t* src_rgb,
+ uint8_t* dst_r,
+ uint8_t* dst_g,
+ uint8_t* dst_b,
+ int width) {
+ size_t w = (size_t)width;
+ do {
+ size_t vl = __riscv_vsetvl_e8m2(w);
+ vuint8m2x3_t v_src = __riscv_vlseg3e8_v_u8m2x3(src_rgb, vl);
+ vuint8m2_t v_r = __riscv_vget_v_u8m2x3_u8m2(v_src, 0);
+ vuint8m2_t v_g = __riscv_vget_v_u8m2x3_u8m2(v_src, 1);
+ vuint8m2_t v_b = __riscv_vget_v_u8m2x3_u8m2(v_src, 2);
+ __riscv_vse8_v_u8m2(dst_r, v_r, vl);
+ __riscv_vse8_v_u8m2(dst_g, v_g, vl);
+ __riscv_vse8_v_u8m2(dst_b, v_b, vl);
+ w -= vl;
+ dst_r += vl;
+ dst_g += vl;
+ dst_b += vl;
+ src_rgb += vl * 3;
+ } while (w > 0);
+}
+#else
void SplitRGBRow_RVV(const uint8_t* src_rgb,
uint8_t* dst_r,
uint8_t* dst_g,
@@ -885,8 +1713,31 @@
} while (w > 0);
}
#endif
+#endif
#ifdef HAS_MERGERGBROW_RVV
+#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE
+void MergeRGBRow_RVV(const uint8_t* src_r,
+ const uint8_t* src_g,
+ const uint8_t* src_b,
+ uint8_t* dst_rgb,
+ int width) {
+ size_t w = (size_t)width;
+ do {
+ size_t vl = __riscv_vsetvl_e8m2(w);
+ vuint8m2_t v_r = __riscv_vle8_v_u8m2(src_r, vl);
+ vuint8m2_t v_g = __riscv_vle8_v_u8m2(src_g, vl);
+ vuint8m2_t v_b = __riscv_vle8_v_u8m2(src_b, vl);
+ vuint8m2x3_t v_dst = __riscv_vcreate_v_u8m2x3(v_r, v_g, v_b);
+ __riscv_vsseg3e8_v_u8m2x3(dst_rgb, v_dst, vl);
+ w -= vl;
+ src_r += vl;
+ src_g += vl;
+ src_b += vl;
+ dst_rgb += vl * 3;
+ } while (w > 0);
+}
+#else
void MergeRGBRow_RVV(const uint8_t* src_r,
const uint8_t* src_g,
const uint8_t* src_b,
@@ -907,8 +1758,37 @@
} while (w > 0);
}
#endif
+#endif
#ifdef HAS_SPLITARGBROW_RVV
+#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE
+void SplitARGBRow_RVV(const uint8_t* src_argb,
+ uint8_t* dst_r,
+ uint8_t* dst_g,
+ uint8_t* dst_b,
+ uint8_t* dst_a,
+ int width) {
+ size_t w = (size_t)width;
+ do {
+ size_t vl = __riscv_vsetvl_e8m2(w);
+ vuint8m2x4_t v_src = __riscv_vlseg4e8_v_u8m2x4(src_argb, vl);
+ vuint8m2_t v_b = __riscv_vget_v_u8m2x4_u8m2(v_src, 0);
+ vuint8m2_t v_g = __riscv_vget_v_u8m2x4_u8m2(v_src, 1);
+ vuint8m2_t v_r = __riscv_vget_v_u8m2x4_u8m2(v_src, 2);
+ vuint8m2_t v_a = __riscv_vget_v_u8m2x4_u8m2(v_src, 3);
+ __riscv_vse8_v_u8m2(dst_a, v_a, vl);
+ __riscv_vse8_v_u8m2(dst_r, v_r, vl);
+ __riscv_vse8_v_u8m2(dst_g, v_g, vl);
+ __riscv_vse8_v_u8m2(dst_b, v_b, vl);
+ w -= vl;
+ dst_a += vl;
+ dst_r += vl;
+ dst_g += vl;
+ dst_b += vl;
+ src_argb += vl * 4;
+ } while (w > 0);
+}
+#else
void SplitARGBRow_RVV(const uint8_t* src_argb,
uint8_t* dst_r,
uint8_t* dst_g,
@@ -933,8 +1813,34 @@
} while (w > 0);
}
#endif
+#endif
#ifdef HAS_MERGEARGBROW_RVV
+#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE
+void MergeARGBRow_RVV(const uint8_t* src_r,
+ const uint8_t* src_g,
+ const uint8_t* src_b,
+ const uint8_t* src_a,
+ uint8_t* dst_argb,
+ int width) {
+ size_t w = (size_t)width;
+ do {
+ size_t vl = __riscv_vsetvl_e8m2(w);
+ vuint8m2_t v_r = __riscv_vle8_v_u8m2(src_r, vl);
+ vuint8m2_t v_g = __riscv_vle8_v_u8m2(src_g, vl);
+ vuint8m2_t v_b = __riscv_vle8_v_u8m2(src_b, vl);
+ vuint8m2_t v_a = __riscv_vle8_v_u8m2(src_a, vl);
+ vuint8m2x4_t v_dst = __riscv_vcreate_v_u8m2x4(v_b, v_g, v_r, v_a);
+ __riscv_vsseg4e8_v_u8m2x4(dst_argb, v_dst, vl);
+ w -= vl;
+ src_r += vl;
+ src_g += vl;
+ src_b += vl;
+ src_a += vl;
+ dst_argb += vl * 4;
+ } while (w > 0);
+}
+#else
void MergeARGBRow_RVV(const uint8_t* src_r,
const uint8_t* src_g,
const uint8_t* src_b,
@@ -958,8 +1864,33 @@
} while (w > 0);
}
#endif
+#endif
#ifdef HAS_SPLITXRGBROW_RVV
+#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE
+void SplitXRGBRow_RVV(const uint8_t* src_argb,
+ uint8_t* dst_r,
+ uint8_t* dst_g,
+ uint8_t* dst_b,
+ int width) {
+ size_t w = (size_t)width;
+ do {
+ size_t vl = __riscv_vsetvl_e8m2(w);
+ vuint8m2x4_t v_src = __riscv_vlseg4e8_v_u8m2x4(src_argb, vl);
+ vuint8m2_t v_b = __riscv_vget_v_u8m2x4_u8m2(v_src, 0);
+ vuint8m2_t v_g = __riscv_vget_v_u8m2x4_u8m2(v_src, 1);
+ vuint8m2_t v_r = __riscv_vget_v_u8m2x4_u8m2(v_src, 2);
+ __riscv_vse8_v_u8m2(dst_r, v_r, vl);
+ __riscv_vse8_v_u8m2(dst_g, v_g, vl);
+ __riscv_vse8_v_u8m2(dst_b, v_b, vl);
+ w -= vl;
+ dst_r += vl;
+ dst_g += vl;
+ dst_b += vl;
+ src_argb += vl * 4;
+ } while (w > 0);
+}
+#else
void SplitXRGBRow_RVV(const uint8_t* src_argb,
uint8_t* dst_r,
uint8_t* dst_g,
@@ -981,8 +1912,33 @@
} while (w > 0);
}
#endif
+#endif
#ifdef HAS_MERGEXRGBROW_RVV
+#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE
+void MergeXRGBRow_RVV(const uint8_t* src_r,
+ const uint8_t* src_g,
+ const uint8_t* src_b,
+ uint8_t* dst_argb,
+ int width) {
+ size_t w = (size_t)width;
+ size_t vl = __riscv_vsetvl_e8m2(w);
+ vuint8m2_t v_a = __riscv_vmv_v_x_u8m2(255u, vl);
+ do {
+ vuint8m2_t v_r = __riscv_vle8_v_u8m2(src_r, vl);
+ vuint8m2_t v_g = __riscv_vle8_v_u8m2(src_g, vl);
+ vuint8m2_t v_b = __riscv_vle8_v_u8m2(src_b, vl);
+ vuint8m2x4_t v_dst = __riscv_vcreate_v_u8m2x4(v_b, v_g, v_r, v_a);
+ __riscv_vsseg4e8_v_u8m2x4(dst_argb, v_dst, vl);
+ w -= vl;
+ src_r += vl;
+ src_g += vl;
+ src_b += vl;
+ dst_argb += vl * 4;
+ vl = __riscv_vsetvl_e8m2(w);
+ } while (w > 0);
+}
+#else
void MergeXRGBRow_RVV(const uint8_t* src_r,
const uint8_t* src_g,
const uint8_t* src_b,
@@ -1006,8 +1962,29 @@
} while (w > 0);
}
#endif
+#endif
#ifdef HAS_SPLITUVROW_RVV
+#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE
+void SplitUVRow_RVV(const uint8_t* src_uv,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ size_t w = (size_t)width;
+ do {
+ size_t vl = __riscv_vsetvl_e8m4(w);
+ vuint8m4x2_t v_src = __riscv_vlseg2e8_v_u8m4x2(src_uv, vl);
+ vuint8m4_t v_u = __riscv_vget_v_u8m4x2_u8m4(v_src, 0);
+ vuint8m4_t v_v = __riscv_vget_v_u8m4x2_u8m4(v_src, 1);
+ __riscv_vse8_v_u8m4(dst_u, v_u, vl);
+ __riscv_vse8_v_u8m4(dst_v, v_v, vl);
+ w -= vl;
+ dst_u += vl;
+ dst_v += vl;
+ src_uv += 2 * vl;
+ } while (w > 0);
+}
+#else
void SplitUVRow_RVV(const uint8_t* src_uv,
uint8_t* dst_u,
uint8_t* dst_v,
@@ -1026,8 +2003,28 @@
} while (w > 0);
}
#endif
+#endif
#ifdef HAS_MERGEUVROW_RVV
+#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE
+void MergeUVRow_RVV(const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_uv,
+ int width) {
+ size_t w = (size_t)width;
+ do {
+ size_t vl = __riscv_vsetvl_e8m4(w);
+ vuint8m4_t v_u = __riscv_vle8_v_u8m4(src_u, vl);
+ vuint8m4_t v_v = __riscv_vle8_v_u8m4(src_v, vl);
+ vuint8m4x2_t v_dst = __riscv_vcreate_v_u8m4x2(v_u, v_v);
+ __riscv_vsseg2e8_v_u8m4x2(dst_uv, v_dst, vl);
+ w -= vl;
+ src_u += vl;
+ src_v += vl;
+ dst_uv += 2 * vl;
+ } while (w > 0);
+}
+#else
void MergeUVRow_RVV(const uint8_t* src_u,
const uint8_t* src_v,
uint8_t* dst_uv,
@@ -1046,6 +2043,7 @@
} while (w > 0);
}
#endif
+#endif
struct RgbConstants {
uint8_t kRGBToY[4];
@@ -1080,10 +2078,44 @@
// ARGB expects first 3 values to contain RGB and 4th value is ignored
#ifdef HAS_ARGBTOYMATRIXROW_RVV
-void ARGBToYMatrixRow_RVV(const uint8_t* src_argb,
- uint8_t* dst_y,
- int width,
- const struct RgbConstants* rgbconstants) {
+#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE
+static void ARGBToYMatrixRow_RVV(const uint8_t* src_argb,
+ uint8_t* dst_y,
+ int width,
+ const struct RgbConstants* rgbconstants) {
+ assert(width != 0);
+ size_t w = (size_t)width;
+ vuint8m2_t v_by, v_gy, v_ry; // vectors are to store RGBToY constant
+ vuint16m4_t v_addy; // vector is to store kAddY
+ size_t vl = __riscv_vsetvl_e8m2(w);
+ v_by = __riscv_vmv_v_x_u8m2(rgbconstants->kRGBToY[0], vl);
+ v_gy = __riscv_vmv_v_x_u8m2(rgbconstants->kRGBToY[1], vl);
+ v_ry = __riscv_vmv_v_x_u8m2(rgbconstants->kRGBToY[2], vl);
+ v_addy = __riscv_vmv_v_x_u16m4(rgbconstants->kAddY, vl);
+ do {
+ vuint8m2_t v_y;
+ vuint16m4_t v_y_u16;
+ size_t vl = __riscv_vsetvl_e8m2(w);
+ vuint8m2x4_t v_src_argb = __riscv_vlseg4e8_v_u8m2x4(src_argb, vl);
+ vuint8m2_t v_b = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 0);
+ vuint8m2_t v_g = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 1);
+ vuint8m2_t v_r = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 2);
+ v_y_u16 = __riscv_vwmulu_vv_u16m4(v_r, v_ry, vl);
+ v_y_u16 = __riscv_vwmaccu_vv_u16m4(v_y_u16, v_gy, v_g, vl);
+ v_y_u16 = __riscv_vwmaccu_vv_u16m4(v_y_u16, v_by, v_b, vl);
+ v_y_u16 = __riscv_vadd_vv_u16m4(v_y_u16, v_addy, vl);
+ v_y = __riscv_vnsrl_wx_u8m2(v_y_u16, 8, vl);
+ __riscv_vse8_v_u8m2(dst_y, v_y, vl);
+ w -= vl;
+ src_argb += 4 * vl;
+ dst_y += vl;
+ } while (w > 0);
+}
+#else
+static void ARGBToYMatrixRow_RVV(const uint8_t* src_argb,
+ uint8_t* dst_y,
+ int width,
+ const struct RgbConstants* rgbconstants) {
assert(width != 0);
size_t w = (size_t)width;
vuint8m2_t v_by, v_gy, v_ry; // vectors are to store RGBToY constant
@@ -1110,6 +2142,7 @@
} while (w > 0);
}
#endif
+#endif
#ifdef HAS_ARGBTOYROW_RVV
void ARGBToYRow_RVV(const uint8_t* src_argb, uint8_t* dst_y, int width) {
@@ -1137,10 +2170,44 @@
// RGBA expects first value to be A and ignored, then 3 values to contain RGB.
#ifdef HAS_RGBATOYMATRIXROW_RVV
-void RGBAToYMatrixRow_RVV(const uint8_t* src_rgba,
- uint8_t* dst_y,
- int width,
- const struct RgbConstants* rgbconstants) {
+#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE
+static void RGBAToYMatrixRow_RVV(const uint8_t* src_rgba,
+ uint8_t* dst_y,
+ int width,
+ const struct RgbConstants* rgbconstants) {
+ assert(width != 0);
+ size_t w = (size_t)width;
+ vuint8m2_t v_by, v_gy, v_ry; // vectors are to store RGBToY constant
+ vuint16m4_t v_addy; // vector is to store kAddY
+ size_t vl = __riscv_vsetvl_e8m2(w);
+ v_by = __riscv_vmv_v_x_u8m2(rgbconstants->kRGBToY[0], vl);
+ v_gy = __riscv_vmv_v_x_u8m2(rgbconstants->kRGBToY[1], vl);
+ v_ry = __riscv_vmv_v_x_u8m2(rgbconstants->kRGBToY[2], vl);
+ v_addy = __riscv_vmv_v_x_u16m4(rgbconstants->kAddY, vl);
+ do {
+ vuint8m2_t v_y;
+ vuint16m4_t v_y_u16;
+ size_t vl = __riscv_vsetvl_e8m2(w);
+ vuint8m2x4_t v_src_rgba = __riscv_vlseg4e8_v_u8m2x4(src_rgba, vl);
+ vuint8m2_t v_b = __riscv_vget_v_u8m2x4_u8m2(v_src_rgba, 1);
+ vuint8m2_t v_g = __riscv_vget_v_u8m2x4_u8m2(v_src_rgba, 2);
+ vuint8m2_t v_r = __riscv_vget_v_u8m2x4_u8m2(v_src_rgba, 3);
+ v_y_u16 = __riscv_vwmulu_vv_u16m4(v_r, v_ry, vl);
+ v_y_u16 = __riscv_vwmaccu_vv_u16m4(v_y_u16, v_gy, v_g, vl);
+ v_y_u16 = __riscv_vwmaccu_vv_u16m4(v_y_u16, v_by, v_b, vl);
+ v_y_u16 = __riscv_vadd_vv_u16m4(v_y_u16, v_addy, vl);
+ v_y = __riscv_vnsrl_wx_u8m2(v_y_u16, 8, vl);
+ __riscv_vse8_v_u8m2(dst_y, v_y, vl);
+ w -= vl;
+ src_rgba += 4 * vl;
+ dst_y += vl;
+ } while (w > 0);
+}
+#else
+static void RGBAToYMatrixRow_RVV(const uint8_t* src_rgba,
+ uint8_t* dst_y,
+ int width,
+ const struct RgbConstants* rgbconstants) {
assert(width != 0);
size_t w = (size_t)width;
vuint8m2_t v_by, v_gy, v_ry; // vectors are to store RGBToY constant
@@ -1167,6 +2234,7 @@
} while (w > 0);
}
#endif
+#endif
#ifdef HAS_RGBATOYROW_RVV
void RGBAToYRow_RVV(const uint8_t* src_rgba, uint8_t* dst_y, int width) {
@@ -1187,10 +2255,44 @@
#endif
#ifdef HAS_RGBTOYMATRIXROW_RVV
-void RGBToYMatrixRow_RVV(const uint8_t* src_rgb,
- uint8_t* dst_y,
- int width,
- const struct RgbConstants* rgbconstants) {
+#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE
+static void RGBToYMatrixRow_RVV(const uint8_t* src_rgb,
+ uint8_t* dst_y,
+ int width,
+ const struct RgbConstants* rgbconstants) {
+ assert(width != 0);
+ size_t w = (size_t)width;
+ vuint8m2_t v_by, v_gy, v_ry; // vectors are to store RGBToY constant
+ vuint16m4_t v_addy; // vector is to store kAddY
+ size_t vl = __riscv_vsetvl_e8m2(w);
+ v_by = __riscv_vmv_v_x_u8m2(rgbconstants->kRGBToY[0], vl);
+ v_gy = __riscv_vmv_v_x_u8m2(rgbconstants->kRGBToY[1], vl);
+ v_ry = __riscv_vmv_v_x_u8m2(rgbconstants->kRGBToY[2], vl);
+ v_addy = __riscv_vmv_v_x_u16m4(rgbconstants->kAddY, vl);
+ do {
+ vuint8m2_t v_y;
+ vuint16m4_t v_y_u16;
+ size_t vl = __riscv_vsetvl_e8m2(w);
+ vuint8m2x3_t v_src_rgb = __riscv_vlseg3e8_v_u8m2x3(src_rgb, vl);
+ vuint8m2_t v_b = __riscv_vget_v_u8m2x3_u8m2(v_src_rgb, 0);
+ vuint8m2_t v_g = __riscv_vget_v_u8m2x3_u8m2(v_src_rgb, 1);
+ vuint8m2_t v_r = __riscv_vget_v_u8m2x3_u8m2(v_src_rgb, 2);
+ v_y_u16 = __riscv_vwmulu_vv_u16m4(v_r, v_ry, vl);
+ v_y_u16 = __riscv_vwmaccu_vv_u16m4(v_y_u16, v_gy, v_g, vl);
+ v_y_u16 = __riscv_vwmaccu_vv_u16m4(v_y_u16, v_by, v_b, vl);
+ v_y_u16 = __riscv_vadd_vv_u16m4(v_y_u16, v_addy, vl);
+ v_y = __riscv_vnsrl_wx_u8m2(v_y_u16, 8, vl);
+ __riscv_vse8_v_u8m2(dst_y, v_y, vl);
+ w -= vl;
+ src_rgb += 3 * vl;
+ dst_y += vl;
+ } while (w > 0);
+}
+#else
+static void RGBToYMatrixRow_RVV(const uint8_t* src_rgb,
+ uint8_t* dst_y,
+ int width,
+ const struct RgbConstants* rgbconstants) {
assert(width != 0);
size_t w = (size_t)width;
vuint8m2_t v_by, v_gy, v_ry; // vectors are to store RGBToY constant
@@ -1217,6 +2319,7 @@
} while (w > 0);
}
#endif
+#endif
#ifdef HAS_RGB24TOYJROW_RVV
void RGB24ToYJRow_RVV(const uint8_t* src_rgb24, uint8_t* dst_yj, int width) {
@@ -1246,6 +2349,54 @@
// dst_argb may be src_argb or src_argb1.
// src_argb: RGB values have already been pre-multiplied by the a.
#ifdef HAS_ARGBBLENDROW_RVV
+#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE
+void ARGBBlendRow_RVV(const uint8_t* src_argb,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width) {
+ size_t w = (size_t)width;
+ size_t vl = __riscv_vsetvlmax_e8m2();
+ // clamp255((((256 - a) * b) >> 8) + f)
+ // = b * (256 - a) / 256 + f
+ // = b - (b * a / 256) + f
+ vuint8m2_t v_255 = __riscv_vmv_v_x_u8m2(255, vl);
+ do {
+ vuint8m2_t v_tmp_b, v_tmp_g, v_tmp_r;
+ vuint8m2_t v_dst_b, v_dst_g, v_dst_r;
+ vuint8m2x4_t v_dst_argb;
+ size_t vl = __riscv_vsetvl_e8m2(w);
+ vuint8m2x4_t v_src0_argb = __riscv_vlseg4e8_v_u8m2x4(src_argb, vl);
+ vuint8m2_t v_src0_b = __riscv_vget_v_u8m2x4_u8m2(v_src0_argb, 0);
+ vuint8m2_t v_src0_g = __riscv_vget_v_u8m2x4_u8m2(v_src0_argb, 1);
+ vuint8m2_t v_src0_r = __riscv_vget_v_u8m2x4_u8m2(v_src0_argb, 2);
+ vuint8m2_t v_src0_a = __riscv_vget_v_u8m2x4_u8m2(v_src0_argb, 3);
+ vuint8m2x4_t v_src1_argb = __riscv_vlseg4e8_v_u8m2x4(src_argb1, vl);
+ vuint8m2_t v_src1_b = __riscv_vget_v_u8m2x4_u8m2(v_src1_argb, 0);
+ vuint8m2_t v_src1_g = __riscv_vget_v_u8m2x4_u8m2(v_src1_argb, 1);
+ vuint8m2_t v_src1_r = __riscv_vget_v_u8m2x4_u8m2(v_src1_argb, 2);
+
+ v_tmp_b = __riscv_vmulhu_vv_u8m2(v_src1_b, v_src0_a, vl);
+ v_tmp_g = __riscv_vmulhu_vv_u8m2(v_src1_g, v_src0_a, vl);
+ v_tmp_r = __riscv_vmulhu_vv_u8m2(v_src1_r, v_src0_a, vl);
+
+ v_dst_b = __riscv_vsub_vv_u8m2(v_src1_b, v_tmp_b, vl);
+ v_dst_g = __riscv_vsub_vv_u8m2(v_src1_g, v_tmp_g, vl);
+ v_dst_r = __riscv_vsub_vv_u8m2(v_src1_r, v_tmp_r, vl);
+
+ v_dst_b = __riscv_vsaddu_vv_u8m2(v_dst_b, v_src0_b, vl);
+ v_dst_g = __riscv_vsaddu_vv_u8m2(v_dst_g, v_src0_g, vl);
+ v_dst_r = __riscv_vsaddu_vv_u8m2(v_dst_r, v_src0_r, vl);
+
+ v_dst_argb = __riscv_vcreate_v_u8m2x4(v_dst_b, v_dst_g, v_dst_r, v_255);
+ __riscv_vsseg4e8_v_u8m2x4(dst_argb, v_dst_argb, vl);
+
+ w -= vl;
+ src_argb += 4 * vl;
+ src_argb1 += 4 * vl;
+ dst_argb += 4 * vl;
+ } while (w > 0);
+}
+#else
void ARGBBlendRow_RVV(const uint8_t* src_argb,
const uint8_t* src_argb1,
uint8_t* dst_argb,
@@ -1287,6 +2438,7 @@
} while (w > 0);
}
#endif
+#endif
#ifdef HAS_BLENDPLANEROW_RVV
void BlendPlaneRow_RVV(const uint8_t* src0,
@@ -1323,6 +2475,41 @@
// Attenuate: (f * a + 255) >> 8
#ifdef HAS_ARGBATTENUATEROW_RVV
+#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE
+void ARGBAttenuateRow_RVV(const uint8_t* src_argb,
+ uint8_t* dst_argb,
+ int width) {
+ size_t w = (size_t)width;
+ do {
+ vuint16m4_t v_ba_16, v_ga_16, v_ra_16;
+ vuint8m2x4_t v_dst_argb;
+ size_t vl = __riscv_vsetvl_e8m2(w);
+ vuint8m2x4_t v_src_argb = __riscv_vlseg4e8_v_u8m2x4(src_argb, vl);
+ vuint8m2_t v_b = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 0);
+ vuint8m2_t v_g = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 1);
+ vuint8m2_t v_r = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 2);
+ vuint8m2_t v_a = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 3);
+ // f * a
+ v_ba_16 = __riscv_vwmulu_vv_u16m4(v_b, v_a, vl);
+ v_ga_16 = __riscv_vwmulu_vv_u16m4(v_g, v_a, vl);
+ v_ra_16 = __riscv_vwmulu_vv_u16m4(v_r, v_a, vl);
+ // f * a + 255
+ v_ba_16 = __riscv_vadd_vx_u16m4(v_ba_16, 255u, vl);
+ v_ga_16 = __riscv_vadd_vx_u16m4(v_ga_16, 255u, vl);
+ v_ra_16 = __riscv_vadd_vx_u16m4(v_ra_16, 255u, vl);
+ // (f * a + 255) >> 8
+ v_b = __riscv_vnsrl_wx_u8m2(v_ba_16, 8, vl);
+ v_g = __riscv_vnsrl_wx_u8m2(v_ga_16, 8, vl);
+ v_r = __riscv_vnsrl_wx_u8m2(v_ra_16, 8, vl);
+
+ v_dst_argb = __riscv_vcreate_v_u8m2x4(v_b, v_g, v_r, v_a);
+ __riscv_vsseg4e8_v_u8m2x4(dst_argb, v_dst_argb, vl);
+ w -= vl;
+ src_argb += vl * 4;
+ dst_argb += vl * 4;
+ } while (w > 0);
+}
+#else
void ARGBAttenuateRow_RVV(const uint8_t* src_argb,
uint8_t* dst_argb,
int width) {
@@ -1351,8 +2538,25 @@
} while (w > 0);
}
#endif
+#endif
#ifdef HAS_ARGBEXTRACTALPHAROW_RVV
+#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE
+void ARGBExtractAlphaRow_RVV(const uint8_t* src_argb,
+ uint8_t* dst_a,
+ int width) {
+ size_t w = (size_t)width;
+ do {
+ size_t vl = __riscv_vsetvl_e8m2(w);
+ vuint8m2x4_t v_src_argb = __riscv_vlseg4e8_v_u8m2x4(src_argb, vl);
+ vuint8m2_t v_a = __riscv_vget_v_u8m2x4_u8m2(v_src_argb, 3);
+ __riscv_vse8_v_u8m2(dst_a, v_a, vl);
+ w -= vl;
+ src_argb += vl * 4;
+ dst_a += vl;
+ } while (w > 0);
+}
+#else
void ARGBExtractAlphaRow_RVV(const uint8_t* src_argb,
uint8_t* dst_a,
int width) {
@@ -1368,6 +2572,7 @@
} while (w > 0);
}
#endif
+#endif
#ifdef HAS_ARGBCOPYYTOALPHAROW_RVV
void ARGBCopyYToAlphaRow_RVV(const uint8_t* src, uint8_t* dst, int width) {
diff --git a/source/row_sme.cc b/source/row_sme.cc
new file mode 100644
index 0000000..1cbc42f
--- /dev/null
+++ b/source/row_sme.cc
@@ -0,0 +1,1092 @@
+/*
+ * Copyright 2024 The LibYuv Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "libyuv/row.h"
+#include "libyuv/row_sve.h"
+
+#ifdef __cplusplus
+namespace libyuv {
+extern "C" {
+#endif
+
+#if !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && \
+ defined(__aarch64__)
+
+// Read twice as much data from YUV, putting the even elements from the Y data
+// in z0.h and odd elements in z1.h.
+#define READYUV444_SVE_2X \
+ "ld1b {z0.b}, p1/z, [%[src_y]] \n" \
+ "ld1b {z2.b}, p1/z, [%[src_u]] \n" \
+ "ld1b {z3.b}, p1/z, [%[src_v]] \n" \
+ "incb %[src_y] \n" \
+ "incb %[src_u] \n" \
+ "incb %[src_v] \n" \
+ "prfm pldl1keep, [%[src_y], 448] \n" \
+ "prfm pldl1keep, [%[src_u], 128] \n" \
+ "prfm pldl1keep, [%[src_v], 128] \n" \
+ "trn2 z1.b, z0.b, z0.b \n" \
+ "trn1 z0.b, z0.b, z0.b \n"
+
+#define I444TORGB_SVE_2X \
+ "umulh z0.h, z24.h, z0.h \n" /* Y0 */ \
+ "umulh z1.h, z24.h, z1.h \n" /* Y1 */ \
+ "umullb z6.h, z30.b, z2.b \n" \
+ "umullt z7.h, z30.b, z2.b \n" \
+ "umullb z4.h, z28.b, z2.b \n" /* DB */ \
+ "umullt z2.h, z28.b, z2.b \n" /* DB */ \
+ "umlalb z6.h, z31.b, z3.b \n" /* DG */ \
+ "umlalt z7.h, z31.b, z3.b \n" /* DG */ \
+ "umullb z5.h, z29.b, z3.b \n" /* DR */ \
+ "umullt z3.h, z29.b, z3.b \n" /* DR */ \
+ "add z17.h, z0.h, z26.h \n" /* G */ \
+ "add z21.h, z1.h, z26.h \n" /* G */ \
+ "add z16.h, z0.h, z4.h \n" /* B */ \
+ "add z20.h, z1.h, z2.h \n" /* B */ \
+ "add z18.h, z0.h, z5.h \n" /* R */ \
+ "add z22.h, z1.h, z3.h \n" /* R */ \
+ "uqsub z17.h, z17.h, z6.h \n" /* G */ \
+ "uqsub z21.h, z21.h, z7.h \n" /* G */ \
+ "uqsub z16.h, z16.h, z25.h \n" /* B */ \
+ "uqsub z20.h, z20.h, z25.h \n" /* B */ \
+ "uqsub z18.h, z18.h, z27.h \n" /* R */ \
+ "uqsub z22.h, z22.h, z27.h \n" /* R */
+
+#define RGBTOARGB8_SVE_2X \
+ /* Inputs: B: z16.h, G: z17.h, R: z18.h, A: z19.b */ \
+ "uqshrnb z16.b, z16.h, #6 \n" /* B0 */ \
+ "uqshrnb z17.b, z17.h, #6 \n" /* G0 */ \
+ "uqshrnb z18.b, z18.h, #6 \n" /* R0 */ \
+ "uqshrnt z16.b, z20.h, #6 \n" /* B1 */ \
+ "uqshrnt z17.b, z21.h, #6 \n" /* G1 */ \
+ "uqshrnt z18.b, z22.h, #6 \n" /* R1 */
+
+__arm_locally_streaming void I444ToARGBRow_SME(
+ const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ // Streaming-SVE only, no use of ZA tile.
+ uint64_t vl;
+ asm volatile(
+ "cntb %[vl] \n"
+ "ptrue p0.b \n" //
+ YUVTORGB_SVE_SETUP
+ "dup z19.b, #255 \n" // A
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.b \n"
+ "1: \n" //
+ READYUV444_SVE_2X I444TORGB_SVE_2X RGBTOARGB8_SVE_2X
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st4b {z16.b, z17.b, z18.b, z19.b}, p1, [%[dst_argb]] \n"
+ "incb %[dst_argb], all, mul #4 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p1.b, wzr, %w[width] \n" //
+ READYUV444_SVE_2X I444TORGB_SVE_2X RGBTOARGB8_SVE_2X
+ "st4b {z16.b, z17.b, z18.b, z19.b}, p1, [%[dst_argb]] \n"
+
+ "99: \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_SVE_REGS);
+}
+
+__arm_locally_streaming void I400ToARGBRow_SME(
+ const uint8_t* src_y,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ // Streaming-SVE only, no use of ZA tile.
+ I400ToARGBRow_SVE_SC(src_y, dst_argb, yuvconstants, width);
+}
+
+__arm_locally_streaming void I422ToARGBRow_SME(
+ const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ // Streaming-SVE only, no use of ZA tile.
+ I422ToARGBRow_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width);
+}
+
+__arm_locally_streaming void I422ToRGB24Row_SME(
+ const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I422ToRGB24Row_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width);
+}
+
+__arm_locally_streaming void I422ToRGB565Row_SME(
+ const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_rgb565,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I422ToRGB565Row_SVE_SC(src_y, src_u, src_v, dst_rgb565, yuvconstants, width);
+}
+
+__arm_locally_streaming void I422ToARGB1555Row_SME(
+ const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb1555,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I422ToARGB1555Row_SVE_SC(src_y, src_u, src_v, dst_argb1555, yuvconstants,
+ width);
+}
+
+__arm_locally_streaming void I422ToARGB4444Row_SME(
+ const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb4444,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I422ToARGB4444Row_SVE_SC(src_y, src_u, src_v, dst_argb4444, yuvconstants,
+ width);
+}
+
+__arm_locally_streaming void I422ToRGBARow_SME(
+ const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I422ToRGBARow_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width);
+}
+
+__arm_locally_streaming void I422AlphaToARGBRow_SME(
+ const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ const uint8_t* src_a,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I422AlphaToARGBRow_SVE_SC(src_y, src_u, src_v, src_a, dst_argb, yuvconstants,
+ width);
+}
+
+__arm_locally_streaming void I444AlphaToARGBRow_SME(
+ const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ const uint8_t* src_a,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I444AlphaToARGBRow_SVE_SC(src_y, src_u, src_v, src_a, dst_argb, yuvconstants,
+ width);
+}
+
+__arm_locally_streaming void NV12ToARGBRow_SME(
+ const uint8_t* src_y,
+ const uint8_t* src_uv,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ NV12ToARGBRow_SVE_SC(src_y, src_uv, dst_argb, yuvconstants, width);
+}
+
+__arm_locally_streaming void NV21ToARGBRow_SME(
+ const uint8_t* src_y,
+ const uint8_t* src_vu,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ NV21ToARGBRow_SVE_SC(src_y, src_vu, dst_argb, yuvconstants, width);
+}
+
+__arm_locally_streaming void NV12ToRGB24Row_SME(
+ const uint8_t* src_y,
+ const uint8_t* src_uv,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ NV12ToRGB24Row_SVE_SC(src_y, src_uv, dst_rgb24, yuvconstants, width);
+}
+
+__arm_locally_streaming void NV21ToRGB24Row_SME(
+ const uint8_t* src_y,
+ const uint8_t* src_vu,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ NV21ToRGB24Row_SVE_SC(src_y, src_vu, dst_rgb24, yuvconstants, width);
+}
+
+__arm_locally_streaming void YUY2ToARGBRow_SME(
+ const uint8_t* src_yuy2,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ YUY2ToARGBRow_SVE_SC(src_yuy2, dst_argb, yuvconstants, width);
+}
+
+__arm_locally_streaming void UYVYToARGBRow_SME(
+ const uint8_t* src_uyvy,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ UYVYToARGBRow_SVE_SC(src_uyvy, dst_argb, yuvconstants, width);
+}
+
+__arm_locally_streaming void I210ToARGBRow_SME(
+ const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I210ToARGBRow_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width);
+}
+
+__arm_locally_streaming void I210AlphaToARGBRow_SME(
+ const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ const uint16_t* src_a,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I210AlphaToARGBRow_SVE_SC(src_y, src_u, src_v, src_a, dst_argb, yuvconstants,
+ width);
+}
+
+__arm_locally_streaming void I210ToAR30Row_SME(
+ const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I210ToAR30Row_SVE_SC(src_y, src_u, src_v, dst_ar30, yuvconstants, width);
+}
+
+__arm_locally_streaming void P210ToARGBRow_SME(
+ const uint16_t* src_y,
+ const uint16_t* src_uv,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ P210ToARGBRow_SVE_SC(src_y, src_uv, dst_argb, yuvconstants, width);
+}
+
+__arm_locally_streaming void P210ToAR30Row_SME(
+ const uint16_t* src_y,
+ const uint16_t* src_uv,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ P210ToAR30Row_SVE_SC(src_y, src_uv, dst_ar30, yuvconstants, width);
+}
+
+__arm_locally_streaming void I410ToARGBRow_SME(
+ const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I410ToARGBRow_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width);
+}
+
+__arm_locally_streaming void I410AlphaToARGBRow_SME(
+ const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ const uint16_t* src_a,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I410AlphaToARGBRow_SVE_SC(src_y, src_u, src_v, src_a, dst_argb, yuvconstants,
+ width);
+}
+
+__arm_locally_streaming void I410ToAR30Row_SME(
+ const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I410ToAR30Row_SVE_SC(src_y, src_u, src_v, dst_ar30, yuvconstants, width);
+}
+
+__arm_locally_streaming void P410ToARGBRow_SME(
+ const uint16_t* src_y,
+ const uint16_t* src_uv,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ P410ToARGBRow_SVE_SC(src_y, src_uv, dst_argb, yuvconstants, width);
+}
+
+__arm_locally_streaming void P410ToAR30Row_SME(
+ const uint16_t* src_y,
+ const uint16_t* src_uv,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ P410ToAR30Row_SVE_SC(src_y, src_uv, dst_ar30, yuvconstants, width);
+}
+
+__arm_locally_streaming void I212ToAR30Row_SME(
+ const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I212ToAR30Row_SVE_SC(src_y, src_u, src_v, dst_ar30, yuvconstants, width);
+}
+
+__arm_locally_streaming void I212ToARGBRow_SME(
+ const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I212ToARGBRow_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width);
+}
+
+__arm_locally_streaming void MultiplyRow_16_SME(const uint16_t* src_y,
+ uint16_t* dst_y,
+ int scale,
+ int width) {
+ // Streaming-SVE only, no use of ZA tile.
+ int vl;
+ asm volatile(
+ "cnth %x[vl] \n"
+ "mov z0.h, %w[scale] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p0.h \n"
+ "1: \n"
+ "ld1h {z1.h}, p0/z, [%[src_y]] \n"
+ "incb %[src_y] \n"
+ "mul z1.h, z0.h, z1.h \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st1h {z1.h}, p0, [%[dst_y]] \n"
+ "incb %[dst_y] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p0.h, wzr, %w[width] \n"
+ "ld1h {z1.h}, p0/z, [%[src_y]] \n"
+ "mul z1.h, z0.h, z1.h \n"
+ "st1h {z1.h}, p0, [%[dst_y]] \n"
+
+ "99: \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [dst_y] "+r"(dst_y), // %[dst_y]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ : [scale] "r"(scale) // %[scale]
+ : "memory", "cc", "z0", "z1", "p0");
+}
+
+__arm_locally_streaming void ARGBMultiplyRow_SME(const uint8_t* src_argb,
+ const uint8_t* src_argb1,
+ uint8_t* dst_argb,
+ int width) {
+ // Streaming-SVE only, no use of ZA tile.
+ width *= 4;
+ int vl;
+ asm volatile(
+ "cntb %x[vl] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p0.b \n"
+ "1: \n"
+ "ld1b {z0.b}, p0/z, [%[src_argb]] \n"
+ "ld1b {z1.b}, p0/z, [%[src_argb1]] \n"
+ "incb %[src_argb] \n"
+ "incb %[src_argb1] \n"
+ "umullb z2.h, z0.b, z1.b \n"
+ "umullt z1.h, z0.b, z1.b \n"
+ "rshrnb z0.b, z2.h, #8 \n"
+ "rshrnt z0.b, z1.h, #8 \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st1b {z0.b}, p0, [%[dst_argb]] \n"
+ "incb %[dst_argb] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p0.b, wzr, %w[width] \n"
+ "ld1b {z0.b}, p0/z, [%[src_argb]] \n"
+ "ld1b {z1.b}, p0/z, [%[src_argb1]] \n"
+ "umullb z2.h, z0.b, z1.b \n"
+ "umullt z1.h, z0.b, z1.b \n"
+ "rshrnb z0.b, z2.h, #8 \n"
+ "rshrnt z0.b, z1.h, #8 \n"
+ "st1b {z0.b}, p0, [%[dst_argb]] \n"
+
+ "99: \n"
+ : [src_argb] "+r"(src_argb), // %[src_argb]
+ [src_argb1] "+r"(src_argb1), // %[src_argb1]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ :
+ : "memory", "cc", "z0", "z1", "z2", "p0", "p1");
+}
+
+__arm_locally_streaming void MergeUVRow_SME(const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_uv,
+ int width) {
+ // Streaming-SVE only, no use of ZA tile.
+ int vl;
+ asm volatile(
+ "cntb %x[vl] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p0.b \n"
+ "1: \n"
+ "ld1b {z1.b}, p0/z, [%[src_u]] \n"
+ "ld1b {z2.b}, p0/z, [%[src_v]] \n"
+ "incb %[src_u] \n"
+ "incb %[src_v] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st2b {z1.b, z2.b}, p0, [%[dst_uv]] \n"
+ "incb %[dst_uv], all, mul #2 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p0.b, wzr, %w[width] \n"
+ "ld1b {z1.b}, p0/z, [%[src_u]] \n"
+ "ld1b {z2.b}, p0/z, [%[src_v]] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st2b {z1.b, z2.b}, p0, [%[dst_uv]] \n"
+
+ "99: \n"
+ : [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_uv] "+r"(dst_uv), // %[dst_uv]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ :
+ : "memory", "cc", "z0", "z1", "z2", "p0");
+}
+
+__arm_locally_streaming void MergeUVRow_16_SME(const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint16_t* dst_uv,
+ int depth,
+ int width) {
+ int shift = 16 - depth;
+ // Streaming-SVE only, no use of ZA tile.
+ int vl;
+ asm volatile(
+ "cnth %x[vl] \n"
+ "mov z0.h, %w[shift] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p0.h \n"
+ "1: \n"
+ "ld1h {z1.h}, p0/z, [%[src_u]] \n"
+ "ld1h {z2.h}, p0/z, [%[src_v]] \n"
+ "incb %[src_u] \n"
+ "incb %[src_v] \n"
+ "lsl z1.h, p0/m, z1.h, z0.h \n"
+ "lsl z2.h, p0/m, z2.h, z0.h \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st2h {z1.h, z2.h}, p0, [%[dst_uv]] \n"
+ "incb %[dst_uv], all, mul #2 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p0.h, wzr, %w[width] \n"
+ "ld1h {z1.h}, p0/z, [%[src_u]] \n"
+ "ld1h {z2.h}, p0/z, [%[src_v]] \n"
+ "lsl z1.h, p0/m, z1.h, z0.h \n"
+ "lsl z2.h, p0/m, z2.h, z0.h \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st2h {z1.h, z2.h}, p0, [%[dst_uv]] \n"
+
+ "99: \n"
+ : [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_uv] "+r"(dst_uv), // %[dst_uv]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ : [shift] "r"(shift) // %[shift]
+ : "memory", "cc", "z0", "z1", "z2", "p0");
+}
+
+// Use scale to convert lsb formats to msb, depending how many bits there are:
+// 32768 = 9 bits = shr 1
+// 16384 = 10 bits = shr 2
+// 4096 = 12 bits = shr 4
+// 256 = 16 bits = shr 8
+__arm_locally_streaming void Convert16To8Row_SME(const uint16_t* src_y,
+ uint8_t* dst_y,
+ int scale,
+ int width) {
+ // 15 - clz(scale), + 8 to shift result into the high half of the lane to
+ // saturate, then we can just use UZP2 to narrow rather than a pair of
+ // saturating narrow instructions.
+ int shift = 23 - __builtin_clz((int32_t)scale);
+ int vl;
+ asm volatile(
+ "cntb %x[vl] \n"
+ "dup z0.h, %w[shift] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p0.b \n"
+ "1: \n"
+ "ld1h {z1.h}, p0/z, [%[src_y]] \n"
+ "ld1h {z2.h}, p0/z, [%[src_y], #1, mul vl] \n"
+ "incb %[src_y], all, mul #2 \n"
+ "uqshl z1.h, p0/m, z1.h, z0.h \n"
+ "uqshl z2.h, p0/m, z2.h, z0.h \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "uzp2 z1.b, z1.b, z2.b \n"
+ "st1b {z1.b}, p0, [%[dst_y]] \n"
+ "incb %[dst_y] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ // We need separate predicates for the load and store instructions since
+ // they are operating on different element sizes (.b vs .h).
+ "cnth %x[vl] \n"
+ "whilelt p0.h, wzr, %w[width] \n"
+ "whilelt p1.h, %w[vl], %w[width] \n"
+ "whilelt p2.b, wzr, %w[width] \n"
+ "ld1h {z1.h}, p0/z, [%[src_y]] \n"
+ "ld1h {z2.h}, p1/z, [%[src_y], #1, mul vl] \n"
+ "uqshl z1.h, p0/m, z1.h, z0.h \n"
+ "uqshl z2.h, p1/m, z2.h, z0.h \n"
+ "uzp2 z1.b, z1.b, z2.b \n"
+ "st1b {z1.b}, p2, [%[dst_y]] \n"
+
+ "99: \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [dst_y] "+r"(dst_y), // %[dst_y]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ : [shift] "r"(shift) // %[shift]
+ : "cc", "memory", "z0", "z1", "z2", "p0", "p1", "p2");
+}
+
+__arm_locally_streaming void CopyRow_SME(const uint8_t* src,
+ uint8_t* dst,
+ int width) {
+ // Streaming-SVE only, no use of ZA tile.
+ int vl;
+ asm volatile(
+ "cntb %x[vl] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p0.b \n"
+ "1: \n"
+ "ld1b {z0.b}, p0/z, [%[src]] \n"
+ "incb %[src] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st1b {z0.b}, p0, [%[dst]] \n"
+ "incb %[dst] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p0.b, wzr, %w[width] \n"
+ "ld1b {z0.b}, p0/z, [%[src]] \n"
+ "st1b {z0.b}, p0, [%[dst]] \n"
+
+ "99: \n"
+ : [src] "+r"(src), // %[src]
+ [dst] "+r"(dst), // %[dst]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ :
+ : "memory", "cc", "z0", "p0");
+}
+
+__arm_locally_streaming static void HalfRow_SME(uint8_t* dst_ptr,
+ const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ int width) {
+ const uint8_t* src_ptr1 = src_ptr + src_stride;
+
+ int vl;
+ asm volatile(
+ "cntb %x[vl] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p0.b \n"
+ "1: \n"
+ "ld1b {z2.b}, p0/z, [%[src_ptr]] \n"
+ "ld1b {z3.b}, p0/z, [%[src_ptr1]] \n"
+ "incb %[src_ptr] \n"
+ "incb %[src_ptr1] \n"
+ "urhadd z2.b, p0/m, z2.b, z3.b \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st1b {z2.b}, p0, [%[dst_ptr]] \n"
+ "incb %[dst_ptr] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p0.b, wzr, %w[width] \n"
+ "ld1b {z2.b}, p0/z, [%[src_ptr]] \n"
+ "ld1b {z3.b}, p0/z, [%[src_ptr1]] \n"
+ "urhadd z2.b, p0/m, z2.b, z3.b \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st1b {z2.b}, p0, [%[dst_ptr]] \n"
+
+ "99: \n"
+ : [src_ptr] "+r"(src_ptr), // %[src_ptr]
+ [src_ptr1] "+r"(src_ptr1), // %[src_ptr1]
+ [dst_ptr] "+r"(dst_ptr), // %[dst_ptr]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ :
+ : "cc", "memory", "z0", "z1", "z2", "z3", "p0");
+}
+
+__arm_locally_streaming void InterpolateRow_SME(uint8_t* dst_ptr,
+ const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ int width,
+ int source_y_fraction) {
+ int y1_fraction = source_y_fraction;
+ int y0_fraction = 256 - y1_fraction;
+ const uint8_t* src_ptr1 = src_ptr + src_stride;
+
+ if (y0_fraction == 0) {
+ CopyRow_SME(src_ptr1, dst_ptr, width);
+ return;
+ }
+ if (y0_fraction == 128) {
+ HalfRow_SME(dst_ptr, src_ptr, src_stride, width);
+ return;
+ }
+ if (y0_fraction == 256) {
+ CopyRow_SME(src_ptr, dst_ptr, width);
+ return;
+ }
+
+ int vl;
+ asm volatile(
+ "cntb %x[vl] \n"
+ "dup z0.b, %w[y0_fraction] \n"
+ "dup z1.b, %w[y1_fraction] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p0.b \n"
+ "1: \n"
+ "ld1b {z2.b}, p0/z, [%[src_ptr]] \n"
+ "ld1b {z3.b}, p0/z, [%[src_ptr1]] \n"
+ "incb %[src_ptr] \n"
+ "incb %[src_ptr1] \n"
+ "umullb z4.h, z2.b, z0.b \n"
+ "umullt z2.h, z2.b, z0.b \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "umlalb z4.h, z3.b, z1.b \n"
+ "umlalt z2.h, z3.b, z1.b \n"
+ "rshrnb z3.b, z4.h, #8 \n"
+ "rshrnt z3.b, z2.h, #8 \n"
+ "st1b {z3.b}, p0, [%[dst_ptr]] \n"
+ "incb %[dst_ptr] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p0.b, wzr, %w[width] \n"
+ "ld1b {z2.b}, p0/z, [%[src_ptr]] \n"
+ "ld1b {z3.b}, p0/z, [%[src_ptr1]] \n"
+ "umullb z4.h, z2.b, z0.b \n"
+ "umullt z2.h, z2.b, z0.b \n"
+ "umlalb z4.h, z3.b, z1.b \n"
+ "umlalt z2.h, z3.b, z1.b \n"
+ "rshrnb z3.b, z4.h, #8 \n"
+ "rshrnt z3.b, z2.h, #8 \n"
+ "st1b {z3.b}, p0, [%[dst_ptr]] \n"
+
+ "99: \n"
+ : [src_ptr] "+r"(src_ptr), // %[src_ptr]
+ [src_ptr1] "+r"(src_ptr1), // %[src_ptr1]
+ [dst_ptr] "+r"(dst_ptr), // %[dst_ptr]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ : [y0_fraction] "r"(y0_fraction), // %[y0_fraction]
+ [y1_fraction] "r"(y1_fraction) // %[y1_fraction]
+ : "cc", "memory", "z0", "z1", "z2", "z3", "z4", "p0");
+}
+
+__arm_locally_streaming static void HalfRow_16_SME(uint16_t* dst_ptr,
+ const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ int width) {
+ const uint16_t* src_ptr1 = src_ptr + src_stride;
+
+ int vl;
+ asm volatile(
+ "cnth %x[vl] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p0.h \n"
+ "1: \n"
+ "ld1h {z2.h}, p0/z, [%[src_ptr]] \n"
+ "ld1h {z3.h}, p0/z, [%[src_ptr1]] \n"
+ "incb %[src_ptr] \n"
+ "incb %[src_ptr1] \n"
+ "urhadd z2.h, p0/m, z2.h, z3.h \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st1h {z2.h}, p0, [%[dst_ptr]] \n"
+ "incb %[dst_ptr] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p0.h, wzr, %w[width] \n"
+ "ld1h {z2.h}, p0/z, [%[src_ptr]] \n"
+ "ld1h {z3.h}, p0/z, [%[src_ptr1]] \n"
+ "urhadd z2.h, p0/m, z2.h, z3.h \n"
+ "st1h {z2.h}, p0, [%[dst_ptr]] \n"
+
+ "99: \n"
+ : [src_ptr] "+r"(src_ptr), // %[src_ptr]
+ [src_ptr1] "+r"(src_ptr1), // %[src_ptr1]
+ [dst_ptr] "+r"(dst_ptr), // %[dst_ptr]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ :
+ : "cc", "memory", "z0", "z1", "z2", "z3", "p0");
+}
+
+__arm_locally_streaming void InterpolateRow_16_SME(uint16_t* dst_ptr,
+ const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ int width,
+ int source_y_fraction) {
+ int y1_fraction = source_y_fraction;
+ int y0_fraction = 256 - y1_fraction;
+ const uint16_t* src_ptr1 = src_ptr + src_stride;
+
+ if (y0_fraction == 0) {
+ CopyRow_SME((const uint8_t*)src_ptr1, (uint8_t*)dst_ptr,
+ width * sizeof(uint16_t));
+ return;
+ }
+ if (y0_fraction == 128) {
+ HalfRow_16_SME(dst_ptr, src_ptr, src_stride, width);
+ return;
+ }
+ if (y0_fraction == 256) {
+ CopyRow_SME((const uint8_t*)src_ptr, (uint8_t*)dst_ptr,
+ width * sizeof(uint16_t));
+ return;
+ }
+
+ int vl;
+ asm volatile(
+ "cnth %x[vl] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "dup z0.h, %w[y0_fraction] \n"
+ "dup z1.h, %w[y1_fraction] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p0.h \n"
+ "1: \n"
+ "ld1h {z2.h}, p0/z, [%[src_ptr]] \n"
+ "ld1h {z3.h}, p0/z, [%[src_ptr1]] \n"
+ "incb %[src_ptr] \n"
+ "incb %[src_ptr1] \n"
+ "umullb z4.s, z2.h, z0.h \n"
+ "umullt z2.s, z2.h, z0.h \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "umlalb z4.s, z3.h, z1.h \n"
+ "umlalt z2.s, z3.h, z1.h \n"
+ "rshrnb z3.h, z4.s, #8 \n"
+ "rshrnt z3.h, z2.s, #8 \n"
+ "st1h {z3.h}, p0, [%[dst_ptr]] \n"
+ "incb %[dst_ptr] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p0.h, wzr, %w[width] \n"
+ "ld1h {z2.h}, p0/z, [%[src_ptr]] \n"
+ "ld1h {z3.h}, p0/z, [%[src_ptr1]] \n"
+ "umullb z4.s, z2.h, z0.h \n"
+ "umullt z2.s, z2.h, z0.h \n"
+ "umlalb z4.s, z3.h, z1.h \n"
+ "umlalt z2.s, z3.h, z1.h \n"
+ "rshrnb z3.h, z4.s, #8 \n"
+ "rshrnt z3.h, z2.s, #8 \n"
+ "st1h {z3.h}, p0, [%[dst_ptr]] \n"
+
+ "99: \n"
+ : [src_ptr] "+r"(src_ptr), // %[src_ptr]
+ [src_ptr1] "+r"(src_ptr1), // %[src_ptr1]
+ [dst_ptr] "+r"(dst_ptr), // %[dst_ptr]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ : [y0_fraction] "r"(y0_fraction), // %[y0_fraction]
+ [y1_fraction] "r"(y1_fraction) // %[y1_fraction]
+ : "cc", "memory", "z0", "z1", "z2", "z3", "z4", "p0");
+}
+
+__arm_locally_streaming static void HalfRow_16To8_SME(uint8_t* dst_ptr,
+ const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ int scale,
+ int width) {
+ const uint16_t* src_ptr1 = src_ptr + src_stride;
+
+ // 15 - clz(scale), + 8 to shift result into the high half of the lane to
+ // saturate, then we can just use UZP2 to narrow rather than a pair of
+ // saturating narrow instructions.
+ int shift = 23 - __builtin_clz((int32_t)scale);
+
+ int vl;
+ asm volatile(
+ "cnth %x[vl] \n"
+ "dup z31.h, %w[shift] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p0.h \n"
+ "1: \n"
+ "ld1h {z2.h}, p0/z, [%[src_ptr]] \n"
+ "ld1h {z3.h}, p0/z, [%[src_ptr1]] \n"
+ "incb %[src_ptr] \n"
+ "incb %[src_ptr1] \n"
+ "urhadd z2.h, p0/m, z2.h, z3.h \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "uqshl z2.h, p0/m, z2.h, z31.h \n"
+ "shrnb z2.b, z2.h, #8 \n"
+ "st1b {z2.h}, p0, [%[dst_ptr]] \n"
+ "inch %[dst_ptr] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p0.h, wzr, %w[width] \n"
+ "ld1h {z2.h}, p0/z, [%[src_ptr]] \n"
+ "ld1h {z3.h}, p0/z, [%[src_ptr1]] \n"
+ "urhadd z2.h, p0/m, z2.h, z3.h \n"
+ "uqshl z2.h, p0/m, z2.h, z31.h \n"
+ "shrnb z2.b, z2.h, #8 \n"
+ "st1b {z2.h}, p0, [%[dst_ptr]] \n"
+
+ "99: \n"
+ : [src_ptr] "+r"(src_ptr), // %[src_ptr]
+ [src_ptr1] "+r"(src_ptr1), // %[src_ptr1]
+ [dst_ptr] "+r"(dst_ptr), // %[dst_ptr]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ : [shift] "r"(shift) // %[shift]
+ : "cc", "memory", "z0", "z1", "z2", "z3", "z31", "p0");
+}
+
+// Use scale to convert lsb formats to msb, depending how many bits there are:
+// 32768 = 9 bits
+// 16384 = 10 bits
+// 4096 = 12 bits
+// 256 = 16 bits
+// TODO(fbarchard): change scale to bits
+__arm_locally_streaming void InterpolateRow_16To8_SME(uint8_t* dst_ptr,
+ const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ int scale,
+ int width,
+ int source_y_fraction) {
+ int y1_fraction = source_y_fraction;
+ int y0_fraction = 256 - y1_fraction;
+ const uint16_t* src_ptr1 = src_ptr + src_stride;
+
+ // y0_fraction == 0 is never called here.
+ if (y0_fraction == 128) {
+ HalfRow_16To8_SME(dst_ptr, src_ptr, src_stride, scale, width);
+ return;
+ }
+ if (y0_fraction == 256) {
+ Convert16To8Row_SME(src_ptr, dst_ptr, scale, width);
+ return;
+ }
+
+ // 15 - clz(scale), + 8 to shift result into the high half of the lane to
+ // saturate, then we can just use UZP2 to narrow rather than a pair of
+ // saturating narrow instructions.
+ int shift = 23 - __builtin_clz((int32_t)scale);
+
+ int vl;
+ asm volatile(
+ "cnth %x[vl] \n"
+ "dup z31.h, %w[shift] \n"
+ "dup z0.h, %w[y0_fraction] \n"
+ "dup z1.h, %w[y1_fraction] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p0.h \n"
+ "1: \n"
+ "ld1h {z2.h}, p0/z, [%[src_ptr]] \n"
+ "ld1h {z3.h}, p0/z, [%[src_ptr1]] \n"
+ "incb %[src_ptr] \n"
+ "incb %[src_ptr1] \n"
+ "umullb z4.s, z2.h, z0.h \n"
+ "umullt z2.s, z2.h, z0.h \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "umlalb z4.s, z3.h, z1.h \n"
+ "umlalt z2.s, z3.h, z1.h \n"
+ "rshrnb z3.h, z4.s, #8 \n"
+ "rshrnt z3.h, z2.s, #8 \n"
+ "uqshl z3.h, p0/m, z3.h, z31.h \n"
+ "shrnb z3.b, z3.h, #8 \n"
+ "st1b {z3.h}, p0, [%[dst_ptr]] \n"
+ "inch %[dst_ptr] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p0.h, wzr, %w[width] \n"
+ "ld1h {z2.h}, p0/z, [%[src_ptr]] \n"
+ "ld1h {z3.h}, p0/z, [%[src_ptr1]] \n"
+ "umullb z4.s, z2.h, z0.h \n"
+ "umullt z2.s, z2.h, z0.h \n"
+ "umlalb z4.s, z3.h, z1.h \n"
+ "umlalt z2.s, z3.h, z1.h \n"
+ "rshrnb z3.h, z4.s, #8 \n"
+ "rshrnt z3.h, z2.s, #8 \n"
+ "uqshl z3.h, p0/m, z3.h, z31.h \n"
+ "shrnb z3.b, z3.h, #8 \n"
+ "st1b {z3.h}, p0, [%[dst_ptr]] \n"
+
+ "99: \n"
+ : [src_ptr] "+r"(src_ptr), // %[src_ptr]
+ [src_ptr1] "+r"(src_ptr1), // %[src_ptr1]
+ [dst_ptr] "+r"(dst_ptr), // %[dst_ptr]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ : [y0_fraction] "r"(y0_fraction), // %[y0_fraction]
+ [y1_fraction] "r"(y1_fraction), // %[y1_fraction]
+ [shift] "r"(shift) // %[shift]
+ : "cc", "memory", "z0", "z1", "z2", "z3", "z4", "z31", "p0");
+}
+
+__arm_locally_streaming void Convert8To8Row_SME(const uint8_t* src_y,
+ uint8_t* dst_y,
+ int scale,
+ int bias,
+ int width) {
+ Convert8To8Row_SVE_SC(src_y, dst_y, scale, bias, width);
+}
+
+#endif // !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) &&
+ // defined(__aarch64__)
+
+#ifdef __cplusplus
+} // extern "C"
+} // namespace libyuv
+#endif
diff --git a/source/row_sve.cc b/source/row_sve.cc
new file mode 100644
index 0000000..ba89b16
--- /dev/null
+++ b/source/row_sve.cc
@@ -0,0 +1,1276 @@
+/*
+ * Copyright 2024 The LibYuv Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "libyuv/row_sve.h"
+#include "libyuv/row.h"
+
+#ifdef __cplusplus
+namespace libyuv {
+extern "C" {
+#endif
+
+#if !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__)
+
+#define RGBTOARGB8_SVE_2X \
+ /* Inputs: B: z16.h, G: z17.h, R: z18.h, A: z19.b */ \
+ "uqshrnb z16.b, z16.h, #6 \n" /* B0 */ \
+ "uqshrnb z17.b, z17.h, #6 \n" /* G0 */ \
+ "uqshrnb z18.b, z18.h, #6 \n" /* R0 */ \
+ "uqshrnt z16.b, z20.h, #6 \n" /* B1 */ \
+ "uqshrnt z17.b, z21.h, #6 \n" /* G1 */ \
+ "uqshrnt z18.b, z22.h, #6 \n" /* R1 */
+
+#define RGBTOARGB8_SVE_TOP_2X \
+ /* Inputs: B: z16.h, G: z17.h, R: z18.h */ \
+ "uqshl z16.h, p0/m, z16.h, #2 \n" /* B0 */ \
+ "uqshl z17.h, p0/m, z17.h, #2 \n" /* G0 */ \
+ "uqshl z18.h, p0/m, z18.h, #2 \n" /* R0 */ \
+ "uqshl z20.h, p0/m, z20.h, #2 \n" /* B1 */ \
+ "uqshl z21.h, p0/m, z21.h, #2 \n" /* G1 */ \
+ "uqshl z22.h, p0/m, z22.h, #2 \n" /* R1 */
+
+void I444ToARGBRow_SVE2(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ uint64_t vl;
+ asm volatile(
+ "cnth %[vl] \n"
+ "ptrue p0.b \n" //
+ YUVTORGB_SVE_SETUP
+ "dup z19.b, #255 \n" // Alpha
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with an all-true predicate to avoid predicate
+ // generation overhead.
+ "ptrue p1.h \n"
+ "1: \n" //
+ READYUV444_SVE I4XXTORGB_SVE RGBTOARGB8_SVE
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
+ "add %[dst_argb], %[dst_argb], %[vl], lsl #2 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p1.h, wzr, %w[width] \n" //
+ READYUV444_SVE I4XXTORGB_SVE RGBTOARGB8_SVE
+ "st2h {z16.h, z17.h}, p1, [%[dst_argb]] \n"
+
+ "99: \n"
+ : [src_y] "+r"(src_y), // %[src_y]
+ [src_u] "+r"(src_u), // %[src_u]
+ [src_v] "+r"(src_v), // %[src_v]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff]
+ [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias]
+ : "cc", "memory", YUVTORGB_SVE_REGS);
+}
+
+void I400ToARGBRow_SVE2(const uint8_t* src_y,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I400ToARGBRow_SVE_SC(src_y, dst_argb, yuvconstants, width);
+}
+
+void I422ToARGBRow_SVE2(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I422ToARGBRow_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width);
+}
+
+void I422ToRGB24Row_SVE2(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I422ToRGB24Row_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width);
+}
+
+void I422ToRGB565Row_SVE2(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_rgb565,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I422ToRGB565Row_SVE_SC(src_y, src_u, src_v, dst_rgb565, yuvconstants, width);
+}
+
+void I422ToARGB1555Row_SVE2(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb1555,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I422ToARGB1555Row_SVE_SC(src_y, src_u, src_v, dst_argb1555, yuvconstants,
+ width);
+}
+
+void I422ToARGB4444Row_SVE2(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb4444,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I422ToARGB4444Row_SVE_SC(src_y, src_u, src_v, dst_argb4444, yuvconstants,
+ width);
+}
+
+void I422ToRGBARow_SVE2(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I422ToRGBARow_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width);
+}
+
+void I422AlphaToARGBRow_SVE2(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ const uint8_t* src_a,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I422AlphaToARGBRow_SVE_SC(src_y, src_u, src_v, src_a, dst_argb, yuvconstants,
+ width);
+}
+
+void I444AlphaToARGBRow_SVE2(const uint8_t* src_y,
+ const uint8_t* src_u,
+ const uint8_t* src_v,
+ const uint8_t* src_a,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I444AlphaToARGBRow_SVE_SC(src_y, src_u, src_v, src_a, dst_argb, yuvconstants,
+ width);
+}
+
+void NV12ToARGBRow_SVE2(const uint8_t* src_y,
+ const uint8_t* src_uv,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ NV12ToARGBRow_SVE_SC(src_y, src_uv, dst_argb, yuvconstants, width);
+}
+
+void NV21ToARGBRow_SVE2(const uint8_t* src_y,
+ const uint8_t* src_vu,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ NV21ToARGBRow_SVE_SC(src_y, src_vu, dst_argb, yuvconstants, width);
+}
+
+void NV12ToRGB24Row_SVE2(const uint8_t* src_y,
+ const uint8_t* src_uv,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ NV12ToRGB24Row_SVE_SC(src_y, src_uv, dst_rgb24, yuvconstants, width);
+}
+
+void NV21ToRGB24Row_SVE2(const uint8_t* src_y,
+ const uint8_t* src_vu,
+ uint8_t* dst_rgb24,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ NV21ToRGB24Row_SVE_SC(src_y, src_vu, dst_rgb24, yuvconstants, width);
+}
+
+// Dot-product constants are stored as four-tuples with the two innermost
+// elements flipped to account for the interleaving nature of the widening
+// addition instructions.
+
+// RGB to BT601 coefficients
+// UB 0.875 coefficient = 112
+// UG -0.5781 coefficient = -74
+// UR -0.2969 coefficient = -38
+// VB -0.1406 coefficient = -18
+// VG -0.7344 coefficient = -94
+// VR 0.875 coefficient = 112
+
+// SVE constants are not negated
+static const int16_t kARGBToUVCoefficients[] = {
+ // UB, -UR, -UG, 0, -VB, VR, -VG, 0
+ 112, -38, -74, 0, -18, 112, -94, 0,
+};
+
+static const int16_t kRGBAToUVCoefficients[] = {
+ // 0, -UG, UB, -UR, 0, -VG, -VB, VR
+ 0, -74, 112, -38, 0, -94, -18, 112,
+};
+
+static const int16_t kBGRAToUVCoefficients[] = {
+ // 0, -UG, -UR, UB, 0, -VG, VR, -VB
+ 0, -74, -38, 112, 0, -94, 112, -18,
+};
+
+static const int16_t kABGRToUVCoefficients[] = {
+ // -UR, UB, -UG, 0, VR, -VB, -VG, 0
+ -38, 112, -74, 0, 112, -18, -94, 0,
+};
+
+// RGB to JPEG coefficients
+// UB 0.500 coefficient = 128
+// UG -0.33126 coefficient = -85
+// UR -0.16874 coefficient = -43
+// VB -0.08131 coefficient = -21
+// VG -0.41869 coefficient = -107
+// VR 0.500 coefficient = 128
+
+static const int16_t kARGBToUVJCoefficients[] = {
+ // UB, -UR, -UG, 0, -VB, VR, -VG, 0
+ 128, -43, -85, 0, -21, 128, -107, 0,
+};
+
+static const int16_t kABGRToUVJCoefficients[] = {
+ // -UR, UB, -UG, 0, VR, -VB, -VG, 0
+ -43, 128, -85, 0, 128, -21, -107, 0,
+};
+
+static void ARGBToUVMatrixRow_SVE2(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width,
+ const int16_t* uvconstants) {
+ const uint8_t* src_argb_1 = src_argb + src_stride_argb;
+ uint64_t vl;
+ asm volatile(
+ "ptrue p0.b \n"
+ "ld1rd {z24.d}, p0/z, [%[uvconstants]] \n"
+ "ld1rd {z25.d}, p0/z, [%[uvconstants], #8] \n"
+ "mov z26.h, #0x8000 \n" // 128.0 (0x8000)
+ "cntb %[vl] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Process 4x vectors from each input row per iteration.
+ // Cannot use predication here due to unrolling.
+ "1: \n" // e.g.
+ "ld1b {z0.b}, p0/z, [%[src0], #0, mul vl] \n" // bgrabgra
+ "ld1b {z4.b}, p0/z, [%[src1], #0, mul vl] \n" // bgrabgra
+ "ld1b {z1.b}, p0/z, [%[src0], #1, mul vl] \n" // bgrabgra
+ "ld1b {z5.b}, p0/z, [%[src1], #1, mul vl] \n" // bgrabgra
+ "ld1b {z2.b}, p0/z, [%[src0], #2, mul vl] \n" // bgrabgra
+ "ld1b {z6.b}, p0/z, [%[src1], #2, mul vl] \n" // bgrabgra
+ "ld1b {z3.b}, p0/z, [%[src0], #3, mul vl] \n" // bgrabgra
+ "ld1b {z7.b}, p0/z, [%[src1], #3, mul vl] \n" // bgrabgra
+ "incb %[src0], all, mul #4 \n"
+ "incb %[src1], all, mul #4 \n"
+
+ "uaddlb z16.h, z0.b, z4.b \n" // brbrbrbr
+ "uaddlt z17.h, z0.b, z4.b \n" // gagagaga
+ "uaddlb z18.h, z1.b, z5.b \n" // brbrbrbr
+ "uaddlt z19.h, z1.b, z5.b \n" // gagagaga
+ "uaddlb z20.h, z2.b, z6.b \n" // brbrbrbr
+ "uaddlt z21.h, z2.b, z6.b \n" // gagagaga
+ "uaddlb z22.h, z3.b, z7.b \n" // brbrbrbr
+ "uaddlt z23.h, z3.b, z7.b \n" // gagagaga
+
+ "trn1 z0.s, z16.s, z17.s \n" // brgabgra
+ "trn2 z1.s, z16.s, z17.s \n" // brgabgra
+ "trn1 z2.s, z18.s, z19.s \n" // brgabgra
+ "trn2 z3.s, z18.s, z19.s \n" // brgabgra
+ "trn1 z4.s, z20.s, z21.s \n" // brgabgra
+ "trn2 z5.s, z20.s, z21.s \n" // brgabgra
+ "trn1 z6.s, z22.s, z23.s \n" // brgabgra
+ "trn2 z7.s, z22.s, z23.s \n" // brgabgra
+
+ "subs %w[width], %w[width], %w[vl] \n" // 4*VL per loop
+
+ "add z0.h, p0/m, z0.h, z1.h \n" // brgabrga
+ "add z2.h, p0/m, z2.h, z3.h \n" // brgabrga
+ "add z4.h, p0/m, z4.h, z5.h \n" // brgabrga
+ "add z6.h, p0/m, z6.h, z7.h \n" // brgabrga
+
+ "urshr z0.h, p0/m, z0.h, #2 \n" // brgabrga
+ "urshr z2.h, p0/m, z2.h, #2 \n" // brgabrga
+ "urshr z4.h, p0/m, z4.h, #2 \n" // brgabrga
+ "urshr z6.h, p0/m, z6.h, #2 \n" // brgabrga
+
+ "movi v16.8h, #0 \n"
+ "movi v17.8h, #0 \n"
+ "movi v18.8h, #0 \n"
+ "movi v19.8h, #0 \n"
+
+ "movi v20.8h, #0 \n"
+ "movi v21.8h, #0 \n"
+ "movi v22.8h, #0 \n"
+ "movi v23.8h, #0 \n"
+
+ "sdot z16.d, z0.h, z24.h \n" // UUxxxxxx
+ "sdot z17.d, z2.h, z24.h \n" // UUxxxxxx
+ "sdot z18.d, z4.h, z24.h \n" // UUxxxxxx
+ "sdot z19.d, z6.h, z24.h \n" // UUxxxxxx
+
+ "sdot z20.d, z0.h, z25.h \n" // VVxxxxxx
+ "sdot z21.d, z2.h, z25.h \n" // VVxxxxxx
+ "sdot z22.d, z4.h, z25.h \n" // VVxxxxxx
+ "sdot z23.d, z6.h, z25.h \n" // VVxxxxxx
+
+ "uzp1 z16.s, z16.s, z17.s \n" // UUxx
+ "uzp1 z18.s, z18.s, z19.s \n" // UUxx
+ "uzp1 z20.s, z20.s, z21.s \n" // VVxx
+ "uzp1 z22.s, z22.s, z23.s \n" // VVxx
+
+ "uzp1 z16.h, z16.h, z18.h \n" // UU
+ "uzp1 z20.h, z20.h, z22.h \n" // VV
+
+ "addhnb z16.b, z16.h, z26.h \n" // U
+ "addhnb z20.b, z20.h, z26.h \n" // V
+
+ "st1b {z16.h}, p0, [%[dst_u]] \n" // U
+ "st1b {z20.h}, p0, [%[dst_v]] \n" // V
+ "inch %[dst_u] \n"
+ "inch %[dst_v] \n"
+
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n" // VL per loop
+ "b.le 99f \n"
+
+ // Process remaining pixels from each input row.
+ // Use predication to do one vector from each input array, so may loop up
+ // to three iterations.
+ "cntw %x[vl] \n"
+
+ "3: \n"
+ "whilelt p1.s, wzr, %w[width] \n"
+ "ld1d {z0.d}, p1/z, [%[src0]] \n" // bgrabgra
+ "ld1d {z4.d}, p1/z, [%[src1]] \n" // bgrabgra
+ "incb %[src0] \n"
+ "incb %[src1] \n"
+
+ "uaddlb z16.h, z0.b, z4.b \n" // brbrbrbr
+ "uaddlt z17.h, z0.b, z4.b \n" // gagagaga
+
+ "trn1 z0.s, z16.s, z17.s \n" // brgabgra
+ "trn2 z1.s, z16.s, z17.s \n" // brgabgra
+
+ "add z0.h, p0/m, z0.h, z1.h \n" // brgabrga
+
+ "urshr z0.h, p0/m, z0.h, #2 \n" // brgabrga
+
+ "subs %w[width], %w[width], %w[vl] \n" // VL per loop
+
+ "movi v16.8h, #0 \n"
+ "movi v20.8h, #0 \n"
+
+ "sdot z16.d, z0.h, z24.h \n"
+ "sdot z20.d, z0.h, z25.h \n"
+
+ "addhnb z16.b, z16.h, z26.h \n" // U
+ "addhnb z20.b, z20.h, z26.h \n" // V
+
+ "st1b {z16.d}, p0, [%[dst_u]] \n" // U
+ "st1b {z20.d}, p0, [%[dst_v]] \n" // V
+ "incd %[dst_u] \n"
+ "incd %[dst_v] \n"
+ "b.gt 3b \n"
+
+ "99: \n"
+ : [src0] "+r"(src_argb), // %[src0]
+ [src1] "+r"(src_argb_1), // %[src1]
+ [dst_u] "+r"(dst_u), // %[dst_u]
+ [dst_v] "+r"(dst_v), // %[dst_v]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ : [uvconstants] "r"(uvconstants)
+ : "cc", "memory", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z16",
+ "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26",
+ "p0");
+}
+
+void ARGBToUVRow_SVE2(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ ARGBToUVMatrixRow_SVE2(src_argb, src_stride_argb, dst_u, dst_v, width,
+ kARGBToUVCoefficients);
+}
+
+void ARGBToUVJRow_SVE2(const uint8_t* src_argb,
+ int src_stride_argb,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ ARGBToUVMatrixRow_SVE2(src_argb, src_stride_argb, dst_u, dst_v, width,
+ kARGBToUVJCoefficients);
+}
+
+void ABGRToUVJRow_SVE2(const uint8_t* src_abgr,
+ int src_stride_abgr,
+ uint8_t* dst_uj,
+ uint8_t* dst_vj,
+ int width) {
+ ARGBToUVMatrixRow_SVE2(src_abgr, src_stride_abgr, dst_uj, dst_vj, width,
+ kABGRToUVJCoefficients);
+}
+
+void BGRAToUVRow_SVE2(const uint8_t* src_bgra,
+ int src_stride_bgra,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ ARGBToUVMatrixRow_SVE2(src_bgra, src_stride_bgra, dst_u, dst_v, width,
+ kBGRAToUVCoefficients);
+}
+
+void ABGRToUVRow_SVE2(const uint8_t* src_abgr,
+ int src_stride_abgr,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ ARGBToUVMatrixRow_SVE2(src_abgr, src_stride_abgr, dst_u, dst_v, width,
+ kABGRToUVCoefficients);
+}
+
+void RGBAToUVRow_SVE2(const uint8_t* src_rgba,
+ int src_stride_rgba,
+ uint8_t* dst_u,
+ uint8_t* dst_v,
+ int width) {
+ ARGBToUVMatrixRow_SVE2(src_rgba, src_stride_rgba, dst_u, dst_v, width,
+ kRGBAToUVCoefficients);
+}
+
+#define ARGBTORGB565_SVE \
+ /* Inputs: \
+ * z0: rrrrrxxxbbbbbxxx \
+ * z1: xxxxxxxxggggggxx \
+ * z3: 0000000000000011 (3, 0, 3, 0, ...) \
+ * z4: 0000011111100000 \
+ */ \
+ "lsr z0.b, p0/m, z0.b, z3.b \n" \
+ "lsl z1.h, z1.h, #3 \n" \
+ "bsl z1.d, z1.d, z0.d, z4.d \n"
+
+void ARGBToRGB565Row_SVE2(const uint8_t* src_argb,
+ uint8_t* dst_rgb,
+ int width) {
+ unsigned bsl_mask = 0x7e0;
+ uint64_t vl;
+ width *= 2;
+ asm volatile(
+ "mov z3.h, #3 \n"
+ "dup z4.h, %w[bsl_mask] \n"
+
+ "cntb %[vl] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ "ptrue p0.b \n"
+ "1: \n"
+ "ld2b {z0.b, z1.b}, p0/z, [%[src]] \n" // BR, GA
+ "incb %[src], all, mul #2 \n"
+ "subs %w[width], %w[width], %w[vl] \n" //
+ ARGBTORGB565_SVE
+ "st1b {z1.b}, p0, [%[dst]] \n"
+ "incb %[dst] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ "whilelt p0.b, wzr, %w[width] \n"
+ "ld2b {z0.b, z1.b}, p0/z, [%[src]] \n" // BR, GA
+ ARGBTORGB565_SVE
+ "st1b {z1.b}, p0, [%[dst]] \n"
+
+ "99: \n"
+ : [src] "+r"(src_argb), // %[src]
+ [dst] "+r"(dst_rgb), // %[dst]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ : [bsl_mask] "r"(bsl_mask) // %[bsl_mask]
+ : "cc", "memory", "z0", "z1", "z3", "z4", "p0");
+}
+
+void ARGBToRGB565DitherRow_SVE2(const uint8_t* src_argb,
+ uint8_t* dst_rgb,
+ uint32_t dither4,
+ int width) {
+ unsigned bsl_mask = 0x7e0;
+ uint64_t vl;
+ width *= 2;
+ asm volatile(
+ "mov z3.h, #3 \n"
+ "dup z4.h, %w[bsl_mask] \n"
+ "dup z2.s, %w[dither4] \n"
+ "zip1 z2.b, z2.b, z2.b \n"
+
+ "cntb %[vl] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ "ptrue p0.b \n"
+ "1: \n"
+ "ld2b {z0.b, z1.b}, p0/z, [%[src]] \n" // BR, GA
+ "incb %[src], all, mul #2 \n"
+ "uqadd z0.b, z0.b, z2.b \n"
+ "uqadd z1.b, z1.b, z2.b \n"
+ "subs %w[width], %w[width], %w[vl] \n" //
+ ARGBTORGB565_SVE
+ "st1b {z1.b}, p0, [%[dst]] \n"
+ "incb %[dst] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ "whilelt p0.b, wzr, %w[width] \n"
+ "ld2b {z0.b, z1.b}, p0/z, [%[src]] \n" // BR, GA
+ "uqadd z0.b, z0.b, z2.b \n"
+ "uqadd z1.b, z1.b, z2.b \n" //
+ ARGBTORGB565_SVE
+ "st1b {z1.b}, p0, [%[dst]] \n"
+
+ "99: \n"
+ : [src] "+r"(src_argb), // %[src]
+ [dst] "+r"(dst_rgb), // %[dst]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ : [bsl_mask] "r"(bsl_mask), // %[bsl_mask]
+ [dither4] "r"(dither4) // %[dither4]
+ : "cc", "memory", "z0", "z1", "z3", "z4", "p0");
+}
+
+#define ARGB1555TOARGB \
+ /* Input: z1/z3.h = arrrrrgggggbbbbb */ \
+ "lsl z0.h, z1.h, #3 \n" /* rrrgggggbbbbb000 */ \
+ "lsl z2.h, z3.h, #3 \n" /* rrrgggggbbbbb000 */ \
+ "asr z1.h, z1.h, #7 \n" /* aaaaaaaarrrrrggg */ \
+ "asr z3.h, z3.h, #7 \n" /* aaaaaaaarrrrrggg */ \
+ "lsl z0.b, p0/m, z0.b, z4.b \n" /* ggggg000bbbbb000 */ \
+ "lsl z2.b, p0/m, z2.b, z4.b \n" /* ggggg000bbbbb000 */ \
+ "sri z1.b, z1.b, #5 \n" /* aaaaaaaarrrrrrrr */ \
+ "sri z3.b, z3.b, #5 \n" /* aaaaaaaarrrrrrrr */ \
+ "sri z0.b, z0.b, #5 \n" /* ggggggggbbbbbbbb */ \
+ "sri z2.b, z2.b, #5 \n" /* ggggggggbbbbbbbb */
+
+void ARGB1555ToARGBRow_SVE2(const uint8_t* src_argb1555,
+ uint8_t* dst_argb,
+ int width) {
+ uint64_t vl;
+ asm volatile(
+ "mov z4.h, #0x0300 \n"
+ "ptrue p0.b \n"
+
+ "cnth %x[vl] \n"
+ "subs %w[width], %w[width], %w[vl], lsl #1 \n"
+ "b.lt 2f \n"
+
+ "1: \n"
+ "ld1h {z1.h}, p0/z, [%[src]] \n"
+ "ld1h {z3.h}, p0/z, [%[src], #1, mul vl] \n"
+ "incb %[src], all, mul #2 \n" //
+ ARGB1555TOARGB
+ "subs %w[width], %w[width], %w[vl], lsl #1 \n"
+ "st2h {z0.h, z1.h}, p0, [%[dst]] \n"
+ "st2h {z2.h, z3.h}, p0, [%[dst], #2, mul vl] \n"
+ "incb %[dst], all, mul #4 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl], lsl #1 \n"
+ "b.eq 99f \n"
+
+ "whilelt p1.h, wzr, %w[width] \n"
+ "whilelt p2.h, %w[vl], %w[width] \n"
+ "ld1h {z1.h}, p1/z, [%[src]] \n"
+ "ld1h {z3.h}, p2/z, [%[src], #1, mul vl] \n" //
+ ARGB1555TOARGB
+ "st2h {z0.h, z1.h}, p1, [%[dst]] \n"
+ "st2h {z2.h, z3.h}, p2, [%[dst], #2, mul vl] \n"
+
+ "99: \n"
+ : [src] "+r"(src_argb1555), // %[src]
+ [dst] "+r"(dst_argb), // %[dst]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ :
+ : "cc", "memory", "z0", "z1", "z2", "z3", "z4", "p0", "p1", "p2");
+}
+
+// clang-format off
+#define AYUVTOUV_SVE(zU0, zV0, zU1, zV1) /* e.g. */ \
+ "ld2h {z0.h, z1.h}, p0/z, [%[src0]] \n" /* VUVU.. YAYA.. */ \
+ "ld2h {z1.h, z2.h}, p1/z, [%[src0], #2, mul vl] \n" /* VUVU.. YAYA.. */ \
+ "ld2h {z2.h, z3.h}, p0/z, [%[src1]] \n" /* VUVU.. YAYA.. */ \
+ "ld2h {z3.h, z4.h}, p1/z, [%[src1], #2, mul vl] \n" /* VUVU.. YAYA.. */ \
+ "incb %[src0], all, mul #4 \n" \
+ "incb %[src1], all, mul #4 \n" \
+ "uaddlb z4.h, z0.b, z2.b \n" /* V */ \
+ "uaddlt z5.h, z0.b, z2.b \n" /* U */ \
+ "uaddlb z6.h, z1.b, z3.b \n" /* V */ \
+ "uaddlt z7.h, z1.b, z3.b \n" /* U */ \
+ "addp " #zU0 ".h, p0/m, " #zU0 ".h, " #zV0 ".h \n" /* UV */ \
+ "addp " #zU1 ".h, p1/m, " #zU1 ".h, " #zV1 ".h \n" /* UV */ \
+ "subs %w[width], %w[width], %w[vl] \n" \
+ "urshr " #zU0 ".h, p0/m, " #zU0 ".h, #2 \n" /* U0V0 */ \
+ "urshr " #zU1 ".h, p1/m, " #zU1 ".h, #2 \n" /* U0V0 */ \
+ "st1b {" #zU0 ".h}, p0, [%[dst]] \n" \
+ "st1b {" #zU1 ".h}, p1, [%[dst], #1, mul vl] \n" \
+ "incb %[dst] \n"
+// clang-format on
+
+// Filter 2 rows of AYUV UV's (444) into UV (420).
+// AYUV is VUYA in memory. UV for NV12 is UV order in memory.
+void AYUVToUVRow_SVE2(const uint8_t* src_ayuv,
+ int src_stride_ayuv,
+ uint8_t* dst_uv,
+ int width) {
+ // Output a row of UV values, filtering 2x2 rows of AYUV.
+ const uint8_t* src_ayuv1 = src_ayuv + src_stride_ayuv;
+ int vl;
+ asm volatile (
+ "cntb %x[vl] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ "ptrue p0.h \n"
+ "ptrue p1.h \n"
+ "1: \n"
+ AYUVTOUV_SVE(z5, z4, z7, z6)
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ "cnth %x[vl] \n"
+ "whilelt p0.h, wzr, %w[width] \n" // first row
+ "whilelt p1.h, %w[vl], %w[width] \n" // second row
+ AYUVTOUV_SVE(z5, z4, z7, z6)
+
+ "99: \n"
+ : [src0]"+r"(src_ayuv), // %[src0]
+ [src1]"+r"(src_ayuv1), // %[src1]
+ [dst]"+r"(dst_uv), // %[dst]
+ [width]"+r"(width), // %[width]
+ [vl]"=&r"(vl) // %[vl]
+ :
+ : "cc", "memory", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "p0",
+ "p1");
+}
+
+// Filter 2 rows of AYUV UV's (444) into VU (420).
+void AYUVToVURow_SVE2(const uint8_t* src_ayuv,
+ int src_stride_ayuv,
+ uint8_t* dst_vu,
+ int width) {
+ // Output a row of VU values, filtering 2x2 rows of AYUV.
+ const uint8_t* src_ayuv1 = src_ayuv + src_stride_ayuv;
+ int vl;
+ asm volatile (
+ "cntb %x[vl] \n"
+ "cmp %w[width], %w[vl] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ "ptrue p0.h \n"
+ "ptrue p1.h \n"
+ "1: \n"
+ AYUVTOUV_SVE(z4, z5, z6, z7)
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ "cnth %x[vl] \n"
+ "whilelt p0.h, wzr, %w[width] \n" // first row
+ "whilelt p1.h, %w[vl], %w[width] \n" // second row
+ AYUVTOUV_SVE(z4, z5, z6, z7)
+
+ "99: \n"
+ : [src0]"+r"(src_ayuv), // %[src0]
+ [src1]"+r"(src_ayuv1), // %[src1]
+ [dst]"+r"(dst_vu), // %[dst]
+ [width]"+r"(width), // %[width]
+ [vl]"=&r"(vl) // %[vl]
+ :
+ : "cc", "memory", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "p0",
+ "p1");
+}
+
+void YUY2ToARGBRow_SVE2(const uint8_t* src_yuy2,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ YUY2ToARGBRow_SVE_SC(src_yuy2, dst_argb, yuvconstants, width);
+}
+
+void UYVYToARGBRow_SVE2(const uint8_t* src_uyvy,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ UYVYToARGBRow_SVE_SC(src_uyvy, dst_argb, yuvconstants, width);
+}
+
+static inline void RAWToWXYZRow_SVE2(const uint8_t* src_raw,
+ uint8_t* dst_wxyz,
+ int width,
+ uint32_t idx_start,
+ uint32_t idx_step,
+ uint32_t alpha) {
+ uint32_t vl;
+ asm("cntw %x0" : "=r"(vl));
+ uint32_t vl_mul3 = vl * 3;
+ uint32_t rem_mul3;
+ asm volatile(
+ "index z31.s, %w[idx_start], %w[idx_step] \n"
+ "dup z30.s, %w[alpha] \n"
+ "subs %w[width], %w[width], %w[vl], lsl #1 \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with the same predicates to avoid predicate
+ // generation overhead. We set up p1 to only load 3/4 of a vector.
+ "ptrue p0.s \n"
+ "whilelt p1.b, wzr, %w[vl_mul3] \n"
+ "1: \n"
+ "ld1b {z0.b}, p1/z, [%[src]] \n"
+ "add %[src], %[src], %x[vl_mul3] \n"
+ "ld1b {z1.b}, p1/z, [%[src]] \n"
+ "add %[src], %[src], %x[vl_mul3] \n"
+ "tbl z0.b, {z0.b}, z31.b \n"
+ "tbl z1.b, {z1.b}, z31.b \n"
+ "subs %w[width], %w[width], %w[vl], lsl #1 \n"
+ "orr z0.d, z0.d, z30.d \n"
+ "orr z1.d, z1.d, z30.d \n"
+ "st1w {z0.s}, p0, [%[dst]] \n"
+ "st1w {z1.s}, p0, [%[dst], #1, mul vl] \n"
+ "incb %[dst], all, mul #2 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl], lsl #1 \n"
+ "b.eq 99f \n"
+
+ // Calculate a pair of predicates for the final iteration to deal with
+ // the tail.
+ "3: \n"
+ "add %w[rem_mul3], %w[width], %w[width], lsl #1 \n"
+ "whilelt p0.s, wzr, %w[width] \n"
+ "whilelt p1.b, wzr, %w[rem_mul3] \n"
+ "ld1b {z0.b}, p1/z, [%[src]] \n"
+ "add %[src], %[src], %x[vl_mul3] \n"
+ "tbl z0.b, {z0.b}, z31.b \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "orr z0.d, z0.d, z30.d \n"
+ "st1w {z0.s}, p0, [%[dst]] \n"
+ "incb %[dst] \n"
+ "b.gt 3b \n"
+
+ "99: \n"
+ : [src] "+r"(src_raw), // %[src]
+ [dst] "+r"(dst_wxyz), // %[dst]
+ [width] "+r"(width), // %[width]
+ [vl_mul3] "+r"(vl_mul3), // %[vl_mul3]
+ [rem_mul3] "=&r"(rem_mul3) // %[rem_mul3]
+ : [idx_start] "r"(idx_start), // %[idx_start]
+ [idx_step] "r"(idx_step), // %[idx_step]
+ [alpha] "r"(alpha), // %[alpha]
+ [vl] "r"(vl) // %[vl]
+ : "cc", "memory", "z0", "z1", "z30", "z31", "p0", "p1");
+}
+
+void RAWToARGBRow_SVE2(const uint8_t* src_raw, uint8_t* dst_argb, int width) {
+ RAWToWXYZRow_SVE2(src_raw, dst_argb, width, 0xff000102U, 0x00030303U,
+ 0xff000000U);
+}
+
+void RAWToRGBARow_SVE2(const uint8_t* src_raw, uint8_t* dst_rgba, int width) {
+ RAWToWXYZRow_SVE2(src_raw, dst_rgba, width, 0x000102ffU, 0x03030300U,
+ 0x000000ffU);
+}
+
+void RGB24ToARGBRow_SVE2(const uint8_t* src_rgb24,
+ uint8_t* dst_argb,
+ int width) {
+ RAWToWXYZRow_SVE2(src_rgb24, dst_argb, width, 0xff020100U, 0x00030303U,
+ 0xff000000U);
+}
+
+static const uint8_t kRAWToRGB24Indices[] = {
+ 2, 1, 0, 5, 4, 3, 8, 7, 6, 11, 10, 9, 14, 13, 12,
+ 17, 16, 15, 20, 19, 18, 23, 22, 21, 26, 25, 24, 29, 28, 27,
+ 32, 31, 30, 35, 34, 33, 38, 37, 36, 41, 40, 39, 44, 43, 42,
+ 47, 46, 45, 50, 49, 48, 53, 52, 51, 56, 55, 54, 59, 58, 57,
+ 62, 61, 60, 65, 64, 63, 68, 67, 66, 71, 70, 69, 74, 73, 72,
+ 77, 76, 75, 80, 79, 78, 83, 82, 81, 86, 85, 84, 89, 88, 87,
+ 92, 91, 90, 95, 94, 93, 98, 97, 96, 101, 100, 99, 104, 103, 102,
+ 107, 106, 105, 110, 109, 108, 113, 112, 111, 116, 115, 114, 119, 118, 117,
+ 122, 121, 120, 125, 124, 123, 128, 127, 126, 131, 130, 129, 134, 133, 132,
+ 137, 136, 135, 140, 139, 138, 143, 142, 141, 146, 145, 144, 149, 148, 147,
+ 152, 151, 150, 155, 154, 153, 158, 157, 156, 161, 160, 159, 164, 163, 162,
+ 167, 166, 165, 170, 169, 168, 173, 172, 171, 176, 175, 174, 179, 178, 177,
+ 182, 181, 180, 185, 184, 183, 188, 187, 186, 191, 190, 189, 194, 193, 192,
+ 197, 196, 195, 200, 199, 198, 203, 202, 201, 206, 205, 204, 209, 208, 207,
+ 212, 211, 210, 215, 214, 213, 218, 217, 216, 221, 220, 219, 224, 223, 222,
+ 227, 226, 225, 230, 229, 228, 233, 232, 231, 236, 235, 234, 239, 238, 237,
+ 242, 241, 240, 245, 244, 243, 248, 247, 246, 251, 250, 249, 254, 253, 252};
+
+void RAWToRGB24Row_SVE2(const uint8_t* src_raw, uint8_t* dst_rgb24, int width) {
+ // width is in elements, convert to bytes.
+ width *= 3;
+ // we use the mul3 predicate pattern throughout to use the largest multiple
+ // of three number of lanes, for instance with a vector length of 16 bytes
+ // only the first 15 bytes will be used for load/store instructions.
+ uint32_t vl;
+ asm volatile(
+ "cntb %x[vl], mul3 \n"
+ "ptrue p0.b, mul3 \n"
+ "ld1b {z31.b}, p0/z, [%[kIndices]] \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with the same predicate to avoid predicate
+ // generation overhead.
+ "1: \n"
+ "ld1b {z0.b}, p0/z, [%[src]] \n"
+ "add %[src], %[src], %x[vl] \n"
+ "tbl z0.b, {z0.b}, z31.b \n"
+ "subs %w[width], %w[width], %w[vl] \n"
+ "st1b {z0.b}, p0, [%[dst]] \n"
+ "add %[dst], %[dst], %x[vl] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ // Calculate a predicate for the final iteration to deal with the tail.
+ "whilelt p0.b, wzr, %w[width] \n"
+ "ld1b {z0.b}, p0/z, [%[src]] \n"
+ "tbl z0.b, {z0.b}, z31.b \n"
+ "st1b {z0.b}, p0, [%[dst]] \n"
+
+ "99: \n"
+ : [src] "+r"(src_raw), // %[src]
+ [dst] "+r"(dst_rgb24), // %[dst]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ : [kIndices] "r"(kRAWToRGB24Indices) // %[kIndices]
+ : "cc", "memory", "z0", "z31", "p0");
+}
+
+static inline void ARGBToXYZRow_SVE2(const uint8_t* src_argb,
+ uint8_t* dst_xyz,
+ int width,
+ const uint8_t* indices) {
+ uint32_t vl;
+ asm("cntw %x0" : "=r"(vl));
+ uint32_t vl_mul3 = vl * 3;
+ uint32_t rem_mul3;
+ asm volatile(
+ "whilelt p1.b, wzr, %w[vl_mul3] \n"
+ "ld1b {z31.b}, p1/z, [%[indices]] \n"
+ "subs %w[width], %w[width], %w[vl], lsl #1 \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with the same predicates to avoid predicate
+ // generation overhead. We set up p1 to only store 3/4 of a vector.
+ "ptrue p0.s \n"
+ "1: \n"
+ "ld1w {z0.s}, p0/z, [%[src]] \n"
+ "ld1w {z1.s}, p0/z, [%[src], #1, mul vl] \n"
+ "incb %[src], all, mul #2 \n"
+ "tbl z0.b, {z0.b}, z31.b \n"
+ "tbl z1.b, {z1.b}, z31.b \n"
+ "subs %w[width], %w[width], %w[vl], lsl #1 \n"
+ "st1b {z0.b}, p1, [%[dst]] \n"
+ "add %[dst], %[dst], %x[vl_mul3] \n"
+ "st1b {z1.b}, p1, [%[dst]] \n"
+ "add %[dst], %[dst], %x[vl_mul3] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl], lsl #1 \n"
+ "b.eq 99f \n"
+
+ // Calculate predicates for the final iteration to deal with the tail.
+ "add %w[rem_mul3], %w[width], %w[width], lsl #1 \n"
+ "whilelt p0.s, wzr, %w[width] \n"
+ "whilelt p1.b, wzr, %w[rem_mul3] \n"
+ "whilelt p2.s, %w[vl], %w[width] \n"
+ "whilelt p3.b, %w[vl_mul3], %w[rem_mul3] \n"
+ "ld1w {z0.s}, p0/z, [%[src]] \n"
+ "ld1w {z1.s}, p2/z, [%[src], #1, mul vl] \n"
+ "tbl z0.b, {z0.b}, z31.b \n"
+ "tbl z1.b, {z1.b}, z31.b \n"
+ "st1b {z0.b}, p1, [%[dst]] \n"
+ "add %[dst], %[dst], %x[vl_mul3] \n"
+ "st1b {z1.b}, p3, [%[dst]] \n"
+
+ "99: \n"
+ : [src] "+r"(src_argb), // %[src]
+ [dst] "+r"(dst_xyz), // %[dst]
+ [width] "+r"(width), // %[width]
+ [rem_mul3] "=&r"(rem_mul3) // %[rem_mul3]
+ : [indices] "r"(indices), // %[indices]
+ [vl_mul3] "r"(vl_mul3), // %[vl_mul3]
+ [vl] "r"(vl) // %[vl]
+ : "cc", "memory", "z0", "z1", "z31", "p0", "p1", "p2", "p3");
+}
+
+static const uint8_t kARGBToRGB24RowIndices[] = {
+ 0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 16, 17, 18,
+ 20, 21, 22, 24, 25, 26, 28, 29, 30, 32, 33, 34, 36, 37, 38,
+ 40, 41, 42, 44, 45, 46, 48, 49, 50, 52, 53, 54, 56, 57, 58,
+ 60, 61, 62, 64, 65, 66, 68, 69, 70, 72, 73, 74, 76, 77, 78,
+ 80, 81, 82, 84, 85, 86, 88, 89, 90, 92, 93, 94, 96, 97, 98,
+ 100, 101, 102, 104, 105, 106, 108, 109, 110, 112, 113, 114, 116, 117, 118,
+ 120, 121, 122, 124, 125, 126, 128, 129, 130, 132, 133, 134, 136, 137, 138,
+ 140, 141, 142, 144, 145, 146, 148, 149, 150, 152, 153, 154, 156, 157, 158,
+ 160, 161, 162, 164, 165, 166, 168, 169, 170, 172, 173, 174, 176, 177, 178,
+ 180, 181, 182, 184, 185, 186, 188, 189, 190, 192, 193, 194, 196, 197, 198,
+ 200, 201, 202, 204, 205, 206, 208, 209, 210, 212, 213, 214, 216, 217, 218,
+ 220, 221, 222, 224, 225, 226, 228, 229, 230, 232, 233, 234, 236, 237, 238,
+ 240, 241, 242, 244, 245, 246, 248, 249, 250, 252, 253, 254,
+};
+
+static const uint8_t kARGBToRAWRowIndices[] = {
+ 2, 1, 0, 6, 5, 4, 10, 9, 8, 14, 13, 12, 18, 17, 16,
+ 22, 21, 20, 26, 25, 24, 30, 29, 28, 34, 33, 32, 38, 37, 36,
+ 42, 41, 40, 46, 45, 44, 50, 49, 48, 54, 53, 52, 58, 57, 56,
+ 62, 61, 60, 66, 65, 64, 70, 69, 68, 74, 73, 72, 78, 77, 76,
+ 82, 81, 80, 86, 85, 84, 90, 89, 88, 94, 93, 92, 98, 97, 96,
+ 102, 101, 100, 106, 105, 104, 110, 109, 108, 114, 113, 112, 118, 117, 116,
+ 122, 121, 120, 126, 125, 124, 130, 129, 128, 134, 133, 132, 138, 137, 136,
+ 142, 141, 140, 146, 145, 144, 150, 149, 148, 154, 153, 152, 158, 157, 156,
+ 162, 161, 160, 166, 165, 164, 170, 169, 168, 174, 173, 172, 178, 177, 176,
+ 182, 181, 180, 186, 185, 184, 190, 189, 188, 194, 193, 192, 198, 197, 196,
+ 202, 201, 200, 206, 205, 204, 210, 209, 208, 214, 213, 212, 218, 217, 216,
+ 222, 221, 220, 226, 225, 224, 230, 229, 228, 234, 233, 232, 238, 237, 236,
+ 242, 241, 240, 246, 245, 244, 250, 249, 248, 254, 253, 252,
+};
+
+void ARGBToRGB24Row_SVE2(const uint8_t* src_argb, uint8_t* dst_rgb, int width) {
+ ARGBToXYZRow_SVE2(src_argb, dst_rgb, width, kARGBToRGB24RowIndices);
+}
+
+void ARGBToRAWRow_SVE2(const uint8_t* src_argb, uint8_t* dst_rgb, int width) {
+ ARGBToXYZRow_SVE2(src_argb, dst_rgb, width, kARGBToRAWRowIndices);
+}
+
+void DivideRow_16_SVE2(const uint16_t* src_y,
+ uint16_t* dst_y,
+ int scale,
+ int width) {
+ uint64_t vl;
+ asm volatile(
+ "cnth %x[vl] \n"
+ "dup z0.h, %w[scale] \n"
+ "subs %w[width], %w[width], %w[vl], lsl #1 \n"
+ "b.le 2f \n"
+
+ // Run bulk of computation with the same predicates to avoid predicate
+ // generation overhead.
+ "ptrue p0.h \n"
+ "1: \n"
+ "ld1h {z1.h}, p0/z, [%[src]] \n"
+ "ld1h {z2.h}, p0/z, [%[src], #1, mul vl] \n"
+ "incb %[src], all, mul #2 \n"
+ "umulh z1.h, z1.h, z0.h \n"
+ "umulh z2.h, z2.h, z0.h \n"
+ "subs %w[width], %w[width], %w[vl], lsl #1 \n"
+ "st1h {z1.h}, p0, [%[dst]] \n"
+ "st1h {z2.h}, p0, [%[dst], #1, mul vl] \n"
+ "incb %[dst], all, mul #2 \n"
+ "b.gt 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl], lsl #1 \n"
+ "b.eq 99f \n"
+
+ // Calculate a pair of predicates for the final iteration to deal with
+ // the tail.
+ "whilelt p0.h, wzr, %w[width] \n"
+ "whilelt p1.h, %w[vl], %w[width] \n"
+ "ld1h {z1.h}, p0/z, [%[src]] \n"
+ "ld1h {z2.h}, p1/z, [%[src], #1, mul vl] \n"
+ "umulh z1.h, z1.h, z0.h \n"
+ "umulh z2.h, z2.h, z0.h \n"
+ "st1h {z1.h}, p0, [%[dst]] \n"
+ "st1h {z2.h}, p1, [%[dst], #1, mul vl] \n"
+
+ "99: \n"
+ : [src] "+r"(src_y), // %[src]
+ [dst] "+r"(dst_y), // %[dst]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ : [scale] "r"(scale) // %[scale]
+ : "cc", "memory", "z0", "z1", "z2", "p0", "p1");
+}
+
+#define HALFFLOAT_SVE \
+ "scvtf z0.s, p0/m, z0.s \n" \
+ "scvtf z1.s, p0/m, z1.s \n" \
+ "scvtf z2.s, p0/m, z2.s \n" \
+ "scvtf z3.s, p0/m, z3.s \n" \
+ "fmul z0.s, z0.s, z4.s \n" \
+ "fmul z1.s, z1.s, z4.s \n" \
+ "fmul z2.s, z2.s, z4.s \n" \
+ "fmul z3.s, z3.s, z4.s \n" \
+ "uqshrnb z0.h, z0.s, #13 \n" \
+ "uqshrnb z1.h, z1.s, #13 \n" \
+ "uqshrnb z2.h, z2.s, #13 \n" \
+ "uqshrnb z3.h, z3.s, #13 \n"
+
+void HalfFloatRow_SVE2(const uint16_t* src,
+ uint16_t* dst,
+ float scale,
+ int width) {
+ uint64_t vl;
+ asm("cntw %x0" : "=r"(vl));
+ asm volatile(
+ "mov z4.s, %s[scale] \n"
+ "subs %w[width], %w[width], %w[vl], lsl #2 \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with all-true predicates to avoid predicate
+ // generation overhead.
+ "ptrue p0.s \n"
+ "1: \n"
+ "ld1h {z0.s}, p0/z, [%[src]] \n"
+ "ld1h {z1.s}, p0/z, [%[src], #1, mul vl] \n"
+ "ld1h {z2.s}, p0/z, [%[src], #2, mul vl] \n"
+ "ld1h {z3.s}, p0/z, [%[src], #3, mul vl] \n"
+ "incb %[src], all, mul #2 \n" //
+ HALFFLOAT_SVE
+ "subs %w[width], %w[width], %w[vl], lsl #2 \n"
+ "st1h {z0.s}, p0, [%[dst]] \n"
+ "st1h {z1.s}, p0, [%[dst], #1, mul vl] \n"
+ "st1h {z2.s}, p0, [%[dst], #2, mul vl] \n"
+ "st1h {z3.s}, p0, [%[dst], #3, mul vl] \n"
+ "incb %[dst], all, mul #2 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl], lsl #2 \n"
+ "b.eq 99f \n"
+
+ // Calculate predicates for the final iteration to deal with the tail.
+ "whilelt p0.s, wzr, %w[width] \n"
+ "whilelt p1.s, %w[vl], %w[width] \n"
+ "whilelt p2.s, %w[vl2], %w[width] \n"
+ "whilelt p3.s, %w[vl3], %w[width] \n"
+ "ld1h {z0.s}, p0/z, [%[src]] \n"
+ "ld1h {z1.s}, p1/z, [%[src], #1, mul vl] \n"
+ "ld1h {z2.s}, p2/z, [%[src], #2, mul vl] \n"
+ "ld1h {z3.s}, p3/z, [%[src], #3, mul vl] \n" //
+ HALFFLOAT_SVE
+ "st1h {z0.s}, p0, [%[dst]] \n"
+ "st1h {z1.s}, p1, [%[dst], #1, mul vl] \n"
+ "st1h {z2.s}, p2, [%[dst], #2, mul vl] \n"
+ "st1h {z3.s}, p3, [%[dst], #3, mul vl] \n"
+
+ "99: \n"
+ : [src] "+r"(src), // %[src]
+ [dst] "+r"(dst), // %[dst]
+ [width] "+r"(width) // %[width]
+ : [vl] "r"(vl), // %[vl]
+ [vl2] "r"(vl * 2), // %[vl2]
+ [vl3] "r"(vl * 3), // %[vl3]
+ [scale] "w"(scale * 1.9259299444e-34f) // %[scale]
+ : "cc", "memory", "z0", "z1", "z2", "z3", "z4", "p0", "p1", "p2", "p3");
+}
+
+void HalfFloat1Row_SVE2(const uint16_t* src,
+ uint16_t* dst,
+ float scale,
+ int width) {
+ uint64_t vl;
+ asm volatile(
+ "cnth %x[vl] \n"
+ "subs %w[width], %w[width], %w[vl], lsl #1 \n"
+ "b.lt 2f \n"
+
+ // Run bulk of computation with all-true predicates to avoid predicate
+ // generation overhead.
+ "ptrue p0.h \n"
+ "1: \n"
+ "ld1h {z0.h}, p0/z, [%[src]] \n"
+ "ld1h {z1.h}, p0/z, [%[src], #1, mul vl] \n"
+ "incb %[src], all, mul #2 \n"
+ "ucvtf z0.h, p0/m, z0.h \n"
+ "ucvtf z1.h, p0/m, z1.h \n"
+ "subs %w[width], %w[width], %w[vl], lsl #1 \n"
+ "st1h {z0.h}, p0, [%[dst]] \n"
+ "st1h {z1.h}, p0, [%[dst], #1, mul vl] \n"
+ "incb %[dst], all, mul #2 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[width], %w[width], %w[vl], lsl #1 \n"
+ "b.eq 99f \n"
+
+ // Calculate predicates for the final iteration to deal with the tail.
+ "whilelt p0.h, wzr, %w[width] \n"
+ "whilelt p1.h, %w[vl], %w[width] \n"
+ "ld1h {z0.h}, p0/z, [%[src]] \n"
+ "ld1h {z1.h}, p1/z, [%[src], #1, mul vl] \n"
+ "ucvtf z0.h, p0/m, z0.h \n"
+ "ucvtf z1.h, p0/m, z1.h \n"
+ "st1h {z0.h}, p0, [%[dst]] \n"
+ "st1h {z1.h}, p1, [%[dst], #1, mul vl] \n"
+
+ "99: \n"
+ : [src] "+r"(src), // %[src]
+ [dst] "+r"(dst), // %[dst]
+ [width] "+r"(width), // %[width]
+ [vl] "=&r"(vl) // %[vl]
+ :
+ : "cc", "memory", "z0", "z1", "p0", "p1");
+}
+
+void I210ToARGBRow_SVE2(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I210ToARGBRow_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width);
+}
+
+void I210AlphaToARGBRow_SVE2(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ const uint16_t* src_a,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I210AlphaToARGBRow_SVE_SC(src_y, src_u, src_v, src_a, dst_argb, yuvconstants,
+ width);
+}
+
+void I210ToAR30Row_SVE2(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I210ToAR30Row_SVE_SC(src_y, src_u, src_v, dst_ar30, yuvconstants, width);
+}
+
+void P210ToARGBRow_SVE2(const uint16_t* src_y,
+ const uint16_t* src_uv,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ P210ToARGBRow_SVE_SC(src_y, src_uv, dst_argb, yuvconstants, width);
+}
+
+void P210ToAR30Row_SVE2(const uint16_t* src_y,
+ const uint16_t* src_uv,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ P210ToAR30Row_SVE_SC(src_y, src_uv, dst_ar30, yuvconstants, width);
+}
+
+void I410ToARGBRow_SVE2(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I410ToARGBRow_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width);
+}
+
+void I410AlphaToARGBRow_SVE2(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ const uint16_t* src_a,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I410AlphaToARGBRow_SVE_SC(src_y, src_u, src_v, src_a, dst_argb, yuvconstants,
+ width);
+}
+
+void I410ToAR30Row_SVE2(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I410ToAR30Row_SVE_SC(src_y, src_u, src_v, dst_ar30, yuvconstants, width);
+}
+
+void P410ToARGBRow_SVE2(const uint16_t* src_y,
+ const uint16_t* src_uv,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ P410ToARGBRow_SVE_SC(src_y, src_uv, dst_argb, yuvconstants, width);
+}
+
+void P410ToAR30Row_SVE2(const uint16_t* src_y,
+ const uint16_t* src_uv,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ P410ToAR30Row_SVE_SC(src_y, src_uv, dst_ar30, yuvconstants, width);
+}
+
+void I212ToAR30Row_SVE2(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* dst_ar30,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I212ToAR30Row_SVE_SC(src_y, src_u, src_v, dst_ar30, yuvconstants, width);
+}
+
+void I212ToARGBRow_SVE2(const uint16_t* src_y,
+ const uint16_t* src_u,
+ const uint16_t* src_v,
+ uint8_t* dst_argb,
+ const struct YuvConstants* yuvconstants,
+ int width) {
+ I212ToARGBRow_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width);
+}
+
+void Convert8To8Row_SVE2(const uint8_t* src_y,
+ uint8_t* dst_y,
+ int scale,
+ int bias,
+ int width) {
+ Convert8To8Row_SVE_SC(src_y, dst_y, scale, bias, width);
+}
+
+#endif // !defined(LIBYUV_DISABLE_SVE) && defined(__aarch64__)
+
+#ifdef __cplusplus
+} // extern "C"
+} // namespace libyuv
+#endif
diff --git a/source/row_win.cc b/source/row_win.cc
index 5fb2852..1a57ee4 100644
--- a/source/row_win.cc
+++ b/source/row_win.cc
@@ -12,7 +12,8 @@
// This module is for Visual C 32/64 bit
#if !defined(LIBYUV_DISABLE_X86) && defined(_MSC_VER) && \
- !defined(__clang__) && (defined(_M_IX86) || defined(_M_X64))
+ (defined(_M_IX86) || defined(_M_X64)) && \
+ (!defined(__clang__) || defined(LIBYUV_ENABLE_ROWWIN))
#if defined(_M_ARM64EC)
#include <intrin.h>
@@ -182,15 +183,52 @@
// 32 bit
#else // defined(_M_X64)
-#ifdef HAS_ARGBTOYROW_SSSE3
-// Constants for ARGB.
-static const vec8 kARGBToY = {13, 65, 33, 0, 13, 65, 33, 0,
- 13, 65, 33, 0, 13, 65, 33, 0};
+#ifdef HAS_ARGBTOUVROW_SSSE3
+
+// 8 bit fixed point 0.5, for bias of UV.
+static const ulvec8 kBiasUV128 = {
+ 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
+ 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
+ 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80};
+
+// NV21 shuf 8 VU to 16 UV.
+static const lvec8 kShuffleNV21 = {
+ 1, 0, 1, 0, 3, 2, 3, 2, 5, 4, 5, 4, 7, 6, 7, 6,
+ 1, 0, 1, 0, 3, 2, 3, 2, 5, 4, 5, 4, 7, 6, 7, 6,
+};
+
+// YUY2 shuf 16 Y to 32 Y.
+static const lvec8 kShuffleYUY2Y = {0, 0, 2, 2, 4, 4, 6, 6, 8, 8, 10,
+ 10, 12, 12, 14, 14, 0, 0, 2, 2, 4, 4,
+ 6, 6, 8, 8, 10, 10, 12, 12, 14, 14};
+
+// YUY2 shuf 8 UV to 16 UV.
+static const lvec8 kShuffleYUY2UV = {1, 3, 1, 3, 5, 7, 5, 7, 9, 11, 9,
+ 11, 13, 15, 13, 15, 1, 3, 1, 3, 5, 7,
+ 5, 7, 9, 11, 9, 11, 13, 15, 13, 15};
+
+// UYVY shuf 16 Y to 32 Y.
+static const lvec8 kShuffleUYVYY = {1, 1, 3, 3, 5, 5, 7, 7, 9, 9, 11,
+ 11, 13, 13, 15, 15, 1, 1, 3, 3, 5, 5,
+ 7, 7, 9, 9, 11, 11, 13, 13, 15, 15};
+
+// UYVY shuf 8 UV to 16 UV.
+static const lvec8 kShuffleUYVYUV = {0, 2, 0, 2, 4, 6, 4, 6, 8, 10, 8,
+ 10, 12, 14, 12, 14, 0, 2, 0, 2, 4, 6,
+ 4, 6, 8, 10, 8, 10, 12, 14, 12, 14};
// JPeg full range.
static const vec8 kARGBToYJ = {15, 75, 38, 0, 15, 75, 38, 0,
15, 75, 38, 0, 15, 75, 38, 0};
+#endif
+
+// vpermd for vphaddw + vpackuswb vpermd.
+static const lvec32 kPermdARGBToY_AVX = {0, 4, 1, 5, 2, 6, 3, 7};
+
+// Constants for ARGB.
+static const vec8 kARGBToY = {13, 65, 33, 0, 13, 65, 33, 0,
+ 13, 65, 33, 0, 13, 65, 33, 0};
static const vec8 kARGBToU = {112, -74, -38, 0, 112, -74, -38, 0,
112, -74, -38, 0, 112, -74, -38, 0};
@@ -246,12 +284,6 @@
// 7 bit fixed point 0.5.
static const vec16 kAddYJ64 = {64, 64, 64, 64, 64, 64, 64, 64};
-// 8 bit fixed point 0.5, for bias of UV.
-static const ulvec8 kBiasUV128 = {
- 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
- 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
- 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80};
-
// Shuffle table for converting RGB24 to ARGB.
static const uvec8 kShuffleMaskRGB24ToARGB = {
0u, 1u, 2u, 12u, 3u, 4u, 5u, 13u, 6u, 7u, 8u, 14u, 9u, 10u, 11u, 15u};
@@ -287,32 +319,6 @@
static const uvec8 kShuffleMaskARGBToRGB24_0 = {
0u, 1u, 2u, 4u, 5u, 6u, 8u, 9u, 128u, 128u, 128u, 128u, 10u, 12u, 13u, 14u};
-// YUY2 shuf 16 Y to 32 Y.
-static const lvec8 kShuffleYUY2Y = {0, 0, 2, 2, 4, 4, 6, 6, 8, 8, 10,
- 10, 12, 12, 14, 14, 0, 0, 2, 2, 4, 4,
- 6, 6, 8, 8, 10, 10, 12, 12, 14, 14};
-
-// YUY2 shuf 8 UV to 16 UV.
-static const lvec8 kShuffleYUY2UV = {1, 3, 1, 3, 5, 7, 5, 7, 9, 11, 9,
- 11, 13, 15, 13, 15, 1, 3, 1, 3, 5, 7,
- 5, 7, 9, 11, 9, 11, 13, 15, 13, 15};
-
-// UYVY shuf 16 Y to 32 Y.
-static const lvec8 kShuffleUYVYY = {1, 1, 3, 3, 5, 5, 7, 7, 9, 9, 11,
- 11, 13, 13, 15, 15, 1, 1, 3, 3, 5, 5,
- 7, 7, 9, 9, 11, 11, 13, 13, 15, 15};
-
-// UYVY shuf 8 UV to 16 UV.
-static const lvec8 kShuffleUYVYUV = {0, 2, 0, 2, 4, 6, 4, 6, 8, 10, 8,
- 10, 12, 14, 12, 14, 0, 2, 0, 2, 4, 6,
- 4, 6, 8, 10, 8, 10, 12, 14, 12, 14};
-
-// NV21 shuf 8 VU to 16 UV.
-static const lvec8 kShuffleNV21 = {
- 1, 0, 1, 0, 3, 2, 3, 2, 5, 4, 5, 4, 7, 6, 7, 6,
- 1, 0, 1, 0, 3, 2, 3, 2, 5, 4, 5, 4, 7, 6, 7, 6,
-};
-
// Duplicates gray value 3 times and fills in alpha opaque.
__declspec(naked) void J400ToARGBRow_SSE2(const uint8_t* src_y,
uint8_t* dst_argb,
@@ -1240,8 +1246,6 @@
}
#ifdef HAS_ARGBTOYROW_AVX2
-// vpermd for vphaddw + vpackuswb vpermd.
-static const lvec32 kPermdARGBToY_AVX = {0, 4, 1, 5, 2, 6, 3, 7};
// Convert 32 ARGB pixels (128 bytes) to 32 Y values.
__declspec(naked) void ARGBToYRow_AVX2(const uint8_t* src_argb,
@@ -1511,7 +1515,9 @@
mov edx, [esp + 8 + 12] // dst_u
mov edi, [esp + 8 + 16] // dst_v
mov ecx, [esp + 8 + 20] // width
+ // TODO: change biasuv to 0x8000
movdqa xmm5, xmmword ptr kBiasUV128
+ // TODO: use negated coefficients to allow -128
movdqa xmm6, xmmword ptr kARGBToVJ
movdqa xmm7, xmmword ptr kARGBToUJ
sub edi, edx // stride from u to v
@@ -1552,10 +1558,12 @@
pmaddubsw xmm3, xmm6
phaddw xmm0, xmm2
phaddw xmm1, xmm3
+ // TODO: negate by subtracting from 0x8000
paddw xmm0, xmm5 // +.5 rounding -> unsigned
paddw xmm1, xmm5
psraw xmm0, 8
psraw xmm1, 8
+ // TODO: packuswb
packsswb xmm0, xmm1
// step 3 - store 8 U and 8 V values
@@ -1981,7 +1989,6 @@
ret
}
}
-#endif // HAS_ARGBTOYROW_SSSE3
// Read 16 UV from 444
#define READYUV444_AVX2 \
diff --git a/source/scale.cc b/source/scale.cc
index b7a602b..76379fd 100644
--- a/source/scale.cc
+++ b/source/scale.cc
@@ -74,6 +74,13 @@
}
}
#endif
+#if defined(HAS_SCALEROWDOWN2_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ ScaleRowDown2 = filtering == kFilterNone ? ScaleRowDown2_SME
+ : filtering == kFilterLinear ? ScaleRowDown2Linear_SME
+ : ScaleRowDown2Box_SME;
+ }
+#endif
#if defined(HAS_SCALEROWDOWN2_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
ScaleRowDown2 =
@@ -181,8 +188,16 @@
#if defined(HAS_SCALEROWDOWN2_16_NEON)
if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(dst_width, 16)) {
- ScaleRowDown2 =
- filtering ? ScaleRowDown2Box_16_NEON : ScaleRowDown2_16_NEON;
+ ScaleRowDown2 = filtering == kFilterNone ? ScaleRowDown2_16_NEON
+ : filtering == kFilterLinear ? ScaleRowDown2Linear_16_NEON
+ : ScaleRowDown2Box_16_NEON;
+ }
+#endif
+#if defined(HAS_SCALEROWDOWN2_16_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ ScaleRowDown2 = filtering == kFilterNone ? ScaleRowDown2_16_SME
+ : filtering == kFilterLinear ? ScaleRowDown2Linear_16_SME
+ : ScaleRowDown2Box_16_SME;
}
#endif
#if defined(HAS_SCALEROWDOWN2_16_SSE2)
@@ -279,7 +294,7 @@
if (TestCpuFlag(kCpuHasNEON)) {
ScaleRowDown4 =
filtering ? ScaleRowDown4Box_Any_NEON : ScaleRowDown4_Any_NEON;
- if (IS_ALIGNED(dst_width, 8)) {
+ if (IS_ALIGNED(dst_width, 16)) {
ScaleRowDown4 = filtering ? ScaleRowDown4Box_NEON : ScaleRowDown4_NEON;
}
}
@@ -407,14 +422,11 @@
}
#if defined(HAS_SCALEROWDOWN34_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
- if (!filtering) {
- ScaleRowDown34_0 = ScaleRowDown34_Any_NEON;
- ScaleRowDown34_1 = ScaleRowDown34_Any_NEON;
- } else {
- ScaleRowDown34_0 = ScaleRowDown34_0_Box_Any_NEON;
- ScaleRowDown34_1 = ScaleRowDown34_1_Box_Any_NEON;
- }
+#if defined(__aarch64__)
+ if (dst_width % 48 == 0) {
+#else
if (dst_width % 24 == 0) {
+#endif
if (!filtering) {
ScaleRowDown34_0 = ScaleRowDown34_NEON;
ScaleRowDown34_1 = ScaleRowDown34_NEON;
@@ -422,18 +434,19 @@
ScaleRowDown34_0 = ScaleRowDown34_0_Box_NEON;
ScaleRowDown34_1 = ScaleRowDown34_1_Box_NEON;
}
+ } else {
+ if (!filtering) {
+ ScaleRowDown34_0 = ScaleRowDown34_Any_NEON;
+ ScaleRowDown34_1 = ScaleRowDown34_Any_NEON;
+ } else {
+ ScaleRowDown34_0 = ScaleRowDown34_0_Box_Any_NEON;
+ ScaleRowDown34_1 = ScaleRowDown34_1_Box_Any_NEON;
+ }
}
}
#endif
#if defined(HAS_SCALEROWDOWN34_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
- if (!filtering) {
- ScaleRowDown34_0 = ScaleRowDown34_Any_MSA;
- ScaleRowDown34_1 = ScaleRowDown34_Any_MSA;
- } else {
- ScaleRowDown34_0 = ScaleRowDown34_0_Box_Any_MSA;
- ScaleRowDown34_1 = ScaleRowDown34_1_Box_Any_MSA;
- }
if (dst_width % 48 == 0) {
if (!filtering) {
ScaleRowDown34_0 = ScaleRowDown34_MSA;
@@ -442,18 +455,19 @@
ScaleRowDown34_0 = ScaleRowDown34_0_Box_MSA;
ScaleRowDown34_1 = ScaleRowDown34_1_Box_MSA;
}
+ } else {
+ if (!filtering) {
+ ScaleRowDown34_0 = ScaleRowDown34_Any_MSA;
+ ScaleRowDown34_1 = ScaleRowDown34_Any_MSA;
+ } else {
+ ScaleRowDown34_0 = ScaleRowDown34_0_Box_Any_MSA;
+ ScaleRowDown34_1 = ScaleRowDown34_1_Box_Any_MSA;
+ }
}
}
#endif
#if defined(HAS_SCALEROWDOWN34_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
- if (!filtering) {
- ScaleRowDown34_0 = ScaleRowDown34_Any_LSX;
- ScaleRowDown34_1 = ScaleRowDown34_Any_LSX;
- } else {
- ScaleRowDown34_0 = ScaleRowDown34_0_Box_Any_LSX;
- ScaleRowDown34_1 = ScaleRowDown34_1_Box_Any_LSX;
- }
if (dst_width % 48 == 0) {
if (!filtering) {
ScaleRowDown34_0 = ScaleRowDown34_LSX;
@@ -462,18 +476,19 @@
ScaleRowDown34_0 = ScaleRowDown34_0_Box_LSX;
ScaleRowDown34_1 = ScaleRowDown34_1_Box_LSX;
}
+ } else {
+ if (!filtering) {
+ ScaleRowDown34_0 = ScaleRowDown34_Any_LSX;
+ ScaleRowDown34_1 = ScaleRowDown34_Any_LSX;
+ } else {
+ ScaleRowDown34_0 = ScaleRowDown34_0_Box_Any_LSX;
+ ScaleRowDown34_1 = ScaleRowDown34_1_Box_Any_LSX;
+ }
}
}
#endif
#if defined(HAS_SCALEROWDOWN34_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
- if (!filtering) {
- ScaleRowDown34_0 = ScaleRowDown34_Any_SSSE3;
- ScaleRowDown34_1 = ScaleRowDown34_Any_SSSE3;
- } else {
- ScaleRowDown34_0 = ScaleRowDown34_0_Box_Any_SSSE3;
- ScaleRowDown34_1 = ScaleRowDown34_1_Box_Any_SSSE3;
- }
if (dst_width % 24 == 0) {
if (!filtering) {
ScaleRowDown34_0 = ScaleRowDown34_SSSE3;
@@ -482,6 +497,14 @@
ScaleRowDown34_0 = ScaleRowDown34_0_Box_SSSE3;
ScaleRowDown34_1 = ScaleRowDown34_1_Box_SSSE3;
}
+ } else {
+ if (!filtering) {
+ ScaleRowDown34_0 = ScaleRowDown34_Any_SSSE3;
+ ScaleRowDown34_1 = ScaleRowDown34_Any_SSSE3;
+ } else {
+ ScaleRowDown34_0 = ScaleRowDown34_0_Box_Any_SSSE3;
+ ScaleRowDown34_1 = ScaleRowDown34_1_Box_Any_SSSE3;
+ }
}
}
#endif
@@ -1150,6 +1173,11 @@
}
}
#endif
+#if defined(HAS_INTERPOLATEROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ InterpolateRow = InterpolateRow_SME;
+ }
+#endif
#if defined(HAS_INTERPOLATEROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
InterpolateRow = InterpolateRow_Any_MSA;
@@ -1289,6 +1317,11 @@
}
}
#endif
+#if defined(HAS_INTERPOLATEROW_16_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ InterpolateRow = InterpolateRow_16_SME;
+ }
+#endif
#if defined(HAS_SCALEFILTERCOLS_16_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3) && src_width < 32768) {
@@ -1370,6 +1403,11 @@
}
}
#endif
+#if defined(HAS_INTERPOLATEROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ InterpolateRow = InterpolateRow_SME;
+ }
+#endif
#if defined(HAS_INTERPOLATEROW_RVV)
if (TestCpuFlag(kCpuHasRVV)) {
InterpolateRow = InterpolateRow_RVV;
@@ -1870,6 +1908,11 @@
}
}
#endif
+#if defined(HAS_INTERPOLATEROW_16_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ InterpolateRow = InterpolateRow_16_SME;
+ }
+#endif
if (filtering && src_width >= 32768) {
ScaleFilterCols = ScaleFilterCols64_16_C;
@@ -2701,6 +2744,38 @@
return r;
}
+LIBYUV_API
+int NV24Scale(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_uv,
+ int src_stride_uv,
+ int src_width,
+ int src_height,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_uv,
+ int dst_stride_uv,
+ int dst_width,
+ int dst_height,
+ enum FilterMode filtering) {
+ int r;
+
+ if (!src_y || !src_uv || src_width <= 0 || src_height == 0 ||
+ src_width > 32768 || src_height > 32768 || !dst_y || !dst_uv ||
+ dst_width <= 0 || dst_height <= 0) {
+ return -1;
+ }
+
+ r = ScalePlane(src_y, src_stride_y, src_width, src_height, dst_y,
+ dst_stride_y, dst_width, dst_height, filtering);
+ if (r != 0) {
+ return r;
+ }
+ r = UVScale(src_uv, src_stride_uv, src_width, src_height, dst_uv,
+ dst_stride_uv, dst_width, dst_height, filtering);
+ return r;
+}
+
// Deprecated api
LIBYUV_API
int Scale(const uint8_t* src_y,
diff --git a/source/scale_any.cc b/source/scale_any.cc
index f657687..1433446 100644
--- a/source/scale_any.cc
+++ b/source/scale_any.cc
@@ -202,7 +202,7 @@
15)
#endif
#ifdef HAS_SCALEROWDOWN4_NEON
-SDANY(ScaleRowDown4_Any_NEON, ScaleRowDown4_NEON, ScaleRowDown4_C, 4, 1, 7)
+SDANY(ScaleRowDown4_Any_NEON, ScaleRowDown4_NEON, ScaleRowDown4_C, 4, 1, 15)
SDANY(ScaleRowDown4Box_Any_NEON,
ScaleRowDown4Box_NEON,
ScaleRowDown4Box_C,
@@ -249,6 +249,26 @@
23)
#endif
#ifdef HAS_SCALEROWDOWN34_NEON
+#ifdef __aarch64__
+SDANY(ScaleRowDown34_Any_NEON,
+ ScaleRowDown34_NEON,
+ ScaleRowDown34_C,
+ 4 / 3,
+ 1,
+ 47)
+SDANY(ScaleRowDown34_0_Box_Any_NEON,
+ ScaleRowDown34_0_Box_NEON,
+ ScaleRowDown34_0_Box_C,
+ 4 / 3,
+ 1,
+ 47)
+SDANY(ScaleRowDown34_1_Box_Any_NEON,
+ ScaleRowDown34_1_Box_NEON,
+ ScaleRowDown34_1_Box_C,
+ 4 / 3,
+ 1,
+ 47)
+#else
SDANY(ScaleRowDown34_Any_NEON,
ScaleRowDown34_NEON,
ScaleRowDown34_C,
@@ -268,6 +288,7 @@
1,
23)
#endif
+#endif
#ifdef HAS_SCALEROWDOWN34_MSA
SDANY(ScaleRowDown34_Any_MSA,
ScaleRowDown34_MSA,
@@ -749,12 +770,20 @@
#endif
#ifdef HAS_SCALEROWUP2_LINEAR_NEON
+#ifdef __aarch64__
+SUH2LANY(ScaleRowUp2_Linear_Any_NEON,
+ ScaleRowUp2_Linear_NEON,
+ ScaleRowUp2_Linear_C,
+ 31,
+ uint8_t)
+#else
SUH2LANY(ScaleRowUp2_Linear_Any_NEON,
ScaleRowUp2_Linear_NEON,
ScaleRowUp2_Linear_C,
15,
uint8_t)
#endif
+#endif
#ifdef HAS_SCALEROWUP2_LINEAR_12_NEON
SUH2LANY(ScaleRowUp2_Linear_12_Any_NEON,
diff --git a/source/scale_argb.cc b/source/scale_argb.cc
index 18bdeb8..5fa58fd 100644
--- a/source/scale_argb.cc
+++ b/source/scale_argb.cc
@@ -11,6 +11,9 @@
#include "libyuv/scale.h"
#include <assert.h>
+#include <limits.h>
+#include <stdint.h>
+#include <stdlib.h>
#include <string.h>
#include "libyuv/cpu_id.h"
@@ -96,6 +99,14 @@
}
}
#endif
+#if defined(HAS_SCALEARGBROWDOWN2_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ ScaleARGBRowDown2 = filtering == kFilterNone ? ScaleARGBRowDown2_SME
+ : filtering == kFilterLinear
+ ? ScaleARGBRowDown2Linear_SME
+ : ScaleARGBRowDown2Box_SME;
+ }
+#endif
#if defined(HAS_SCALEARGBROWDOWN2_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
ScaleARGBRowDown2 =
@@ -128,7 +139,9 @@
}
}
#endif
-#if defined(HAS_SCALEARGBROWDOWN2_RVV)
+#if defined(HAS_SCALEARGBROWDOWN2_RVV) && \
+ defined(HAS_SCALEARGBROWDOWN2LINEAR_RVV) && \
+ defined(HAS_SCALEARGBROWDOWN2BOX_RVV)
if (TestCpuFlag(kCpuHasRVV)) {
ScaleARGBRowDown2 =
filtering == kFilterNone
@@ -199,7 +212,12 @@
}
}
#endif
-#if defined(HAS_SCALEARGBROWDOWN2_RVV)
+#if defined(HAS_SCALEARGBROWDOWN2_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ ScaleARGBRowDown2 = ScaleARGBRowDown2Box_SME;
+ }
+#endif
+#if defined(HAS_SCALEARGBROWDOWN2BOX_RVV)
if (TestCpuFlag(kCpuHasRVV)) {
ScaleARGBRowDown2 = ScaleARGBRowDown2Box_RVV;
}
@@ -363,6 +381,11 @@
}
}
#endif
+#if defined(HAS_INTERPOLATEROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ InterpolateRow = InterpolateRow_SME;
+ }
+#endif
#if defined(HAS_INTERPOLATEROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
InterpolateRow = InterpolateRow_Any_MSA;
@@ -413,6 +436,12 @@
}
}
#endif
+#if defined(HAS_SCALEARGBFILTERCOLS_RVV)
+ if (TestCpuFlag(kCpuHasRVV)) {
+ ScaleARGBFilterCols = ScaleARGBFilterCols_RVV;
+ }
+#endif
+
// TODO(fbarchard): Consider not allocating row buffer for kFilterLinear.
// Allocate a row of ARGB.
{
@@ -491,6 +520,11 @@
}
}
#endif
+#if defined(HAS_INTERPOLATEROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ InterpolateRow = InterpolateRow_SME;
+ }
+#endif
#if defined(HAS_INTERPOLATEROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
InterpolateRow = InterpolateRow_Any_MSA;
@@ -545,6 +579,11 @@
}
}
#endif
+#if defined(HAS_SCALEARGBFILTERCOLS_RVV)
+ if (filtering && TestCpuFlag(kCpuHasRVV)) {
+ ScaleARGBFilterCols = ScaleARGBFilterCols_RVV;
+ }
+#endif
#if defined(HAS_SCALEARGBCOLS_SSE2)
if (!filtering && TestCpuFlag(kCpuHasSSE2) && src_width < 32768) {
ScaleARGBFilterCols = ScaleARGBCols_SSE2;
@@ -698,6 +737,16 @@
}
}
#endif
+#if defined(HAS_I422TOARGBROW_SVE2)
+ if (TestCpuFlag(kCpuHasSVE2)) {
+ I422ToARGBRow = I422ToARGBRow_SVE2;
+ }
+#endif
+#if defined(HAS_I422TOARGBROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ I422ToARGBRow = I422ToARGBRow_SME;
+ }
+#endif
#if defined(HAS_I422TOARGBROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
I422ToARGBRow = I422ToARGBRow_Any_MSA;
@@ -755,6 +804,11 @@
}
}
#endif
+#if defined(HAS_INTERPOLATEROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ InterpolateRow = InterpolateRow_SME;
+ }
+#endif
#if defined(HAS_INTERPOLATEROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
InterpolateRow = InterpolateRow_Any_MSA;
@@ -813,6 +867,11 @@
}
}
#endif
+#if defined(HAS_SCALEARGBFILTERCOLS_RVV)
+ if (filtering && TestCpuFlag(kCpuHasRVV)) {
+ ScaleARGBFilterCols = ScaleARGBFilterCols_RVV;
+ }
+#endif
#if defined(HAS_SCALEARGBCOLS_SSE2)
if (!filtering && TestCpuFlag(kCpuHasSSE2) && src_width < 32768) {
ScaleARGBFilterCols = ScaleARGBCols_SSE2;
@@ -1054,14 +1113,14 @@
} else {
// Optimized even scale down. ie 2, 4, 6, 8, 10x.
if (!(dx & 0x10000) && !(dy & 0x10000)) {
- if (dx == 0x20000) {
+ if (dx == 0x20000 && dy == 0x20000) {
// Optimized 1/2 downsample.
ScaleARGBDown2(src_width, src_height, clip_width, clip_height,
src_stride, dst_stride, src, dst, x, dx, y, dy,
filtering);
return 0;
}
- if (dx == 0x40000 && filtering == kFilterBox) {
+ if (dx == 0x40000 && dy == 0x40000 && filtering == kFilterBox) {
// Optimized 1/4 box downsample.
return ScaleARGBDown4Box(src_width, src_height, clip_width,
clip_height, src_stride, dst_stride, src,
@@ -1173,18 +1232,28 @@
int clip_height,
enum FilterMode filtering) {
int r;
- uint8_t* argb_buffer = (uint8_t*)malloc(src_width * src_height * 4);
+ (void)src_fourcc; // TODO(fbarchard): implement and/or assert.
+ (void)dst_fourcc;
+ const int abs_src_height = (src_height < 0) ? -src_height : src_height;
+ if (!src_y || !src_u || !src_v || !dst_argb || src_width <= 0 ||
+ src_width > INT_MAX / 4 || src_height == 0 || dst_width <= 0 ||
+ dst_height <= 0 || clip_width <= 0 || clip_height <= 0) {
+ return -1;
+ }
+ const uint64_t argb_buffer_size = (uint64_t)src_width * abs_src_height * 4;
+ if (argb_buffer_size > SIZE_MAX) {
+ return -1; // Invalid size.
+ }
+ uint8_t* argb_buffer = (uint8_t*)malloc((size_t)argb_buffer_size);
if (!argb_buffer) {
return 1; // Out of memory runtime error.
}
- (void)src_fourcc; // TODO(fbarchard): implement and/or assert.
- (void)dst_fourcc;
I420ToARGB(src_y, src_stride_y, src_u, src_stride_u, src_v, src_stride_v,
argb_buffer, src_width * 4, src_width, src_height);
- r = ARGBScaleClip(argb_buffer, src_width * 4, src_width, src_height, dst_argb,
- dst_stride_argb, dst_width, dst_height, clip_x, clip_y,
- clip_width, clip_height, filtering);
+ r = ARGBScaleClip(argb_buffer, src_width * 4, src_width, abs_src_height,
+ dst_argb, dst_stride_argb, dst_width, dst_height, clip_x,
+ clip_y, clip_width, clip_height, filtering);
free(argb_buffer);
return r;
}
diff --git a/source/scale_common.cc b/source/scale_common.cc
index d07a39a..430afc2 100644
--- a/source/scale_common.cc
+++ b/source/scale_common.cc
@@ -1662,6 +1662,11 @@
}
}
#endif
+#if defined(HAS_INTERPOLATEROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ InterpolateRow = InterpolateRow_SME;
+ }
+#endif
#if defined(HAS_INTERPOLATEROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
InterpolateRow = InterpolateRow_Any_MSA;
@@ -1755,6 +1760,11 @@
}
}
#endif
+#if defined(HAS_INTERPOLATEROW_16_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ InterpolateRow = InterpolateRow_16_SME;
+ }
+#endif
for (j = 0; j < dst_height; ++j) {
int yi;
int yf;
@@ -1811,6 +1821,11 @@
}
}
#endif
+#if defined(HAS_INTERPOLATEROW_16TO8_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ InterpolateRow_16To8 = InterpolateRow_16To8_SME;
+ }
+#endif
#if defined(HAS_INTERPOLATEROW_16TO8_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
InterpolateRow_16To8 = InterpolateRow_16To8_Any_AVX2;
diff --git a/source/scale_gcc.cc b/source/scale_gcc.cc
index 17eeffa..92f9e70 100644
--- a/source/scale_gcc.cc
+++ b/source/scale_gcc.cc
@@ -17,7 +17,9 @@
#endif
// This module is for GCC x86 and x64.
-#if !defined(LIBYUV_DISABLE_X86) && (defined(__x86_64__) || defined(__i386__))
+#if !defined(LIBYUV_DISABLE_X86) && \
+ (defined(__x86_64__) || defined(__i386__)) && \
+ !defined(LIBYUV_ENABLE_ROWWIN)
// Offsets for source bytes 0 to 9
static const uvec8 kShuf0 = {0, 1, 3, 4, 5, 7, 8, 9,
@@ -100,7 +102,7 @@
asm volatile(
// 16 pixel loop.
LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n"
"movdqu 0x10(%0),%%xmm1 \n"
"lea 0x20(%0),%0 \n"
@@ -114,8 +116,8 @@
: "+r"(src_ptr), // %0
"+r"(dst_ptr), // %1
"+r"(dst_width) // %2
- ::"memory",
- "cc", "xmm0", "xmm1");
+ :
+ : "memory", "cc", "xmm0", "xmm1");
}
void ScaleRowDown2Linear_SSSE3(const uint8_t* src_ptr,
@@ -124,13 +126,13 @@
int dst_width) {
(void)src_stride;
asm volatile(
- "pcmpeqb %%xmm4,%%xmm4 \n"
- "psrlw $0xf,%%xmm4 \n"
- "packuswb %%xmm4,%%xmm4 \n"
+ "pcmpeqb %%xmm4,%%xmm4 \n" // 0x0101
+ "pabsb %%xmm4,%%xmm4 \n"
+
"pxor %%xmm5,%%xmm5 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n"
"movdqu 0x10(%0),%%xmm1 \n"
"lea 0x20(%0),%0 \n"
@@ -146,8 +148,8 @@
: "+r"(src_ptr), // %0
"+r"(dst_ptr), // %1
"+r"(dst_width) // %2
- ::"memory",
- "cc", "xmm0", "xmm1", "xmm4", "xmm5");
+ :
+ : "memory", "cc", "xmm0", "xmm1", "xmm4", "xmm5");
}
void ScaleRowDown2Box_SSSE3(const uint8_t* src_ptr,
@@ -155,13 +157,12 @@
uint8_t* dst_ptr,
int dst_width) {
asm volatile(
- "pcmpeqb %%xmm4,%%xmm4 \n"
- "psrlw $0xf,%%xmm4 \n"
- "packuswb %%xmm4,%%xmm4 \n"
+ "pcmpeqb %%xmm4,%%xmm4 \n" // 0x0101
+ "pabsb %%xmm4,%%xmm4 \n"
"pxor %%xmm5,%%xmm5 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n"
"movdqu 0x10(%0),%%xmm1 \n"
"movdqu 0x00(%0,%3,1),%%xmm2 \n"
@@ -195,8 +196,8 @@
uint8_t* dst_ptr,
int dst_width) {
(void)src_stride;
- asm volatile(LABELALIGN
- "1: \n"
+ asm volatile(
+ "1: \n"
"vmovdqu (%0),%%ymm0 \n"
"vmovdqu 0x20(%0),%%ymm1 \n"
"lea 0x40(%0),%0 \n"
@@ -208,12 +209,12 @@
"lea 0x20(%1),%1 \n"
"sub $0x20,%2 \n"
"jg 1b \n"
- "vzeroupper \n"
- : "+r"(src_ptr), // %0
- "+r"(dst_ptr), // %1
- "+r"(dst_width) // %2
- ::"memory",
- "cc", "xmm0", "xmm1");
+ "vzeroupper \n"
+ : "+r"(src_ptr), // %0
+ "+r"(dst_ptr), // %1
+ "+r"(dst_width) // %2
+ :
+ : "memory", "cc", "xmm0", "xmm1");
}
void ScaleRowDown2Linear_AVX2(const uint8_t* src_ptr,
@@ -223,12 +224,11 @@
(void)src_stride;
asm volatile(
"vpcmpeqb %%ymm4,%%ymm4,%%ymm4 \n"
- "vpsrlw $0xf,%%ymm4,%%ymm4 \n"
- "vpackuswb %%ymm4,%%ymm4,%%ymm4 \n"
+ "vpabsb %%ymm4,%%ymm4 \n"
"vpxor %%ymm5,%%ymm5,%%ymm5 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"vmovdqu (%0),%%ymm0 \n"
"vmovdqu 0x20(%0),%%ymm1 \n"
"lea 0x40(%0),%0 \n"
@@ -242,12 +242,12 @@
"lea 0x20(%1),%1 \n"
"sub $0x20,%2 \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: "+r"(src_ptr), // %0
"+r"(dst_ptr), // %1
"+r"(dst_width) // %2
- ::"memory",
- "cc", "xmm0", "xmm1", "xmm4", "xmm5");
+ :
+ : "memory", "cc", "xmm0", "xmm1", "xmm4", "xmm5");
}
void ScaleRowDown2Box_AVX2(const uint8_t* src_ptr,
@@ -256,12 +256,11 @@
int dst_width) {
asm volatile(
"vpcmpeqb %%ymm4,%%ymm4,%%ymm4 \n"
- "vpsrlw $0xf,%%ymm4,%%ymm4 \n"
- "vpackuswb %%ymm4,%%ymm4,%%ymm4 \n"
+ "vpabsb %%ymm4,%%ymm4 \n"
"vpxor %%ymm5,%%ymm5,%%ymm5 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"vmovdqu (%0),%%ymm0 \n"
"vmovdqu 0x20(%0),%%ymm1 \n"
"vmovdqu 0x00(%0,%3,1),%%ymm2 \n"
@@ -283,7 +282,7 @@
"lea 0x20(%1),%1 \n"
"sub $0x20,%2 \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: "+r"(src_ptr), // %0
"+r"(dst_ptr), // %1
"+r"(dst_width) // %2
@@ -303,7 +302,7 @@
"pslld $0x10,%%xmm5 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n"
"movdqu 0x10(%0),%%xmm1 \n"
"lea 0x20(%0),%0 \n"
@@ -319,8 +318,8 @@
: "+r"(src_ptr), // %0
"+r"(dst_ptr), // %1
"+r"(dst_width) // %2
- ::"memory",
- "cc", "xmm0", "xmm1", "xmm5");
+ :
+ : "memory", "cc", "xmm0", "xmm1", "xmm5");
}
void ScaleRowDown4Box_SSSE3(const uint8_t* src_ptr,
@@ -330,14 +329,13 @@
intptr_t stridex3;
asm volatile(
"pcmpeqb %%xmm4,%%xmm4 \n"
- "psrlw $0xf,%%xmm4 \n"
- "movdqa %%xmm4,%%xmm5 \n"
- "packuswb %%xmm4,%%xmm4 \n"
- "psllw $0x3,%%xmm5 \n"
+ "pabsw %%xmm4,%%xmm5 \n"
+ "pabsb %%xmm4,%%xmm4 \n" // 0x0101
+ "psllw $0x3,%%xmm5 \n" // 0x0008
"lea 0x00(%4,%4,2),%3 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n"
"movdqu 0x10(%0),%%xmm1 \n"
"movdqu 0x00(%0,%4,1),%%xmm2 \n"
@@ -389,7 +387,7 @@
"vpslld $0x10,%%ymm5,%%ymm5 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"vmovdqu (%0),%%ymm0 \n"
"vmovdqu 0x20(%0),%%ymm1 \n"
"lea 0x40(%0),%0 \n"
@@ -404,12 +402,12 @@
"lea 0x10(%1),%1 \n"
"sub $0x10,%2 \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: "+r"(src_ptr), // %0
"+r"(dst_ptr), // %1
"+r"(dst_width) // %2
- ::"memory",
- "cc", "xmm0", "xmm1", "xmm5");
+ :
+ : "memory", "cc", "xmm0", "xmm1", "xmm5");
}
void ScaleRowDown4Box_AVX2(const uint8_t* src_ptr,
@@ -418,12 +416,12 @@
int dst_width) {
asm volatile(
"vpcmpeqb %%ymm4,%%ymm4,%%ymm4 \n"
- "vpsrlw $0xf,%%ymm4,%%ymm4 \n"
- "vpsllw $0x3,%%ymm4,%%ymm5 \n"
- "vpackuswb %%ymm4,%%ymm4,%%ymm4 \n"
+ "vpabsw %%ymm4,%%ymm5 \n"
+ "vpabsb %%ymm4,%%ymm4 \n" // 0x0101
+ "vpsllw $0x3,%%ymm5,%%ymm5 \n" // 0x0008
LABELALIGN
- "1: \n"
+ "1: \n"
"vmovdqu (%0),%%ymm0 \n"
"vmovdqu 0x20(%0),%%ymm1 \n"
"vmovdqu 0x00(%0,%3,1),%%ymm2 \n"
@@ -457,7 +455,7 @@
"lea 0x10(%1),%1 \n"
"sub $0x10,%2 \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: "+r"(src_ptr), // %0
"+r"(dst_ptr), // %1
"+r"(dst_width) // %2
@@ -481,8 +479,8 @@
"m"(kShuf1), // %1
"m"(kShuf2) // %2
);
- asm volatile(LABELALIGN
- "1: \n"
+ asm volatile(
+ "1: \n"
"movdqu (%0),%%xmm0 \n"
"movdqu 0x10(%0),%%xmm2 \n"
"lea 0x20(%0),%0 \n"
@@ -497,11 +495,11 @@
"lea 0x18(%1),%1 \n"
"sub $0x18,%2 \n"
"jg 1b \n"
- : "+r"(src_ptr), // %0
- "+r"(dst_ptr), // %1
- "+r"(dst_width) // %2
- ::"memory",
- "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
+ : "+r"(src_ptr), // %0
+ "+r"(dst_ptr), // %1
+ "+r"(dst_width) // %2
+ :
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
}
void ScaleRowDown34_1_Box_SSSE3(const uint8_t* src_ptr,
@@ -526,8 +524,8 @@
"m"(kMadd11), // %1
"m"(kRound34) // %2
);
- asm volatile(LABELALIGN
- "1: \n"
+ asm volatile(
+ "1: \n"
"movdqu (%0),%%xmm6 \n"
"movdqu 0x00(%0,%3,1),%%xmm7 \n"
"pavgb %%xmm7,%%xmm6 \n"
@@ -559,13 +557,13 @@
"lea 0x18(%1),%1 \n"
"sub $0x18,%2 \n"
"jg 1b \n"
- : "+r"(src_ptr), // %0
- "+r"(dst_ptr), // %1
- "+r"(dst_width) // %2
- : "r"((intptr_t)(src_stride)), // %3
- "m"(kMadd21) // %4
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
- "xmm6", "xmm7");
+ : "+r"(src_ptr), // %0
+ "+r"(dst_ptr), // %1
+ "+r"(dst_width) // %2
+ : "r"((intptr_t)(src_stride)), // %3
+ "m"(kMadd21) // %4
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ "xmm7");
}
void ScaleRowDown34_0_Box_SSSE3(const uint8_t* src_ptr,
@@ -591,8 +589,8 @@
"m"(kRound34) // %2
);
- asm volatile(LABELALIGN
- "1: \n"
+ asm volatile(
+ "1: \n"
"movdqu (%0),%%xmm6 \n"
"movdqu 0x00(%0,%3,1),%%xmm7 \n"
"pavgb %%xmm6,%%xmm7 \n"
@@ -627,13 +625,13 @@
"lea 0x18(%1),%1 \n"
"sub $0x18,%2 \n"
"jg 1b \n"
- : "+r"(src_ptr), // %0
- "+r"(dst_ptr), // %1
- "+r"(dst_width) // %2
- : "r"((intptr_t)(src_stride)), // %3
- "m"(kMadd21) // %4
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
- "xmm6", "xmm7");
+ : "+r"(src_ptr), // %0
+ "+r"(dst_ptr), // %1
+ "+r"(dst_width) // %2
+ : "r"((intptr_t)(src_stride)), // %3
+ "m"(kMadd21) // %4
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ "xmm7");
}
void ScaleRowDown38_SSSE3(const uint8_t* src_ptr,
@@ -646,7 +644,7 @@
"movdqa %4,%%xmm5 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n"
"movdqu 0x10(%0),%%xmm1 \n"
"lea 0x20(%0),%0 \n"
@@ -682,8 +680,8 @@
"m"(kShufAb2), // %2
"m"(kScaleAb2) // %3
);
- asm volatile(LABELALIGN
- "1: \n"
+ asm volatile(
+ "1: \n"
"movdqu (%0),%%xmm0 \n"
"movdqu 0x00(%0,%3,1),%%xmm1 \n"
"lea 0x10(%0),%0 \n"
@@ -703,12 +701,11 @@
"lea 0x6(%1),%1 \n"
"sub $0x6,%2 \n"
"jg 1b \n"
- : "+r"(src_ptr), // %0
- "+r"(dst_ptr), // %1
- "+r"(dst_width) // %2
- : "r"((intptr_t)(src_stride)) // %3
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
- "xmm6");
+ : "+r"(src_ptr), // %0
+ "+r"(dst_ptr), // %1
+ "+r"(dst_width) // %2
+ : "r"((intptr_t)(src_stride)) // %3
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6");
}
void ScaleRowDown38_3_Box_SSSE3(const uint8_t* src_ptr,
@@ -725,8 +722,8 @@
"m"(kShufAc3), // %1
"m"(kScaleAc33) // %2
);
- asm volatile(LABELALIGN
- "1: \n"
+ asm volatile(
+ "1: \n"
"movdqu (%0),%%xmm0 \n"
"movdqu 0x00(%0,%3,1),%%xmm6 \n"
"movhlps %%xmm0,%%xmm1 \n"
@@ -765,12 +762,12 @@
"lea 0x6(%1),%1 \n"
"sub $0x6,%2 \n"
"jg 1b \n"
- : "+r"(src_ptr), // %0
- "+r"(dst_ptr), // %1
- "+r"(dst_width) // %2
- : "r"((intptr_t)(src_stride)) // %3
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5",
- "xmm6", "xmm7");
+ : "+r"(src_ptr), // %0
+ "+r"(dst_ptr), // %1
+ "+r"(dst_width) // %2
+ : "r"((intptr_t)(src_stride)) // %3
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6",
+ "xmm7");
}
static const uvec8 kLinearShuffleFar = {2, 3, 0, 1, 6, 7, 4, 5,
@@ -790,7 +787,7 @@
"psllw $1,%%xmm6 \n" // all 2
LABELALIGN
- "1: \n"
+ "1: \n"
"movq (%0),%%xmm1 \n" // 01234567
"movq 1(%0),%%xmm2 \n" // 12345678
"movdqa %%xmm1,%%xmm3 \n"
@@ -840,8 +837,7 @@
ptrdiff_t dst_stride,
int dst_width) {
asm volatile(
- LABELALIGN
- "1: \n"
+ "1: \n"
"pxor %%xmm0,%%xmm0 \n" // 0
// above line
"movq (%0),%%xmm1 \n" // 01234567
@@ -960,7 +956,7 @@
"psllw $1,%%xmm4 \n" // all 2
LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n" // 01234567 (16)
"movdqu 2(%0),%%xmm1 \n" // 12345678 (16)
@@ -1012,7 +1008,7 @@
"movdqa %5,%%xmm6 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
// above line
"movdqu (%0),%%xmm0 \n" // 01234567 (16)
"movdqu 2(%0),%%xmm1 \n" // 12345678 (16)
@@ -1110,7 +1106,7 @@
"pslld $1,%%xmm4 \n" // all 2
LABELALIGN
- "1: \n"
+ "1: \n"
"movq (%0),%%xmm0 \n" // 0123 (16b)
"movq 2(%0),%%xmm1 \n" // 1234 (16b)
@@ -1163,7 +1159,7 @@
"pslld $3,%%xmm6 \n" // all 8
LABELALIGN
- "1: \n"
+ "1: \n"
"movq (%0),%%xmm0 \n" // 0011 (16b, 1u1v)
"movq 4(%0),%%xmm1 \n" // 1122 (16b, 1u1v)
"punpcklwd %%xmm7,%%xmm0 \n" // 0011 (near) (32b, 1u1v)
@@ -1271,7 +1267,7 @@
"movdqa %3,%%xmm3 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"movq (%0),%%xmm0 \n" // 01234567
"movq 1(%0),%%xmm1 \n" // 12345678
"punpcklwd %%xmm0,%%xmm0 \n" // 0101232345456767
@@ -1312,7 +1308,7 @@
"movdqa %5,%%xmm7 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"movq (%0),%%xmm0 \n" // 01234567
"movq 1(%0),%%xmm1 \n" // 12345678
"punpcklwd %%xmm0,%%xmm0 \n" // 0101232345456767
@@ -1397,7 +1393,7 @@
"vbroadcastf128 %3,%%ymm3 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"vmovdqu (%0),%%xmm0 \n" // 0123456789ABCDEF
"vmovdqu 1(%0),%%xmm1 \n" // 123456789ABCDEF0
"vpermq $0b11011000,%%ymm0,%%ymm0 \n"
@@ -1419,7 +1415,7 @@
"lea 0x20(%1),%1 \n" // 16 sample to 32 sample
"sub $0x20,%2 \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: "+r"(src_ptr), // %0
"+r"(dst_ptr), // %1
"+r"(dst_width) // %2
@@ -1441,7 +1437,7 @@
"vbroadcastf128 %5,%%ymm7 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"vmovdqu (%0),%%xmm0 \n" // 0123456789ABCDEF
"vmovdqu 1(%0),%%xmm1 \n" // 123456789ABCDEF0
"vpermq $0b11011000,%%ymm0,%%ymm0 \n"
@@ -1500,7 +1496,7 @@
"lea 0x20(%1),%1 \n" // 16 sample to 32 sample
"sub $0x20,%2 \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: "+r"(src_ptr), // %0
"+r"(dst_ptr), // %1
"+r"(dst_width) // %2
@@ -1523,7 +1519,7 @@
"vpsllw $1,%%ymm4,%%ymm4 \n" // all 2
LABELALIGN
- "1: \n"
+ "1: \n"
"vmovdqu (%0),%%ymm0 \n" // 0123456789ABCDEF (16b)
"vmovdqu 2(%0),%%ymm1 \n" // 123456789ABCDEF0 (16b)
@@ -1553,7 +1549,7 @@
"lea 0x40(%1),%1 \n" // 16 sample to 32 sample
"sub $0x20,%2 \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: "+r"(src_ptr), // %0
"+r"(dst_ptr), // %1
"+r"(dst_width) // %2
@@ -1575,7 +1571,7 @@
"vpsllw $3,%%ymm4,%%ymm4 \n" // all 8
LABELALIGN
- "1: \n"
+ "1: \n"
"vmovdqu (%0),%%xmm0 \n" // 01234567 (16b)
"vmovdqu 2(%0),%%xmm1 \n" // 12345678 (16b)
@@ -1615,7 +1611,7 @@
"lea 0x20(%1),%1 \n" // 8 sample to 16 sample
"sub $0x10,%2 \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: "+r"(src_ptr), // %0
"+r"(dst_ptr), // %1
"+r"(dst_width) // %2
@@ -1636,7 +1632,7 @@
"vpslld $1,%%ymm4,%%ymm4 \n" // all 2
LABELALIGN
- "1: \n"
+ "1: \n"
"vmovdqu (%0),%%xmm0 \n" // 01234567 (16b, 1u1v)
"vmovdqu 2(%0),%%xmm1 \n" // 12345678 (16b, 1u1v)
@@ -1665,7 +1661,7 @@
"lea 0x20(%1),%1 \n" // 8 pixel to 16 pixel
"sub $0x10,%2 \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: "+r"(src_ptr), // %0
"+r"(dst_ptr), // %1
"+r"(dst_width) // %2
@@ -1686,7 +1682,7 @@
"vpslld $3,%%ymm6,%%ymm6 \n" // all 8
LABELALIGN
- "1: \n"
+ "1: \n"
"vmovdqu (%0),%%xmm0 \n" // 01234567 (16b, 1u1v)
"vmovdqu 2(%0),%%xmm1 \n" // 12345678 (16b, 1u1v)
@@ -1749,7 +1745,7 @@
"lea 0x20(%1),%1 \n" // 8 pixel to 16 pixel
"sub $0x10,%2 \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: "+r"(src_ptr), // %0
"+r"(dst_ptr), // %1
"+r"(dst_width) // %2
@@ -1763,25 +1759,25 @@
void ScaleAddRow_SSE2(const uint8_t* src_ptr,
uint16_t* dst_ptr,
int src_width) {
- asm volatile("pxor %%xmm5,%%xmm5 \n"
+ asm volatile("pxor %%xmm5,%%xmm5 \n"
// 16 pixel loop.
LABELALIGN
- "1: \n"
- "movdqu (%0),%%xmm3 \n"
- "lea 0x10(%0),%0 \n" // src_ptr += 16
- "movdqu (%1),%%xmm0 \n"
- "movdqu 0x10(%1),%%xmm1 \n"
- "movdqa %%xmm3,%%xmm2 \n"
- "punpcklbw %%xmm5,%%xmm2 \n"
- "punpckhbw %%xmm5,%%xmm3 \n"
- "paddusw %%xmm2,%%xmm0 \n"
- "paddusw %%xmm3,%%xmm1 \n"
- "movdqu %%xmm0,(%1) \n"
- "movdqu %%xmm1,0x10(%1) \n"
- "lea 0x20(%1),%1 \n"
- "sub $0x10,%2 \n"
- "jg 1b \n"
+ "1: \n"
+ "movdqu (%0),%%xmm3 \n"
+ "lea 0x10(%0),%0 \n" // src_ptr += 16
+ "movdqu (%1),%%xmm0 \n"
+ "movdqu 0x10(%1),%%xmm1 \n"
+ "movdqa %%xmm3,%%xmm2 \n"
+ "punpcklbw %%xmm5,%%xmm2 \n"
+ "punpckhbw %%xmm5,%%xmm3 \n"
+ "paddusw %%xmm2,%%xmm0 \n"
+ "paddusw %%xmm3,%%xmm1 \n"
+ "movdqu %%xmm0,(%1) \n"
+ "movdqu %%xmm1,0x10(%1) \n"
+ "lea 0x20(%1),%1 \n"
+ "sub $0x10,%2 \n"
+ "jg 1b \n"
: "+r"(src_ptr), // %0
"+r"(dst_ptr), // %1
"+r"(src_width) // %2
@@ -1794,23 +1790,23 @@
void ScaleAddRow_AVX2(const uint8_t* src_ptr,
uint16_t* dst_ptr,
int src_width) {
- asm volatile("vpxor %%ymm5,%%ymm5,%%ymm5 \n"
+ asm volatile("vpxor %%ymm5,%%ymm5,%%ymm5 \n"
LABELALIGN
- "1: \n"
- "vmovdqu (%0),%%ymm3 \n"
- "lea 0x20(%0),%0 \n" // src_ptr += 32
- "vpermq $0xd8,%%ymm3,%%ymm3 \n"
- "vpunpcklbw %%ymm5,%%ymm3,%%ymm2 \n"
- "vpunpckhbw %%ymm5,%%ymm3,%%ymm3 \n"
- "vpaddusw (%1),%%ymm2,%%ymm0 \n"
- "vpaddusw 0x20(%1),%%ymm3,%%ymm1 \n"
- "vmovdqu %%ymm0,(%1) \n"
- "vmovdqu %%ymm1,0x20(%1) \n"
- "lea 0x40(%1),%1 \n"
- "sub $0x20,%2 \n"
- "jg 1b \n"
- "vzeroupper \n"
+ "1: \n"
+ "vmovdqu (%0),%%ymm3 \n"
+ "lea 0x20(%0),%0 \n" // src_ptr += 32
+ "vpermq $0xd8,%%ymm3,%%ymm3 \n"
+ "vpunpcklbw %%ymm5,%%ymm3,%%ymm2 \n"
+ "vpunpckhbw %%ymm5,%%ymm3,%%ymm3 \n"
+ "vpaddusw (%1),%%ymm2,%%ymm0 \n"
+ "vpaddusw 0x20(%1),%%ymm3,%%ymm1 \n"
+ "vmovdqu %%ymm0,(%1) \n"
+ "vmovdqu %%ymm1,0x20(%1) \n"
+ "lea 0x40(%1),%1 \n"
+ "sub $0x20,%2 \n"
+ "jg 1b \n"
+ "vzeroupper \n"
: "+r"(src_ptr), // %0
"+r"(dst_ptr), // %1
"+r"(src_width) // %2
@@ -1856,7 +1852,7 @@
"pextrw $0x3,%%xmm2,%k4 \n"
LABELALIGN
- "2: \n"
+ "2: \n"
"movdqa %%xmm2,%%xmm1 \n"
"paddd %%xmm3,%%xmm2 \n"
"movzwl 0x00(%1,%3,1),%k2 \n"
@@ -1883,7 +1879,7 @@
"jge 2b \n"
LABELALIGN
- "29: \n"
+ "29: \n"
"addl $0x1,%5 \n"
"jl 99f \n"
"movzwl 0x00(%1,%3,1),%k2 \n"
@@ -1899,7 +1895,7 @@
"packuswb %%xmm2,%%xmm2 \n"
"movd %%xmm2,%k2 \n"
"mov %b2,(%0) \n"
- "99: \n"
+ "99: \n"
: "+r"(dst_ptr), // %0
"+r"(src_ptr), // %1
"=&a"(temp_pixel), // %2
@@ -1932,8 +1928,8 @@
int dx) {
(void)x;
(void)dx;
- asm volatile(LABELALIGN
- "1: \n"
+ asm volatile(
+ "1: \n"
"movdqu (%1),%%xmm0 \n"
"lea 0x10(%1),%1 \n"
"movdqa %%xmm0,%%xmm1 \n"
@@ -1945,11 +1941,11 @@
"sub $0x20,%2 \n"
"jg 1b \n"
- : "+r"(dst_ptr), // %0
- "+r"(src_ptr), // %1
- "+r"(dst_width) // %2
- ::"memory",
- "cc", "xmm0", "xmm1");
+ : "+r"(dst_ptr), // %0
+ "+r"(src_ptr), // %1
+ "+r"(dst_width) // %2
+ :
+ : "memory", "cc", "xmm0", "xmm1");
}
void ScaleARGBRowDown2_SSE2(const uint8_t* src_argb,
@@ -1957,8 +1953,8 @@
uint8_t* dst_argb,
int dst_width) {
(void)src_stride;
- asm volatile(LABELALIGN
- "1: \n"
+ asm volatile(
+ "1: \n"
"movdqu (%0),%%xmm0 \n"
"movdqu 0x10(%0),%%xmm1 \n"
"lea 0x20(%0),%0 \n"
@@ -1967,11 +1963,11 @@
"lea 0x10(%1),%1 \n"
"sub $0x4,%2 \n"
"jg 1b \n"
- : "+r"(src_argb), // %0
- "+r"(dst_argb), // %1
- "+r"(dst_width) // %2
- ::"memory",
- "cc", "xmm0", "xmm1");
+ : "+r"(src_argb), // %0
+ "+r"(dst_argb), // %1
+ "+r"(dst_width) // %2
+ :
+ : "memory", "cc", "xmm0", "xmm1");
}
void ScaleARGBRowDown2Linear_SSE2(const uint8_t* src_argb,
@@ -1979,8 +1975,8 @@
uint8_t* dst_argb,
int dst_width) {
(void)src_stride;
- asm volatile(LABELALIGN
- "1: \n"
+ asm volatile(
+ "1: \n"
"movdqu (%0),%%xmm0 \n"
"movdqu 0x10(%0),%%xmm1 \n"
"lea 0x20(%0),%0 \n"
@@ -1992,19 +1988,19 @@
"lea 0x10(%1),%1 \n"
"sub $0x4,%2 \n"
"jg 1b \n"
- : "+r"(src_argb), // %0
- "+r"(dst_argb), // %1
- "+r"(dst_width) // %2
- ::"memory",
- "cc", "xmm0", "xmm1");
+ : "+r"(src_argb), // %0
+ "+r"(dst_argb), // %1
+ "+r"(dst_width) // %2
+ :
+ : "memory", "cc", "xmm0", "xmm1");
}
void ScaleARGBRowDown2Box_SSE2(const uint8_t* src_argb,
ptrdiff_t src_stride,
uint8_t* dst_argb,
int dst_width) {
- asm volatile(LABELALIGN
- "1: \n"
+ asm volatile(
+ "1: \n"
"movdqu (%0),%%xmm0 \n"
"movdqu 0x10(%0),%%xmm1 \n"
"movdqu 0x00(%0,%3,1),%%xmm2 \n"
@@ -2020,11 +2016,11 @@
"lea 0x10(%1),%1 \n"
"sub $0x4,%2 \n"
"jg 1b \n"
- : "+r"(src_argb), // %0
- "+r"(dst_argb), // %1
- "+r"(dst_width) // %2
- : "r"((intptr_t)(src_stride)) // %3
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3");
+ : "+r"(src_argb), // %0
+ "+r"(dst_argb), // %1
+ "+r"(dst_width) // %2
+ : "r"((intptr_t)(src_stride)) // %3
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3");
}
// Reads 4 pixels at a time.
@@ -2042,7 +2038,7 @@
"lea 0x00(%1,%1,2),%4 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"movd (%0),%%xmm0 \n"
"movd 0x00(%0,%1,1),%%xmm1 \n"
"punpckldq %%xmm1,%%xmm0 \n"
@@ -2060,8 +2056,8 @@
"+r"(dst_argb), // %2
"+r"(dst_width), // %3
"=&r"(src_stepx_x12) // %4
- ::"memory",
- "cc", "xmm0", "xmm1", "xmm2", "xmm3");
+ :
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3");
}
// Blends four 2x2 to 4x1.
@@ -2080,7 +2076,7 @@
"lea 0x00(%0,%5,1),%5 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"movq (%0),%%xmm0 \n"
"movhps 0x00(%0,%1,1),%%xmm0 \n"
"movq 0x00(%0,%1,2),%%xmm1 \n"
@@ -2107,8 +2103,8 @@
"+rm"(dst_width), // %3
"=&r"(src_stepx_x12), // %4
"+r"(row1) // %5
- ::"memory",
- "cc", "xmm0", "xmm1", "xmm2", "xmm3");
+ :
+ : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3");
}
void ScaleARGBCols_SSE2(uint8_t* dst_argb,
@@ -2136,7 +2132,7 @@
"jl 49f \n"
LABELALIGN
- "40: \n"
+ "40: \n"
"movd 0x00(%3,%0,4),%%xmm0 \n"
"movd 0x00(%3,%1,4),%%xmm1 \n"
"pextrw $0x5,%%xmm2,%k0 \n"
@@ -2154,7 +2150,7 @@
"sub $0x4,%4 \n"
"jge 40b \n"
- "49: \n"
+ "49: \n"
"test $0x2,%4 \n"
"je 29f \n"
"movd 0x00(%3,%0,4),%%xmm0 \n"
@@ -2163,12 +2159,12 @@
"punpckldq %%xmm1,%%xmm0 \n"
"movq %%xmm0,(%2) \n"
"lea 0x8(%2),%2 \n"
- "29: \n"
+ "29: \n"
"test $0x1,%4 \n"
"je 99f \n"
"movd 0x00(%3,%0,4),%%xmm0 \n"
"movd %%xmm0,(%2) \n"
- "99: \n"
+ "99: \n"
: "=&a"(x0), // %0
"=&d"(x1), // %1
"+r"(dst_argb), // %2
@@ -2188,8 +2184,8 @@
int dx) {
(void)x;
(void)dx;
- asm volatile(LABELALIGN
- "1: \n"
+ asm volatile(
+ "1: \n"
"movdqu (%1),%%xmm0 \n"
"lea 0x10(%1),%1 \n"
"movdqa %%xmm0,%%xmm1 \n"
@@ -2201,11 +2197,11 @@
"sub $0x8,%2 \n"
"jg 1b \n"
- : "+r"(dst_argb), // %0
- "+r"(src_argb), // %1
- "+r"(dst_width) // %2
- ::"memory",
- "cc", "xmm0", "xmm1");
+ : "+r"(dst_argb), // %0
+ "+r"(src_argb), // %1
+ "+r"(dst_width) // %2
+ :
+ : "memory", "cc", "xmm0", "xmm1");
}
// Shuffle table for arranging 2 pixels into pairs for pmaddubsw
@@ -2250,7 +2246,7 @@
"pextrw $0x3,%%xmm2,%k4 \n"
LABELALIGN
- "2: \n"
+ "2: \n"
"movdqa %%xmm2,%%xmm1 \n"
"paddd %%xmm3,%%xmm2 \n"
"movq 0x00(%1,%3,4),%%xmm0 \n"
@@ -2270,7 +2266,7 @@
"jge 2b \n"
LABELALIGN
- "29: \n"
+ "29: \n"
"add $0x1,%2 \n"
"jl 99f \n"
"psrlw $0x9,%%xmm2 \n"
@@ -2283,8 +2279,7 @@
"packuswb %%xmm0,%%xmm0 \n"
"movd %%xmm0,(%0) \n"
- LABELALIGN
- "99: \n" // clang-format error.
+ LABELALIGN "99: \n"
: "+r"(dst_argb), // %0
"+r"(src_argb), // %1
@@ -2299,7 +2294,7 @@
// Divide num by div and return as 16.16 fixed point result.
int FixedDiv_X86(int num, int div) {
asm volatile(
- "cdq \n"
+ "cdq \n"
"shld $0x10,%%eax,%%edx \n"
"shl $0x10,%%eax \n"
"idiv %1 \n"
@@ -2313,7 +2308,7 @@
// Divide num - 1 by div - 1 and return as 16.16 fixed point result.
int FixedDiv1_X86(int num, int div) {
asm volatile(
- "cdq \n"
+ "cdq \n"
"shld $0x10,%%eax,%%edx \n"
"shl $0x10,%%eax \n"
"sub $0x10001,%%eax \n"
@@ -2353,7 +2348,7 @@
"movdqa %5,%%xmm3 \n" // merge shuffler
LABELALIGN
- "1: \n"
+ "1: \n"
"movdqu (%0),%%xmm0 \n" // 8 UV row 0
"movdqu 0x00(%0,%3,1),%%xmm2 \n" // 8 UV row 1
"lea 0x10(%0),%0 \n"
@@ -2386,14 +2381,13 @@
int dst_width) {
asm volatile(
"vpcmpeqb %%ymm4,%%ymm4,%%ymm4 \n" // 01010101
- "vpsrlw $0xf,%%ymm4,%%ymm4 \n"
- "vpackuswb %%ymm4,%%ymm4,%%ymm4 \n"
+ "vpabsb %%ymm4,%%ymm4 \n"
"vpxor %%ymm5,%%ymm5,%%ymm5 \n" // zero
"vbroadcastf128 %4,%%ymm1 \n" // split shuffler
"vbroadcastf128 %5,%%ymm3 \n" // merge shuffler
LABELALIGN
- "1: \n"
+ "1: \n"
"vmovdqu (%0),%%ymm0 \n" // 16 UV row 0
"vmovdqu 0x00(%0,%3,1),%%ymm2 \n" // 16 UV row 1
"lea 0x20(%0),%0 \n"
@@ -2410,7 +2404,7 @@
"lea 0x10(%1),%1 \n" // 8 UV
"sub $0x8,%2 \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: "+r"(src_ptr), // %0
"+r"(dst_ptr), // %1
"+r"(dst_width) // %2
@@ -2435,7 +2429,7 @@
"movdqa %3,%%xmm3 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"movq (%0),%%xmm0 \n" // 00112233 (1u1v)
"movq 2(%0),%%xmm1 \n" // 11223344 (1u1v)
"punpcklbw %%xmm1,%%xmm0 \n" // 0101121223233434 (2u2v)
@@ -2476,7 +2470,7 @@
"movdqa %5,%%xmm7 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"movq (%0),%%xmm0 \n" // 00112233 (1u1v)
"movq 2(%0),%%xmm1 \n" // 11223344 (1u1v)
"punpcklbw %%xmm1,%%xmm0 \n" // 0101121223233434 (2u2v)
@@ -2560,7 +2554,7 @@
"vbroadcastf128 %3,%%ymm3 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"vmovdqu (%0),%%xmm0 \n"
"vmovdqu 2(%0),%%xmm1 \n"
"vpermq $0b11011000,%%ymm0,%%ymm0 \n"
@@ -2581,7 +2575,7 @@
"lea 0x20(%1),%1 \n" // 8 uv to 16 uv
"sub $0x10,%2 \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: "+r"(src_ptr), // %0
"+r"(dst_ptr), // %1
"+r"(dst_width) // %2
@@ -2603,7 +2597,7 @@
"vbroadcastf128 %5,%%ymm7 \n"
LABELALIGN
- "1: \n"
+ "1: \n"
"vmovdqu (%0),%%xmm0 \n"
"vmovdqu 2(%0),%%xmm1 \n"
"vpermq $0b11011000,%%ymm0,%%ymm0 \n"
@@ -2660,7 +2654,7 @@
"lea 0x20(%1),%1 \n" // 8 uv to 16 uv
"sub $0x10,%2 \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: "+r"(src_ptr), // %0
"+r"(dst_ptr), // %1
"+r"(dst_width) // %2
@@ -2683,7 +2677,7 @@
"pslld $1,%%xmm4 \n" // all 2
LABELALIGN
- "1: \n"
+ "1: \n"
"movq (%0),%%xmm0 \n" // 0011 (16b, 1u1v)
"movq 4(%0),%%xmm1 \n" // 1122 (16b, 1u1v)
@@ -2735,7 +2729,7 @@
"pslld $3,%%xmm6 \n" // all 8
LABELALIGN
- "1: \n"
+ "1: \n"
"movq (%0),%%xmm0 \n" // 0011 (16b, 1u1v)
"movq 4(%0),%%xmm1 \n" // 1122 (16b, 1u1v)
"punpcklwd %%xmm7,%%xmm0 \n" // 0011 (near) (32b, 1u1v)
@@ -2825,7 +2819,7 @@
"vpslld $1,%%ymm4,%%ymm4 \n" // all 2
LABELALIGN
- "1: \n"
+ "1: \n"
"vmovdqu (%0),%%xmm0 \n" // 00112233 (16b, 1u1v)
"vmovdqu 4(%0),%%xmm1 \n" // 11223344 (16b, 1u1v)
@@ -2853,7 +2847,7 @@
"lea 0x20(%1),%1 \n" // 4 uv to 8 uv
"sub $0x8,%2 \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: "+r"(src_ptr), // %0
"+r"(dst_ptr), // %1
"+r"(dst_width) // %2
@@ -2874,7 +2868,7 @@
"vpslld $3,%%ymm6,%%ymm6 \n" // all 8
LABELALIGN
- "1: \n"
+ "1: \n"
"vmovdqu (%0),%%xmm0 \n" // 00112233 (16b, 1u1v)
"vmovdqu 4(%0),%%xmm1 \n" // 11223344 (16b, 1u1v)
@@ -2935,7 +2929,7 @@
"lea 0x20(%1),%1 \n" // 4 uv to 8 uv
"sub $0x8,%2 \n"
"jg 1b \n"
- "vzeroupper \n"
+ "vzeroupper \n"
: "+r"(src_ptr), // %0
"+r"(dst_ptr), // %1
"+r"(dst_width) // %2
diff --git a/source/scale_neon.cc b/source/scale_neon.cc
index ccc7510..0ed3287 100644
--- a/source/scale_neon.cc
+++ b/source/scale_neon.cc
@@ -9,6 +9,7 @@
*/
#include "libyuv/row.h"
+#include "libyuv/scale_row.h"
#ifdef __cplusplus
namespace libyuv {
@@ -29,7 +30,7 @@
int dst_width) {
(void)src_stride;
asm volatile(
- "1: \n"
+ "1: \n"
// load even pixels into q0, odd into q1
"vld2.8 {q0, q1}, [%0]! \n"
"subs %2, %2, #16 \n" // 16 processed per loop
@@ -50,7 +51,7 @@
int dst_width) {
(void)src_stride;
asm volatile(
- "1: \n"
+ "1: \n"
"vld2.8 {q0, q1}, [%0]! \n" // load 32 pixels
"subs %2, %2, #16 \n" // 16 processed per loop
"vrhadd.u8 q0, q0, q1 \n" // rounding half add
@@ -72,7 +73,7 @@
asm volatile(
// change the stride to row 2 pointer
"add %1, %0 \n"
- "1: \n"
+ "1: \n"
"vld1.8 {q0, q1}, [%0]! \n" // load row 1 and post inc
"vld1.8 {q2, q3}, [%1]! \n" // load row 2 and post inc
"subs %3, %3, #16 \n" // 16 processed per loop
@@ -101,7 +102,7 @@
int dst_width) {
(void)src_stride;
asm volatile(
- "1: \n"
+ "1: \n"
"vld4.8 {d0, d1, d2, d3}, [%0]! \n" // src line 0
"subs %2, %2, #8 \n" // 8 processed per loop
"vst1.8 {d2}, [%1]! \n"
@@ -121,7 +122,7 @@
const uint8_t* src_ptr2 = src_ptr + src_stride * 2;
const uint8_t* src_ptr3 = src_ptr + src_stride * 3;
asm volatile(
- "1: \n"
+ "1: \n"
"vld1.8 {q0}, [%0]! \n" // load up 16x4
"vld1.8 {q1}, [%3]! \n"
"vld1.8 {q2}, [%4]! \n"
@@ -155,7 +156,7 @@
int dst_width) {
(void)src_stride;
asm volatile(
- "1: \n"
+ "1: \n"
"vld4.8 {d0, d1, d2, d3}, [%0]! \n" // src line 0
"subs %2, %2, #24 \n"
"vmov d2, d3 \n" // order d0, d1, d2
@@ -175,7 +176,7 @@
asm volatile(
"vmov.u8 d24, #3 \n"
"add %3, %0 \n"
- "1: \n"
+ "1: \n"
"vld4.8 {d0, d1, d2, d3}, [%0]! \n" // src line 0
"vld4.8 {d4, d5, d6, d7}, [%3]! \n" // src line 1
"subs %2, %2, #24 \n"
@@ -232,7 +233,7 @@
asm volatile(
"vmov.u8 d24, #3 \n"
"add %3, %0 \n"
- "1: \n"
+ "1: \n"
"vld4.8 {d0, d1, d2, d3}, [%0]! \n" // src line 0
"vld4.8 {d4, d5, d6, d7}, [%3]! \n" // src line 1
"subs %2, %2, #24 \n"
@@ -283,7 +284,7 @@
(void)src_stride;
asm volatile(
"vld1.8 {q3}, [%3] \n"
- "1: \n"
+ "1: \n"
"vld1.8 {d0, d1, d2, d3}, [%0]! \n"
"subs %2, %2, #12 \n"
"vtbl.u8 d4, {d0, d1, d2, d3}, d6 \n"
@@ -310,7 +311,7 @@
"vld1.8 {q14}, [%6] \n"
"vld1.8 {q15}, [%7] \n"
"add %3, %0 \n"
- "1: \n"
+ "1: \n"
// d0 = 00 40 01 41 02 42 03 43
// d1 = 10 50 11 51 12 52 13 53
@@ -419,7 +420,7 @@
"vld1.16 {q13}, [%4] \n"
"vld1.8 {q14}, [%5] \n"
"add %3, %0 \n"
- "1: \n"
+ "1: \n"
// d0 = 00 40 01 41 02 42 03 43
// d1 = 10 50 11 51 12 52 13 53
@@ -511,7 +512,7 @@
asm volatile(
"vmov.u8 d30, #3 \n"
- "1: \n"
+ "1: \n"
"vld1.8 {d4}, [%0]! \n" // 01234567
"vld1.8 {d5}, [%3]! \n" // 12345678
@@ -549,7 +550,7 @@
"vmov.u16 q15, #3 \n"
"vmov.u8 d28, #3 \n"
- "1: \n"
+ "1: \n"
"vld1.8 {d4}, [%0]! \n" // 01234567
"vld1.8 {d5}, [%5]! \n" // 12345678
@@ -610,7 +611,7 @@
asm volatile(
"vmov.u16 q15, #3 \n"
- "1: \n"
+ "1: \n"
"vld1.16 {q1}, [%0]! \n" // 01234567 (16b)
"vld1.16 {q0}, [%3]! \n" // 12345678 (16b)
@@ -646,7 +647,7 @@
asm volatile(
"vmov.u16 q15, #3 \n"
- "1: \n"
+ "1: \n"
"vld1.16 {q0}, [%0]! \n" // 01234567 (16b)
"vld1.16 {q1}, [%5]! \n" // 12345678 (16b)
@@ -697,7 +698,7 @@
asm volatile(
"vmov.u16 d31, #3 \n"
- "1: \n"
+ "1: \n"
"vld1.16 {q0}, [%0]! \n" // 01234567 (16b)
"vld1.16 {q1}, [%3]! \n" // 12345678 (16b)
@@ -742,7 +743,7 @@
"vmov.u16 d31, #3 \n"
"vmov.u32 q14, #3 \n"
- "1: \n"
+ "1: \n"
"vld1.16 {d0}, [%0]! \n" // 0123 (16b)
"vld1.16 {d1}, [%5]! \n" // 1234 (16b)
"vmovl.u16 q2, d0 \n" // 0123 (32b)
@@ -793,7 +794,7 @@
asm volatile(
"vmov.u8 d30, #3 \n"
- "1: \n"
+ "1: \n"
"vld1.8 {d4}, [%0]! \n" // 00112233 (1u1v)
"vld1.8 {d5}, [%3]! \n" // 11223344 (1u1v)
@@ -831,7 +832,7 @@
"vmov.u16 q15, #3 \n"
"vmov.u8 d28, #3 \n"
- "1: \n"
+ "1: \n"
"vld1.8 {d4}, [%0]! \n" // 00112233 (1u1v)
"vld1.8 {d5}, [%5]! \n" // 11223344 (1u1v)
@@ -892,7 +893,7 @@
asm volatile(
"vmov.u16 d30, #3 \n"
- "1: \n"
+ "1: \n"
"vld1.16 {q0}, [%0]! \n" // 00112233 (1u1v, 16)
"vld1.16 {q1}, [%3]! \n" // 11223344 (1u1v, 16)
@@ -938,7 +939,7 @@
"vmov.u16 d30, #3 \n"
"vmov.u32 q14, #3 \n"
- "1: \n"
+ "1: \n"
"vld1.8 {d0}, [%0]! \n" // 0011 (1u1v)
"vld1.8 {d1}, [%5]! \n" // 1122 (1u1v)
"vmovl.u16 q2, d0 \n" // 0011 (1u1v, 32b)
@@ -988,7 +989,7 @@
uint16_t* dst_ptr,
int src_width) {
asm volatile(
- "1: \n"
+ "1: \n"
"vld1.16 {q1, q2}, [%1] \n" // load accumulator
"vld1.8 {q0}, [%0]! \n" // load 16 bytes
"vaddw.u8 q2, q2, d1 \n" // add
@@ -1035,7 +1036,7 @@
// x + 4 * dx, x + 5 * dx, x + 6 * dx, x + 7 * dx
"vadd.s32 q2, q1, q3 \n"
"vshl.i32 q0, q3, #1 \n" // 8 * dx
- "1: \n"
+ "1: \n"
LOAD2_DATA8_LANE(0)
LOAD2_DATA8_LANE(1)
LOAD2_DATA8_LANE(2)
@@ -1080,98 +1081,13 @@
#undef LOAD2_DATA8_LANE
-// 16x2 -> 16x1
-void ScaleFilterRows_NEON(uint8_t* dst_ptr,
- const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- int dst_width,
- int source_y_fraction) {
- asm volatile(
- "cmp %4, #0 \n"
- "beq 100f \n"
- "add %2, %1 \n"
- "cmp %4, #64 \n"
- "beq 75f \n"
- "cmp %4, #128 \n"
- "beq 50f \n"
- "cmp %4, #192 \n"
- "beq 25f \n"
-
- "vdup.8 d5, %4 \n"
- "rsb %4, #256 \n"
- "vdup.8 d4, %4 \n"
- // General purpose row blend.
- "1: \n"
- "vld1.8 {q0}, [%1]! \n"
- "vld1.8 {q1}, [%2]! \n"
- "subs %3, %3, #16 \n"
- "vmull.u8 q13, d0, d4 \n"
- "vmull.u8 q14, d1, d4 \n"
- "vmlal.u8 q13, d2, d5 \n"
- "vmlal.u8 q14, d3, d5 \n"
- "vrshrn.u16 d0, q13, #8 \n"
- "vrshrn.u16 d1, q14, #8 \n"
- "vst1.8 {q0}, [%0]! \n"
- "bgt 1b \n"
- "b 99f \n"
-
- // Blend 25 / 75.
- "25: \n"
- "vld1.8 {q0}, [%1]! \n"
- "vld1.8 {q1}, [%2]! \n"
- "subs %3, %3, #16 \n"
- "vrhadd.u8 q0, q1 \n"
- "vrhadd.u8 q0, q1 \n"
- "vst1.8 {q0}, [%0]! \n"
- "bgt 25b \n"
- "b 99f \n"
-
- // Blend 50 / 50.
- "50: \n"
- "vld1.8 {q0}, [%1]! \n"
- "vld1.8 {q1}, [%2]! \n"
- "subs %3, %3, #16 \n"
- "vrhadd.u8 q0, q1 \n"
- "vst1.8 {q0}, [%0]! \n"
- "bgt 50b \n"
- "b 99f \n"
-
- // Blend 75 / 25.
- "75: \n"
- "vld1.8 {q1}, [%1]! \n"
- "vld1.8 {q0}, [%2]! \n"
- "subs %3, %3, #16 \n"
- "vrhadd.u8 q0, q1 \n"
- "vrhadd.u8 q0, q1 \n"
- "vst1.8 {q0}, [%0]! \n"
- "bgt 75b \n"
- "b 99f \n"
-
- // Blend 100 / 0 - Copy row unchanged.
- "100: \n"
- "vld1.8 {q0}, [%1]! \n"
- "subs %3, %3, #16 \n"
- "vst1.8 {q0}, [%0]! \n"
- "bgt 100b \n"
-
- "99: \n"
- "vst1.8 {d1[7]}, [%0] \n"
- : "+r"(dst_ptr), // %0
- "+r"(src_ptr), // %1
- "+r"(src_stride), // %2
- "+r"(dst_width), // %3
- "+r"(source_y_fraction) // %4
- :
- : "q0", "q1", "d4", "d5", "q13", "q14", "memory", "cc");
-}
-
void ScaleARGBRowDown2_NEON(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst,
int dst_width) {
(void)src_stride;
asm volatile(
- "1: \n"
+ "1: \n"
"vld4.32 {d0, d2, d4, d6}, [%0]! \n" // load 8 ARGB pixels.
"vld4.32 {d1, d3, d5, d7}, [%0]! \n" // load next 8 ARGB
"subs %2, %2, #8 \n" // 8 processed per loop
@@ -1199,7 +1115,7 @@
int dst_width) {
(void)src_stride;
asm volatile(
- "1: \n"
+ "1: \n"
"vld4.32 {d0, d2, d4, d6}, [%0]! \n" // load 8 ARGB pixels.
"vld4.32 {d1, d3, d5, d7}, [%0]! \n" // load next 8 ARGB
"subs %2, %2, #8 \n" // 8 processed per loop
@@ -1222,7 +1138,7 @@
asm volatile(
// change the stride to row 2 pointer
"add %1, %1, %0 \n"
- "1: \n"
+ "1: \n"
"vld4.8 {d0, d2, d4, d6}, [%0]! \n" // load 8 ARGB pixels.
"vld4.8 {d1, d3, d5, d7}, [%0]! \n" // load next 8 ARGB
"subs %3, %3, #8 \n" // 8 processed per loop.
@@ -1260,7 +1176,7 @@
(void)src_stride;
asm volatile(
"mov r12, %3, lsl #2 \n"
- "1: \n"
+ "1: \n"
"vld1.32 {d0[0]}, [%0], r12 \n"
"vld1.32 {d0[1]}, [%0], r12 \n"
"vld1.32 {d1[0]}, [%0], r12 \n"
@@ -1285,7 +1201,7 @@
asm volatile(
"mov r12, %4, lsl #2 \n"
"add %1, %1, %0 \n"
- "1: \n"
+ "1: \n"
"vld1.8 {d0}, [%0], r12 \n" // 4 2x2 blocks -> 2x1
"vld1.8 {d1}, [%1], r12 \n"
"vld1.8 {d2}, [%0], r12 \n"
@@ -1331,7 +1247,7 @@
int tmp;
const uint8_t* src_tmp = src_argb;
asm volatile(
- "1: \n"
+ "1: \n"
// clang-format off
LOAD1_DATA32_LANE(d0, 0)
LOAD1_DATA32_LANE(d0, 1)
@@ -1384,7 +1300,7 @@
"vmov.i16 q15, #0x7f \n" // 0x7F
// x , x + 1 * dx, x + 2 * dx, x + 3 * dx
"vadd.s32 q8, q1, q0 \n"
- "1: \n"
+ "1: \n"
// d0, d1: a
// d2, d3: b
LOAD2_DATA32_LANE(d0, d2, 0)
@@ -1434,7 +1350,7 @@
int dst_width) {
(void)src_stride;
asm volatile(
- "1: \n"
+ "1: \n"
"vld2.16 {d0, d2}, [%0]! \n" // load 8 UV pixels.
"vld2.16 {d1, d3}, [%0]! \n" // load next 8 UV
"subs %2, %2, #8 \n" // 8 processed per loop.
@@ -1453,7 +1369,7 @@
int dst_width) {
(void)src_stride;
asm volatile(
- "1: \n"
+ "1: \n"
"vld2.16 {d0, d2}, [%0]! \n" // load 8 UV pixels.
"vld2.16 {d1, d3}, [%0]! \n" // load next 8 UV
"subs %2, %2, #8 \n" // 8 processed per loop.
@@ -1474,7 +1390,7 @@
asm volatile(
// change the stride to row 2 pointer
"add %1, %1, %0 \n"
- "1: \n"
+ "1: \n"
"vld2.8 {d0, d2}, [%0]! \n" // load 8 UV pixels.
"vld2.8 {d1, d3}, [%0]! \n" // load next 8 UV
"subs %3, %3, #8 \n" // 8 processed per loop.
@@ -1507,7 +1423,7 @@
const uint8_t* src3_ptr = src_ptr + src_stepx * 6;
(void)src_stride;
asm volatile(
- "1: \n"
+ "1: \n"
"vld1.16 {d0[0]}, [%0], %6 \n"
"vld1.16 {d0[1]}, [%1], %6 \n"
"vld1.16 {d0[2]}, [%2], %6 \n"
diff --git a/source/scale_neon64.cc b/source/scale_neon64.cc
index 7c07238..e7f9f6c 100644
--- a/source/scale_neon64.cc
+++ b/source/scale_neon64.cc
@@ -27,7 +27,7 @@
int dst_width) {
(void)src_stride;
asm volatile(
- "1: \n"
+ "1: \n"
// load even pixels into v0, odd into v1
"ld2 {v0.16b,v1.16b}, [%0], #32 \n"
"subs %w2, %w2, #16 \n" // 16 processed per loop
@@ -38,7 +38,7 @@
"+r"(dst), // %1
"+r"(dst_width) // %2
:
- : "v0", "v1" // Clobber List
+ : "memory", "cc", "v0", "v1" // Clobber List
);
}
@@ -49,7 +49,7 @@
int dst_width) {
(void)src_stride;
asm volatile(
- "1: \n"
+ "1: \n"
// load even pixels into v0, odd into v1
"ld2 {v0.16b,v1.16b}, [%0], #32 \n"
"subs %w2, %w2, #16 \n" // 16 processed per loop
@@ -61,7 +61,7 @@
"+r"(dst), // %1
"+r"(dst_width) // %2
:
- : "v0", "v1" // Clobber List
+ : "memory", "cc", "v0", "v1" // Clobber List
);
}
@@ -73,7 +73,7 @@
asm volatile(
// change the stride to row 2 pointer
"add %1, %1, %0 \n"
- "1: \n"
+ "1: \n"
"ld1 {v0.16b, v1.16b}, [%0], #32 \n" // load row 1 and post inc
"ld1 {v2.16b, v3.16b}, [%1], #32 \n" // load row 2 and post inc
"subs %w3, %w3, #16 \n" // 16 processed per loop
@@ -92,7 +92,7 @@
"+r"(dst), // %2
"+r"(dst_width) // %3
:
- : "v0", "v1", "v2", "v3" // Clobber List
+ : "memory", "cc", "v0", "v1", "v2", "v3" // Clobber List
);
}
@@ -102,17 +102,17 @@
int dst_width) {
(void)src_stride;
asm volatile(
- "1: \n"
- "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // src line 0
- "subs %w2, %w2, #8 \n" // 8 processed per loop
+ "1: \n"
+ "ld4 {v0.16b,v1.16b,v2.16b,v3.16b}, [%0], #64 \n" // src line 0
+ "subs %w2, %w2, #16 \n" // 16 processed per loop
"prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead
- "st1 {v2.8b}, [%1], #8 \n"
+ "st1 {v2.16b}, [%1], #16 \n"
"b.gt 1b \n"
: "+r"(src_ptr), // %0
"+r"(dst_ptr), // %1
"+r"(dst_width) // %2
:
- : "v0", "v1", "v2", "v3", "memory", "cc");
+ : "memory", "cc", "v0", "v1", "v2", "v3");
}
void ScaleRowDown4Box_NEON(const uint8_t* src_ptr,
@@ -123,23 +123,27 @@
const uint8_t* src_ptr2 = src_ptr + src_stride * 2;
const uint8_t* src_ptr3 = src_ptr + src_stride * 3;
asm volatile(
- "1: \n"
- "ld1 {v0.16b}, [%0], #16 \n" // load up 16x4
- "ld1 {v1.16b}, [%2], #16 \n"
- "ld1 {v2.16b}, [%3], #16 \n"
- "ld1 {v3.16b}, [%4], #16 \n"
- "subs %w5, %w5, #4 \n"
+ "1: \n"
+ "ldp q0, q4, [%0], #32 \n" // load up 16x8
+ "ldp q1, q5, [%2], #32 \n"
+ "ldp q2, q6, [%3], #32 \n"
+ "ldp q3, q7, [%4], #32 \n"
+ "subs %w5, %w5, #8 \n"
"uaddlp v0.8h, v0.16b \n"
+ "uaddlp v4.8h, v4.16b \n"
"prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead
"uadalp v0.8h, v1.16b \n"
+ "uadalp v4.8h, v5.16b \n"
"prfm pldl1keep, [%2, 448] \n"
"uadalp v0.8h, v2.16b \n"
+ "uadalp v4.8h, v6.16b \n"
"prfm pldl1keep, [%3, 448] \n"
"uadalp v0.8h, v3.16b \n"
+ "uadalp v4.8h, v7.16b \n"
"prfm pldl1keep, [%4, 448] \n"
- "addp v0.8h, v0.8h, v0.8h \n"
+ "addp v0.8h, v0.8h, v4.8h \n"
"rshrn v0.8b, v0.8h, #4 \n" // divide by 16 w/rounding
- "st1 {v0.s}[0], [%1], #4 \n"
+ "str d0, [%1], #8 \n"
"b.gt 1b \n"
: "+r"(src_ptr), // %0
"+r"(dst_ptr), // %1
@@ -148,30 +152,45 @@
"+r"(src_ptr3), // %4
"+r"(dst_width) // %5
:
- : "v0", "v1", "v2", "v3", "memory", "cc");
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7");
}
-// Down scale from 4 to 3 pixels. Use the neon multilane read/write
-// to load up the every 4th pixel into a 4 different registers.
-// Point samples 32 pixels to 24 pixels.
+static const uvec8 kShuf34_0 = {
+ 0, 1, 3, 4, 5, 7, 8, 9, 11, 12, 13, 15, 16, 17, 19, 20,
+};
+static const uvec8 kShuf34_1 = {
+ 5, 7, 8, 9, 11, 12, 13, 15, 16, 17, 19, 20, 21, 23, 24, 25,
+};
+static const uvec8 kShuf34_2 = {
+ 11, 12, 13, 15, 16, 17, 19, 20, 21, 23, 24, 25, 27, 28, 29, 31,
+};
+
+// Down scale from 4 to 3 pixels. Point samples 64 pixels to 48 pixels.
void ScaleRowDown34_NEON(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int dst_width) {
(void)src_stride;
asm volatile(
- "1: \n"
- "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // src line 0
- "subs %w2, %w2, #24 \n"
- "orr v2.16b, v3.16b, v3.16b \n" // order v0,v1,v2
- "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead
- "st3 {v0.8b,v1.8b,v2.8b}, [%1], #24 \n"
+ "ld1 {v29.16b}, [%[kShuf34_0]] \n"
+ "ld1 {v30.16b}, [%[kShuf34_1]] \n"
+ "ld1 {v31.16b}, [%[kShuf34_2]] \n"
+ "1: \n"
+ "ld1 {v0.16b,v1.16b,v2.16b,v3.16b}, [%[src_ptr]], #64 \n"
+ "subs %w[width], %w[width], #48 \n"
+ "tbl v0.16b, {v0.16b, v1.16b}, v29.16b \n"
+ "prfm pldl1keep, [%[src_ptr], 448] \n"
+ "tbl v1.16b, {v1.16b, v2.16b}, v30.16b \n"
+ "tbl v2.16b, {v2.16b, v3.16b}, v31.16b \n"
+ "st1 {v0.16b,v1.16b,v2.16b}, [%[dst_ptr]], #48 \n"
"b.gt 1b \n"
- : "+r"(src_ptr), // %0
- "+r"(dst_ptr), // %1
- "+r"(dst_width) // %2
- :
- : "v0", "v1", "v2", "v3", "memory", "cc");
+ : [src_ptr] "+r"(src_ptr), // %[src_ptr]
+ [dst_ptr] "+r"(dst_ptr), // %[dst_ptr]
+ [width] "+r"(dst_width) // %[width]
+ : [kShuf34_0] "r"(&kShuf34_0), // %[kShuf34_0]
+ [kShuf34_1] "r"(&kShuf34_1), // %[kShuf34_1]
+ [kShuf34_2] "r"(&kShuf34_2) // %[kShuf34_2]
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v29", "v30", "v31");
}
void ScaleRowDown34_0_Box_NEON(const uint8_t* src_ptr,
@@ -179,12 +198,13 @@
uint8_t* dst_ptr,
int dst_width) {
asm volatile(
- "movi v20.8b, #3 \n"
+ "movi v24.16b, #3 \n"
"add %3, %3, %0 \n"
- "1: \n"
- "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // src line 0
- "ld4 {v4.8b,v5.8b,v6.8b,v7.8b}, [%3], #32 \n" // src line 1
- "subs %w2, %w2, #24 \n"
+
+ "1: \n"
+ "ld4 {v0.16b,v1.16b,v2.16b,v3.16b}, [%0], #64 \n" // src line 0
+ "ld4 {v4.16b,v5.16b,v6.16b,v7.16b}, [%3], #64 \n" // src line 1
+ "subs %w2, %w2, #48 \n"
// filter src line 0 with src line 1
// expand chars to shorts to allow for room
@@ -193,12 +213,20 @@
"ushll v17.8h, v5.8b, #0 \n"
"ushll v18.8h, v6.8b, #0 \n"
"ushll v19.8h, v7.8b, #0 \n"
+ "ushll2 v20.8h, v4.16b, #0 \n"
+ "ushll2 v21.8h, v5.16b, #0 \n"
+ "ushll2 v22.8h, v6.16b, #0 \n"
+ "ushll2 v23.8h, v7.16b, #0 \n"
// 3 * line_0 + line_1
- "umlal v16.8h, v0.8b, v20.8b \n"
- "umlal v17.8h, v1.8b, v20.8b \n"
- "umlal v18.8h, v2.8b, v20.8b \n"
- "umlal v19.8h, v3.8b, v20.8b \n"
+ "umlal v16.8h, v0.8b, v24.8b \n"
+ "umlal v17.8h, v1.8b, v24.8b \n"
+ "umlal v18.8h, v2.8b, v24.8b \n"
+ "umlal v19.8h, v3.8b, v24.8b \n"
+ "umlal2 v20.8h, v0.16b, v24.16b \n"
+ "umlal2 v21.8h, v1.16b, v24.16b \n"
+ "umlal2 v22.8h, v2.16b, v24.16b \n"
+ "umlal2 v23.8h, v3.16b, v24.16b \n"
"prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead
// (3 * line_0 + line_1 + 2) >> 2
@@ -206,22 +234,32 @@
"uqrshrn v1.8b, v17.8h, #2 \n"
"uqrshrn v2.8b, v18.8h, #2 \n"
"uqrshrn v3.8b, v19.8h, #2 \n"
+ "uqrshrn2 v0.16b, v20.8h, #2 \n"
+ "uqrshrn2 v1.16b, v21.8h, #2 \n"
+ "uqrshrn2 v2.16b, v22.8h, #2 \n"
+ "uqrshrn2 v3.16b, v23.8h, #2 \n"
"prfm pldl1keep, [%3, 448] \n"
// a0 = (src[0] * 3 + s[1] * 1 + 2) >> 2
"ushll v16.8h, v1.8b, #0 \n"
- "umlal v16.8h, v0.8b, v20.8b \n"
+ "ushll2 v17.8h, v1.16b, #0 \n"
+ "umlal v16.8h, v0.8b, v24.8b \n"
+ "umlal2 v17.8h, v0.16b, v24.16b \n"
"uqrshrn v0.8b, v16.8h, #2 \n"
+ "uqrshrn2 v0.16b, v17.8h, #2 \n"
// a1 = (src[1] * 1 + s[2] * 1 + 1) >> 1
- "urhadd v1.8b, v1.8b, v2.8b \n"
+ "urhadd v1.16b, v1.16b, v2.16b \n"
// a2 = (src[2] * 1 + s[3] * 3 + 2) >> 2
"ushll v16.8h, v2.8b, #0 \n"
- "umlal v16.8h, v3.8b, v20.8b \n"
+ "ushll2 v17.8h, v2.16b, #0 \n"
+ "umlal v16.8h, v3.8b, v24.8b \n"
+ "umlal2 v17.8h, v3.16b, v24.16b \n"
"uqrshrn v2.8b, v16.8h, #2 \n"
+ "uqrshrn2 v2.16b, v17.8h, #2 \n"
- "st3 {v0.8b,v1.8b,v2.8b}, [%1], #24 \n"
+ "st3 {v0.16b,v1.16b,v2.16b}, [%1], #48 \n"
"b.gt 1b \n"
: "+r"(src_ptr), // %0
@@ -229,8 +267,8 @@
"+r"(dst_width), // %2
"+r"(src_stride) // %3
:
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16", "v17", "v18",
- "v19", "v20", "memory", "cc");
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16",
+ "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24");
}
void ScaleRowDown34_1_Box_NEON(const uint8_t* src_ptr,
@@ -238,53 +276,57 @@
uint8_t* dst_ptr,
int dst_width) {
asm volatile(
- "movi v20.8b, #3 \n"
+ "movi v20.16b, #3 \n"
"add %3, %3, %0 \n"
- "1: \n"
- "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n" // src line 0
- "ld4 {v4.8b,v5.8b,v6.8b,v7.8b}, [%3], #32 \n" // src line 1
- "subs %w2, %w2, #24 \n"
+
+ "1: \n"
+ "ld4 {v0.16b,v1.16b,v2.16b,v3.16b}, [%0], #64 \n" // src line 0
+ "ld4 {v4.16b,v5.16b,v6.16b,v7.16b}, [%3], #64 \n" // src line 1
+ "subs %w2, %w2, #48 \n"
// average src line 0 with src line 1
- "urhadd v0.8b, v0.8b, v4.8b \n"
- "urhadd v1.8b, v1.8b, v5.8b \n"
- "urhadd v2.8b, v2.8b, v6.8b \n"
- "urhadd v3.8b, v3.8b, v7.8b \n"
+ "urhadd v0.16b, v0.16b, v4.16b \n"
+ "urhadd v1.16b, v1.16b, v5.16b \n"
+ "urhadd v2.16b, v2.16b, v6.16b \n"
+ "urhadd v3.16b, v3.16b, v7.16b \n"
"prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead
// a0 = (src[0] * 3 + s[1] * 1 + 2) >> 2
"ushll v4.8h, v1.8b, #0 \n"
+ "ushll2 v5.8h, v1.16b, #0 \n"
"umlal v4.8h, v0.8b, v20.8b \n"
+ "umlal2 v5.8h, v0.16b, v20.16b \n"
"uqrshrn v0.8b, v4.8h, #2 \n"
+ "uqrshrn2 v0.16b, v5.8h, #2 \n"
"prfm pldl1keep, [%3, 448] \n"
// a1 = (src[1] * 1 + s[2] * 1 + 1) >> 1
- "urhadd v1.8b, v1.8b, v2.8b \n"
+ "urhadd v1.16b, v1.16b, v2.16b \n"
// a2 = (src[2] * 1 + s[3] * 3 + 2) >> 2
"ushll v4.8h, v2.8b, #0 \n"
+ "ushll2 v5.8h, v2.16b, #0 \n"
"umlal v4.8h, v3.8b, v20.8b \n"
+ "umlal2 v5.8h, v3.16b, v20.16b \n"
"uqrshrn v2.8b, v4.8h, #2 \n"
+ "uqrshrn2 v2.16b, v5.8h, #2 \n"
- "st3 {v0.8b,v1.8b,v2.8b}, [%1], #24 \n"
+ "st3 {v0.16b,v1.16b,v2.16b}, [%1], #48 \n"
"b.gt 1b \n"
: "+r"(src_ptr), // %0
"+r"(dst_ptr), // %1
"+r"(dst_width), // %2
"+r"(src_stride) // %3
:
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v20", "memory", "cc");
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v20");
}
static const uvec8 kShuf38 = {0, 3, 6, 8, 11, 14, 16, 19,
22, 24, 27, 30, 0, 0, 0, 0};
-static const uvec8 kShuf38_2 = {0, 16, 32, 2, 18, 33, 4, 20,
- 34, 6, 22, 35, 0, 0, 0, 0};
-static const vec16 kMult38_Div6 = {65536 / 12, 65536 / 12, 65536 / 12,
- 65536 / 12, 65536 / 12, 65536 / 12,
- 65536 / 12, 65536 / 12};
-static const vec16 kMult38_Div9 = {65536 / 18, 65536 / 18, 65536 / 18,
- 65536 / 18, 65536 / 18, 65536 / 18,
- 65536 / 18, 65536 / 18};
+static const vec16 kMult38_Div664 = {
+ 65536 / 12, 65536 / 12, 65536 / 8, 65536 / 12, 65536 / 12, 65536 / 8, 0, 0};
+static const vec16 kMult38_Div996 = {65536 / 18, 65536 / 18, 65536 / 12,
+ 65536 / 18, 65536 / 18, 65536 / 12,
+ 0, 0};
// 32 -> 12
void ScaleRowDown38_NEON(const uint8_t* src_ptr,
@@ -293,246 +335,162 @@
int dst_width) {
(void)src_stride;
asm volatile(
- "ld1 {v3.16b}, [%3] \n"
- "1: \n"
- "ld1 {v0.16b,v1.16b}, [%0], #32 \n"
- "subs %w2, %w2, #12 \n"
- "tbl v2.16b, {v0.16b,v1.16b}, v3.16b \n"
- "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead
- "st1 {v2.8b}, [%1], #8 \n"
- "st1 {v2.s}[2], [%1], #4 \n"
+ "ld1 {v3.16b}, [%[kShuf38]] \n"
+ "subs %w[width], %w[width], #12 \n"
+ "b.eq 2f \n"
+
+ "1: \n"
+ "ldp q0, q1, [%[src_ptr]], #32 \n"
+ "subs %w[width], %w[width], #12 \n"
+ "tbl v2.16b, {v0.16b, v1.16b}, v3.16b \n"
+ "prfm pldl1keep, [%[src_ptr], 448] \n" // prefetch 7 lines ahead
+ "str q2, [%[dst_ptr]] \n"
+ "add %[dst_ptr], %[dst_ptr], #12 \n"
"b.gt 1b \n"
- : "+r"(src_ptr), // %0
- "+r"(dst_ptr), // %1
- "+r"(dst_width) // %2
- : "r"(&kShuf38) // %3
- : "v0", "v1", "v2", "v3", "memory", "cc");
+
+ // Store exactly 12 bytes on the final iteration to avoid writing past
+ // the end of the array.
+ "2: \n"
+ "ldp q0, q1, [%[src_ptr]] \n"
+ "tbl v2.16b, {v0.16b, v1.16b}, v3.16b \n"
+ "st1 {v2.8b}, [%[dst_ptr]], #8 \n"
+ "st1 {v2.s}[2], [%[dst_ptr]] \n"
+ : [src_ptr] "+r"(src_ptr), // %[src_ptr]
+ [dst_ptr] "+r"(dst_ptr), // %[dst_ptr]
+ [width] "+r"(dst_width) // %[width]
+ : [kShuf38] "r"(&kShuf38) // %[kShuf38]
+ : "memory", "cc", "v0", "v1", "v2", "v3");
}
-// 32x3 -> 12x1
-void OMITFP ScaleRowDown38_3_Box_NEON(const uint8_t* src_ptr,
- ptrdiff_t src_stride,
- uint8_t* dst_ptr,
- int dst_width) {
- const uint8_t* src_ptr1 = src_ptr + src_stride * 2;
- ptrdiff_t tmp_src_stride = src_stride;
+static const uvec8 kScaleRowDown38_3_BoxIndices1[] = {
+ 0, 1, 6, 7, 12, 13, 16, 17, 22, 23, 28, 29, 255, 255, 255, 255};
+static const uvec8 kScaleRowDown38_3_BoxIndices2[] = {
+ 2, 3, 8, 9, 14, 15, 18, 19, 24, 25, 30, 31, 255, 255, 255, 255};
+static const uvec8 kScaleRowDown38_3_BoxIndices3[] = {
+ 4, 5, 10, 11, 255, 255, 20, 21, 26, 27, 255, 255, 255, 255, 255, 255};
+static const uvec8 kScaleRowDown38_NarrowIndices[] = {
+ 0, 2, 4, 6, 8, 10, 16, 18, 20, 22, 24, 26, 255, 255, 255, 255};
+void ScaleRowDown38_3_Box_NEON(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ const uint8_t* src_ptr1 = src_ptr + src_stride;
+ const uint8_t* src_ptr2 = src_ptr + src_stride * 2;
asm volatile(
- "ld1 {v29.8h}, [%5] \n"
- "ld1 {v30.16b}, [%6] \n"
- "ld1 {v31.8h}, [%7] \n"
- "add %2, %2, %0 \n"
- "1: \n"
+ "ld1 {v27.16b}, [%[tblArray1]] \n"
+ "ld1 {v28.16b}, [%[tblArray2]] \n"
+ "ld1 {v29.16b}, [%[tblArray3]] \n"
+ "ld1 {v31.16b}, [%[tblArray4]] \n"
+ "ld1 {v30.16b}, [%[div996]] \n"
- // 00 40 01 41 02 42 03 43
- // 10 50 11 51 12 52 13 53
- // 20 60 21 61 22 62 23 63
- // 30 70 31 71 32 72 33 73
- "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n"
- "ld4 {v4.8b,v5.8b,v6.8b,v7.8b}, [%2], #32 \n"
- "ld4 {v16.8b,v17.8b,v18.8b,v19.8b}, [%3], #32 \n"
- "subs %w4, %w4, #12 \n"
+ "1: \n"
+ "ldp q20, q0, [%[src_ptr]], #32 \n"
+ "ldp q21, q1, [%[src_ptr1]], #32 \n"
+ "ldp q22, q2, [%[src_ptr2]], #32 \n"
- // Shuffle the input data around to get align the data
- // so adjacent data can be added. 0,1 - 2,3 - 4,5 - 6,7
- // 00 10 01 11 02 12 03 13
- // 40 50 41 51 42 52 43 53
- "trn1 v20.8b, v0.8b, v1.8b \n"
- "trn2 v21.8b, v0.8b, v1.8b \n"
- "trn1 v22.8b, v4.8b, v5.8b \n"
- "trn2 v23.8b, v4.8b, v5.8b \n"
- "trn1 v24.8b, v16.8b, v17.8b \n"
- "trn2 v25.8b, v16.8b, v17.8b \n"
+ "subs %w[width], %w[width], #12 \n"
- // 20 30 21 31 22 32 23 33
- // 60 70 61 71 62 72 63 73
- "trn1 v0.8b, v2.8b, v3.8b \n"
- "trn2 v1.8b, v2.8b, v3.8b \n"
- "trn1 v4.8b, v6.8b, v7.8b \n"
- "trn2 v5.8b, v6.8b, v7.8b \n"
- "trn1 v16.8b, v18.8b, v19.8b \n"
- "trn2 v17.8b, v18.8b, v19.8b \n"
+ // Add across strided rows first.
+ "uaddl v23.8h, v20.8b, v21.8b \n"
+ "uaddl v3.8h, v0.8b, v1.8b \n"
+ "uaddl2 v24.8h, v20.16b, v21.16b \n"
+ "uaddl2 v4.8h, v0.16b, v1.16b \n"
- // 00+10 01+11 02+12 03+13
- // 40+50 41+51 42+52 43+53
- "uaddlp v20.4h, v20.8b \n"
- "uaddlp v21.4h, v21.8b \n"
- "uaddlp v22.4h, v22.8b \n"
- "uaddlp v23.4h, v23.8b \n"
- "uaddlp v24.4h, v24.8b \n"
- "uaddlp v25.4h, v25.8b \n"
+ "uaddw v23.8h, v23.8h, v22.8b \n"
+ "uaddw v3.8h, v3.8h, v2.8b \n"
+ "uaddw2 v24.8h, v24.8h, v22.16b \n" // abcdefgh ...
+ "uaddw2 v4.8h, v4.8h, v2.16b \n"
- // 60+70 61+71 62+72 63+73
- "uaddlp v1.4h, v1.8b \n"
- "uaddlp v5.4h, v5.8b \n"
- "uaddlp v17.4h, v17.8b \n"
+ // Permute groups of {three,three,two} into separate vectors to sum.
+ "tbl v20.16b, {v23.16b, v24.16b}, v27.16b \n" // a d g ...
+ "tbl v0.16b, {v3.16b, v4.16b}, v27.16b \n"
+ "tbl v21.16b, {v23.16b, v24.16b}, v28.16b \n" // b e h ...
+ "tbl v1.16b, {v3.16b, v4.16b}, v28.16b \n"
+ "tbl v22.16b, {v23.16b, v24.16b}, v29.16b \n" // c f 0...
+ "tbl v2.16b, {v3.16b, v4.16b}, v29.16b \n"
- // combine source lines
- "add v20.4h, v20.4h, v22.4h \n"
- "add v21.4h, v21.4h, v23.4h \n"
- "add v20.4h, v20.4h, v24.4h \n"
- "add v21.4h, v21.4h, v25.4h \n"
- "add v2.4h, v1.4h, v5.4h \n"
- "add v2.4h, v2.4h, v17.4h \n"
+ "add v23.8h, v20.8h, v21.8h \n"
+ "add v3.8h, v0.8h, v1.8h \n"
+ "add v24.8h, v23.8h, v22.8h \n" // a+b+c d+e+f g+h
+ "add v4.8h, v3.8h, v2.8h \n"
- // dst_ptr[3] = (s[6 + st * 0] + s[7 + st * 0]
- // + s[6 + st * 1] + s[7 + st * 1]
- // + s[6 + st * 2] + s[7 + st * 2]) / 6
- "sqrdmulh v2.8h, v2.8h, v29.8h \n"
- "xtn v2.8b, v2.8h \n"
-
- // Shuffle 2,3 reg around so that 2 can be added to the
- // 0,1 reg and 3 can be added to the 4,5 reg. This
- // requires expanding from u8 to u16 as the 0,1 and 4,5
- // registers are already expanded. Then do transposes
- // to get aligned.
- // xx 20 xx 30 xx 21 xx 31 xx 22 xx 32 xx 23 xx 33
- "ushll v16.8h, v16.8b, #0 \n"
- "uaddl v0.8h, v0.8b, v4.8b \n"
-
- // combine source lines
- "add v0.8h, v0.8h, v16.8h \n"
-
- // xx 20 xx 21 xx 22 xx 23
- // xx 30 xx 31 xx 32 xx 33
- "trn1 v1.8h, v0.8h, v0.8h \n"
- "trn2 v4.8h, v0.8h, v0.8h \n"
- "xtn v0.4h, v1.4s \n"
- "xtn v4.4h, v4.4s \n"
- "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead
-
- // 0+1+2, 3+4+5
- "add v20.8h, v20.8h, v0.8h \n"
- "add v21.8h, v21.8h, v4.8h \n"
- "prfm pldl1keep, [%2, 448] \n"
-
- // Need to divide, but can't downshift as the the value
- // isn't a power of 2. So multiply by 65536 / n
- // and take the upper 16 bits.
- "sqrdmulh v0.8h, v20.8h, v31.8h \n"
- "sqrdmulh v1.8h, v21.8h, v31.8h \n"
- "prfm pldl1keep, [%3, 448] \n"
-
- // Align for table lookup, vtbl requires registers to be adjacent
- "tbl v3.16b, {v0.16b, v1.16b, v2.16b}, v30.16b \n"
-
- "st1 {v3.8b}, [%1], #8 \n"
- "st1 {v3.s}[2], [%1], #4 \n"
+ "sqrdmulh v24.8h, v24.8h, v30.8h \n" // v /= {9,9,6}
+ "sqrdmulh v25.8h, v4.8h, v30.8h \n"
+ "tbl v21.16b, {v24.16b, v25.16b}, v31.16b \n" // Narrow.
+ "st1 {v21.d}[0], [%[dst_ptr]], #8 \n"
+ "st1 {v21.s}[2], [%[dst_ptr]], #4 \n"
"b.gt 1b \n"
- : "+r"(src_ptr), // %0
- "+r"(dst_ptr), // %1
- "+r"(tmp_src_stride), // %2
- "+r"(src_ptr1), // %3
- "+r"(dst_width) // %4
- : "r"(&kMult38_Div6), // %5
- "r"(&kShuf38_2), // %6
- "r"(&kMult38_Div9) // %7
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16", "v17", "v18",
- "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v29", "v30", "v31",
- "memory", "cc");
+ : [src_ptr] "+r"(src_ptr), // %[src_ptr]
+ [dst_ptr] "+r"(dst_ptr), // %[dst_ptr]
+ [src_ptr1] "+r"(src_ptr1), // %[src_ptr1]
+ [src_ptr2] "+r"(src_ptr2), // %[src_ptr2]
+ [width] "+r"(dst_width) // %[width]
+ : [div996] "r"(&kMult38_Div996), // %[div996]
+ [tblArray1] "r"(kScaleRowDown38_3_BoxIndices1), // %[tblArray1]
+ [tblArray2] "r"(kScaleRowDown38_3_BoxIndices2), // %[tblArray2]
+ [tblArray3] "r"(kScaleRowDown38_3_BoxIndices3), // %[tblArray3]
+ [tblArray4] "r"(kScaleRowDown38_NarrowIndices) // %[tblArray4]
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v20", "v21", "22", "23",
+ "24", "v27", "v28", "v29", "v30", "v31");
}
-// 32x2 -> 12x1
+static const uvec8 kScaleRowDown38_2_BoxIndices1[] = {
+ 0, 1, 3, 4, 6, 7, 8, 9, 11, 12, 14, 15, 255, 255, 255, 255};
+static const uvec8 kScaleRowDown38_2_BoxIndices2[] = {
+ 2, 18, 5, 21, 255, 255, 10, 26, 13, 29, 255, 255, 255, 255, 255, 255};
+
void ScaleRowDown38_2_Box_NEON(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
int dst_width) {
- // TODO(fbarchard): use src_stride directly for clang 3.5+.
- ptrdiff_t tmp_src_stride = src_stride;
+ const uint8_t* src_ptr1 = src_ptr + src_stride;
asm volatile(
- "ld1 {v30.8h}, [%4] \n"
- "ld1 {v31.16b}, [%5] \n"
- "add %2, %2, %0 \n"
- "1: \n"
+ "ld1 {v28.16b}, [%[tblArray1]] \n"
+ "ld1 {v29.16b}, [%[tblArray2]] \n"
+ "ld1 {v31.16b}, [%[tblArray3]] \n"
+ "ld1 {v30.8h}, [%[div664]] \n"
- // 00 40 01 41 02 42 03 43
- // 10 50 11 51 12 52 13 53
- // 20 60 21 61 22 62 23 63
- // 30 70 31 71 32 72 33 73
- "ld4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%0], #32 \n"
- "ld4 {v4.8b,v5.8b,v6.8b,v7.8b}, [%2], #32 \n"
- "subs %w3, %w3, #12 \n"
+ "1: \n"
+ "ldp q20, q0, [%[src_ptr]], #32 \n" // abcdefgh ...
+ "ldp q21, q1, [%[src_ptr1]], #32 \n" // ijklmnop ...
+ "subs %w[width], %w[width], #12 \n"
- // Shuffle the input data around to get align the data
- // so adjacent data can be added. 0,1 - 2,3 - 4,5 - 6,7
- // 00 10 01 11 02 12 03 13
- // 40 50 41 51 42 52 43 53
- "trn1 v16.8b, v0.8b, v1.8b \n"
- "trn2 v17.8b, v0.8b, v1.8b \n"
- "trn1 v18.8b, v4.8b, v5.8b \n"
- "trn2 v19.8b, v4.8b, v5.8b \n"
+ // Permute into groups of six values (three pairs) to be summed.
+ "tbl v22.16b, {v20.16b}, v28.16b \n" // abdegh ...
+ "tbl v2.16b, {v0.16b}, v28.16b \n"
+ "tbl v23.16b, {v21.16b}, v28.16b \n" // ijlmop ...
+ "tbl v3.16b, {v1.16b}, v28.16b \n"
+ "tbl v24.16b, {v20.16b, v21.16b}, v29.16b \n" // ckfn00 ...
+ "tbl v4.16b, {v0.16b, v1.16b}, v29.16b \n"
- // 20 30 21 31 22 32 23 33
- // 60 70 61 71 62 72 63 73
- "trn1 v0.8b, v2.8b, v3.8b \n"
- "trn2 v1.8b, v2.8b, v3.8b \n"
- "trn1 v4.8b, v6.8b, v7.8b \n"
- "trn2 v5.8b, v6.8b, v7.8b \n"
+ "uaddlp v22.8h, v22.16b \n" // a+b d+e g+h ...
+ "uaddlp v2.8h, v2.16b \n"
+ "uaddlp v23.8h, v23.16b \n" // i+j l+m o+p ...
+ "uaddlp v3.8h, v3.16b \n"
+ "uaddlp v24.8h, v24.16b \n" // c+k f+n 0 ...
+ "uaddlp v4.8h, v4.16b \n"
+ "add v20.8h, v22.8h, v23.8h \n"
+ "add v0.8h, v2.8h, v3.8h \n"
+ "add v21.8h, v20.8h, v24.8h \n" // a+b+i+j+c+k ...
+ "add v1.8h, v0.8h, v4.8h \n"
- // 00+10 01+11 02+12 03+13
- // 40+50 41+51 42+52 43+53
- "uaddlp v16.4h, v16.8b \n"
- "uaddlp v17.4h, v17.8b \n"
- "uaddlp v18.4h, v18.8b \n"
- "uaddlp v19.4h, v19.8b \n"
-
- // 60+70 61+71 62+72 63+73
- "uaddlp v1.4h, v1.8b \n"
- "uaddlp v5.4h, v5.8b \n"
-
- // combine source lines
- "add v16.4h, v16.4h, v18.4h \n"
- "add v17.4h, v17.4h, v19.4h \n"
- "add v2.4h, v1.4h, v5.4h \n"
-
- // dst_ptr[3] = (s[6] + s[7] + s[6+st] + s[7+st]) / 4
- "uqrshrn v2.8b, v2.8h, #2 \n"
-
- // Shuffle 2,3 reg around so that 2 can be added to the
- // 0,1 reg and 3 can be added to the 4,5 reg. This
- // requires expanding from u8 to u16 as the 0,1 and 4,5
- // registers are already expanded. Then do transposes
- // to get aligned.
- // xx 20 xx 30 xx 21 xx 31 xx 22 xx 32 xx 23 xx 33
-
- // combine source lines
- "uaddl v0.8h, v0.8b, v4.8b \n"
-
- // xx 20 xx 21 xx 22 xx 23
- // xx 30 xx 31 xx 32 xx 33
- "trn1 v1.8h, v0.8h, v0.8h \n"
- "trn2 v4.8h, v0.8h, v0.8h \n"
- "xtn v0.4h, v1.4s \n"
- "xtn v4.4h, v4.4s \n"
- "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead
-
- // 0+1+2, 3+4+5
- "add v16.8h, v16.8h, v0.8h \n"
- "add v17.8h, v17.8h, v4.8h \n"
- "prfm pldl1keep, [%2, 448] \n"
-
- // Need to divide, but can't downshift as the the value
- // isn't a power of 2. So multiply by 65536 / n
- // and take the upper 16 bits.
- "sqrdmulh v0.8h, v16.8h, v30.8h \n"
- "sqrdmulh v1.8h, v17.8h, v30.8h \n"
-
- // Align for table lookup, vtbl requires registers to
- // be adjacent
-
- "tbl v3.16b, {v0.16b, v1.16b, v2.16b}, v31.16b \n"
-
- "st1 {v3.8b}, [%1], #8 \n"
- "st1 {v3.s}[2], [%1], #4 \n"
+ "sqrdmulh v21.8h, v21.8h, v30.8h \n" // v /= {6,6,4}
+ "sqrdmulh v22.8h, v1.8h, v30.8h \n"
+ "tbl v21.16b, {v21.16b, v22.16b}, v31.16b \n" // Narrow.
+ "st1 {v21.d}[0], [%[dst_ptr]], #8 \n"
+ "st1 {v21.s}[2], [%[dst_ptr]], #4 \n"
"b.gt 1b \n"
- : "+r"(src_ptr), // %0
- "+r"(dst_ptr), // %1
- "+r"(tmp_src_stride), // %2
- "+r"(dst_width) // %3
- : "r"(&kMult38_Div6), // %4
- "r"(&kShuf38_2) // %5
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16", "v17", "v18",
- "v19", "v30", "v31", "memory", "cc");
+ : [src_ptr] "+r"(src_ptr), // %[src_ptr]
+ [dst_ptr] "+r"(dst_ptr), // %[dst_ptr]
+ [src_ptr1] "+r"(src_ptr1), // %[src_ptr1]
+ [width] "+r"(dst_width) // %[width]
+ : [div664] "r"(&kMult38_Div664), // %[div664]
+ [tblArray1] "r"(kScaleRowDown38_2_BoxIndices1), // %[tblArray1]
+ [tblArray2] "r"(kScaleRowDown38_2_BoxIndices2), // %[tblArray2]
+ [tblArray3] "r"(kScaleRowDown38_NarrowIndices) // %[tblArray3]
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v20", "v21", "v22",
+ "v23", "v24", "v28", "v29", "v30", "v31");
}
void ScaleRowUp2_Linear_NEON(const uint8_t* src_ptr,
@@ -540,31 +498,38 @@
int dst_width) {
const uint8_t* src_temp = src_ptr + 1;
asm volatile(
- "movi v31.8b, #3 \n"
+ "movi v31.16b, #3 \n"
- "1: \n"
- "ldr d0, [%0], #8 \n" // 01234567
- "ldr d1, [%1], #8 \n" // 12345678
+ "1: \n"
+ "ldr q0, [%0], #16 \n" // 0123456789abcdef
+ "ldr q1, [%1], #16 \n" // 123456789abcdefg
"prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead
"ushll v2.8h, v0.8b, #0 \n" // 01234567 (16b)
"ushll v3.8h, v1.8b, #0 \n" // 12345678 (16b)
+ "ushll2 v4.8h, v0.16b, #0 \n" // 89abcdef (16b)
+ "ushll2 v5.8h, v1.16b, #0 \n" // 9abcdefg (16b)
"umlal v2.8h, v1.8b, v31.8b \n" // 3*near+far (odd)
"umlal v3.8h, v0.8b, v31.8b \n" // 3*near+far (even)
+ "umlal2 v4.8h, v1.16b, v31.16b \n" // 3*near+far (odd)
+ "umlal2 v5.8h, v0.16b, v31.16b \n" // 3*near+far (even)
"rshrn v2.8b, v2.8h, #2 \n" // 3/4*near+1/4*far (odd)
"rshrn v1.8b, v3.8h, #2 \n" // 3/4*near+1/4*far (even)
+ "rshrn2 v2.16b, v4.8h, #2 \n" // 3/4*near+1/4*far (odd)
+ "rshrn2 v1.16b, v5.8h, #2 \n" // 3/4*near+1/4*far (even)
- "st2 {v1.8b, v2.8b}, [%2], #16 \n" // store
- "subs %w3, %w3, #16 \n" // 8 sample -> 16 sample
+ "st2 {v1.16b, v2.16b}, [%2], #32 \n"
+ "subs %w3, %w3, #32 \n"
"b.gt 1b \n"
: "+r"(src_ptr), // %0
"+r"(src_temp), // %1
"+r"(dst_ptr), // %2
"+r"(dst_width) // %3
:
- : "memory", "cc", "v0", "v1", "v2", "v3", "v31" // Clobber List
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5",
+ "v31" // Clobber List
);
}
@@ -582,7 +547,7 @@
"movi v31.8b, #3 \n"
"movi v30.8h, #3 \n"
- "1: \n"
+ "1: \n"
"ldr d0, [%0], #8 \n" // 01234567
"ldr d1, [%2], #8 \n" // 12345678
"prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead
@@ -637,7 +602,7 @@
asm volatile(
"movi v31.8h, #3 \n"
- "1: \n"
+ "1: \n"
"ld1 {v0.8h}, [%0], #16 \n" // 01234567 (16b)
"ld1 {v1.8h}, [%1], #16 \n" // 12345678 (16b)
"prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead
@@ -674,7 +639,7 @@
asm volatile(
"movi v31.8h, #3 \n"
- "1: \n"
+ "1: \n"
"ld1 {v2.8h}, [%0], #16 \n" // 01234567 (16b)
"ld1 {v3.8h}, [%2], #16 \n" // 12345678 (16b)
"prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead
@@ -728,7 +693,7 @@
asm volatile(
"movi v31.8h, #3 \n"
- "1: \n"
+ "1: \n"
"ld1 {v0.8h}, [%0], #16 \n" // 01234567 (16b)
"ld1 {v1.8h}, [%1], #16 \n" // 12345678 (16b)
"prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead
@@ -774,7 +739,7 @@
"movi v31.4h, #3 \n"
"movi v30.4s, #3 \n"
- "1: \n"
+ "1: \n"
"ldr d0, [%0], #8 \n" // 0123 (16b)
"ldr d1, [%2], #8 \n" // 1234 (16b)
"prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead
@@ -828,7 +793,7 @@
asm volatile(
"movi v31.8b, #3 \n"
- "1: \n"
+ "1: \n"
"ldr d0, [%0], #8 \n" // 00112233 (1u1v)
"ldr d1, [%1], #8 \n" // 11223344 (1u1v)
"prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead
@@ -868,7 +833,7 @@
"movi v31.8b, #3 \n"
"movi v30.8h, #3 \n"
- "1: \n"
+ "1: \n"
"ldr d0, [%0], #8 \n"
"ldr d1, [%2], #8 \n"
"prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead
@@ -923,7 +888,7 @@
asm volatile(
"movi v31.8h, #3 \n"
- "1: \n"
+ "1: \n"
"ld1 {v0.8h}, [%0], #16 \n" // 01234567 (16b)
"ld1 {v1.8h}, [%1], #16 \n" // 12345678 (16b)
"prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead
@@ -971,7 +936,7 @@
"movi v31.4h, #3 \n"
"movi v30.4s, #3 \n"
- "1: \n"
+ "1: \n"
"ldr d0, [%0], #8 \n"
"ldr d1, [%2], #8 \n"
"prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead
@@ -1023,7 +988,7 @@
uint16_t* dst_ptr,
int src_width) {
asm volatile(
- "1: \n"
+ "1: \n"
"ld1 {v1.8h, v2.8h}, [%1] \n" // load accumulator
"ld1 {v0.16b}, [%0], #16 \n" // load 16 bytes
"uaddw2 v2.8h, v2.8h, v0.16b \n" // add
@@ -1040,15 +1005,12 @@
);
}
-// TODO(Yang Zhang): Investigate less load instructions for
-// the x/dx stepping
-#define LOAD2_DATA8_LANE(n) \
- "lsr %5, %3, #16 \n" \
- "add %6, %1, %5 \n" \
- "add %3, %3, %4 \n" \
- "ld2 {v4.b, v5.b}[" #n "], [%6] \n"
+#define SCALE_FILTER_COLS_STEP_ADDR \
+ "lsr %[tmp_offset], %x[x], #16 \n" \
+ "add %[tmp_ptr], %[src_ptr], %[tmp_offset] \n" \
+ "add %x[x], %x[x], %x[dx] \n"
-// The NEON version mimics this formula (from row_common.cc):
+// The Neon version mimics this formula (from scale_common.cc):
// #define BLENDER(a, b, f) (uint8_t)((int)(a) +
// ((((int)((f)) * ((int)(b) - (int)(a))) + 0x8000) >> 16))
@@ -1058,65 +1020,69 @@
int x,
int dx) {
int dx_offset[4] = {0, 1, 2, 3};
- int* tmp = dx_offset;
- const uint8_t* src_tmp = src_ptr;
- int64_t x64 = (int64_t)x; // NOLINT
- int64_t dx64 = (int64_t)dx; // NOLINT
- asm volatile (
- "dup v0.4s, %w3 \n" // x
- "dup v1.4s, %w4 \n" // dx
- "ld1 {v2.4s}, [%5] \n" // 0 1 2 3
+ int64_t tmp_offset;
+ uint8_t* tmp_ptr;
+ asm volatile(
+ "dup v0.4s, %w[x] \n"
+ "dup v1.4s, %w[dx] \n"
+ "ld1 {v2.4s}, [%[dx_offset]] \n" // 0 1 2 3
"shl v3.4s, v1.4s, #2 \n" // 4 * dx
+ "shl v22.4s, v1.4s, #3 \n" // 8 * dx
+
"mul v1.4s, v1.4s, v2.4s \n"
- // x , x + 1 * dx, x + 2 * dx, x + 3 * dx
+ // x , x + 1 * dx, x + 2 * dx, x + 3 * dx
"add v1.4s, v1.4s, v0.4s \n"
- // x + 4 * dx, x + 5 * dx, x + 6 * dx, x + 7 * dx
+ // x + 4 * dx, x + 5 * dx, x + 6 * dx, x + 7 * dx
"add v2.4s, v1.4s, v3.4s \n"
- "shl v0.4s, v3.4s, #1 \n" // 8 * dx
- "1: \n"
- LOAD2_DATA8_LANE(0)
- LOAD2_DATA8_LANE(1)
- LOAD2_DATA8_LANE(2)
- LOAD2_DATA8_LANE(3)
- LOAD2_DATA8_LANE(4)
- LOAD2_DATA8_LANE(5)
- LOAD2_DATA8_LANE(6)
- LOAD2_DATA8_LANE(7)
- "mov v6.16b, v1.16b \n"
- "mov v7.16b, v2.16b \n"
- "uzp1 v6.8h, v6.8h, v7.8h \n"
- "ushll v4.8h, v4.8b, #0 \n"
- "ushll v5.8h, v5.8b, #0 \n"
+
+ "movi v0.8h, #0 \n"
+
+ // truncate to uint16_t
+ "trn1 v22.8h, v22.8h, v0.8h \n"
+ "trn1 v20.8h, v1.8h, v0.8h \n"
+ "trn1 v21.8h, v2.8h, v0.8h \n"
+
+ "1: \n" SCALE_FILTER_COLS_STEP_ADDR
+ "ldr h6, [%[tmp_ptr]] \n" SCALE_FILTER_COLS_STEP_ADDR
+ "ld1 {v6.h}[1], [%[tmp_ptr]] \n" SCALE_FILTER_COLS_STEP_ADDR
+ "ld1 {v6.h}[2], [%[tmp_ptr]] \n" SCALE_FILTER_COLS_STEP_ADDR
+ "ld1 {v6.h}[3], [%[tmp_ptr]] \n" SCALE_FILTER_COLS_STEP_ADDR
+ "ld1 {v6.h}[4], [%[tmp_ptr]] \n" SCALE_FILTER_COLS_STEP_ADDR
+ "ld1 {v6.h}[5], [%[tmp_ptr]] \n" SCALE_FILTER_COLS_STEP_ADDR
+ "ld1 {v6.h}[6], [%[tmp_ptr]] \n" SCALE_FILTER_COLS_STEP_ADDR
+ "ld1 {v6.h}[7], [%[tmp_ptr]] \n"
+
+ "subs %w[width], %w[width], #8 \n" // 8 processed per loop
+ "trn1 v4.16b, v6.16b, v0.16b \n"
+ "trn2 v5.16b, v6.16b, v0.16b \n"
+
"ssubl v16.4s, v5.4h, v4.4h \n"
"ssubl2 v17.4s, v5.8h, v4.8h \n"
- "ushll v7.4s, v6.4h, #0 \n"
- "ushll2 v6.4s, v6.8h, #0 \n"
- "mul v16.4s, v16.4s, v7.4s \n"
- "mul v17.4s, v17.4s, v6.4s \n"
+ "mul v16.4s, v16.4s, v20.4s \n"
+ "mul v17.4s, v17.4s, v21.4s \n"
"rshrn v6.4h, v16.4s, #16 \n"
"rshrn2 v6.8h, v17.4s, #16 \n"
"add v4.8h, v4.8h, v6.8h \n"
"xtn v4.8b, v4.8h \n"
- "st1 {v4.8b}, [%0], #8 \n" // store pixels
- "add v1.4s, v1.4s, v0.4s \n"
- "add v2.4s, v2.4s, v0.4s \n"
- "subs %w2, %w2, #8 \n" // 8 processed per loop
+ "add v20.8h, v20.8h, v22.8h \n"
+ "add v21.8h, v21.8h, v22.8h \n"
+
+ "st1 {v4.8b}, [%[dst_ptr]], #8 \n" // store pixels
"b.gt 1b \n"
- : "+r"(dst_ptr), // %0
- "+r"(src_ptr), // %1
- "+r"(dst_width), // %2
- "+r"(x64), // %3
- "+r"(dx64), // %4
- "+r"(tmp), // %5
- "+r"(src_tmp) // %6
- :
- : "memory", "cc", "v0", "v1", "v2", "v3",
- "v4", "v5", "v6", "v7", "v16", "v17"
- );
+ : [src_ptr] "+r"(src_ptr), // %[src_ptr]
+ [dst_ptr] "+r"(dst_ptr), // %[dst_ptr]
+ [width] "+r"(dst_width), // %[width]
+ [x] "+r"(x), // %[x]
+ [dx] "+r"(dx), // %[dx]
+ [tmp_offset] "=&r"(tmp_offset), // %[tmp_offset]
+ [tmp_ptr] "=&r"(tmp_ptr) // %[tmp_ptr]
+ : [dx_offset] "r"(dx_offset) // %[dx_offset]
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v16", "v17",
+ "v20", "v21", "v22");
}
-#undef LOAD2_DATA8_LANE
+#undef SCALE_FILTER_COLS_STEP_ADDR
void ScaleARGBRowDown2_NEON(const uint8_t* src_ptr,
ptrdiff_t src_stride,
@@ -1124,20 +1090,19 @@
int dst_width) {
(void)src_stride;
asm volatile(
- "1: \n"
- // load 16 ARGB pixels with even pixels into q0/q2, odd into q1/q3
- "ld4 {v0.4s,v1.4s,v2.4s,v3.4s}, [%0], #64 \n"
- "subs %w2, %w2, #8 \n" // 8 processed per loop
- "mov v2.16b, v3.16b \n"
- "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead
- "st2 {v1.4s,v2.4s}, [%1], #32 \n" // store 8 odd pixels
+ "1: \n"
+ "ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%[src]], #64 \n"
+ "subs %w[width], %w[width], #8 \n"
+ "prfm pldl1keep, [%[src], 448] \n"
+ "uzp2 v0.4s, v0.4s, v1.4s \n"
+ "uzp2 v1.4s, v2.4s, v3.4s \n"
+ "st1 {v0.4s, v1.4s}, [%[dst]], #32 \n"
"b.gt 1b \n"
- : "+r"(src_ptr), // %0
- "+r"(dst), // %1
- "+r"(dst_width) // %2
+ : [src] "+r"(src_ptr), // %[src]
+ [dst] "+r"(dst), // %[dst]
+ [width] "+r"(dst_width) // %[width]
:
- : "memory", "cc", "v0", "v1", "v2", "v3" // Clobber List
- );
+ : "memory", "cc", "v0", "v1", "v2", "v3");
}
void ScaleARGBRowDown2Linear_NEON(const uint8_t* src_argb,
@@ -1145,83 +1110,79 @@
uint8_t* dst_argb,
int dst_width) {
(void)src_stride;
+ const uint8_t* src_argb1 = src_argb + 32;
asm volatile(
- "1: \n"
- // load 16 ARGB pixels with even pixels into q0/q2, odd into q1/q3
- "ld4 {v0.4s,v1.4s,v2.4s,v3.4s}, [%0], #64 \n"
- "subs %w2, %w2, #8 \n" // 8 processed per loop
-
- "urhadd v0.16b, v0.16b, v1.16b \n" // rounding half add
- "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead
+ "1: \n"
+ "ld2 {v0.4s, v1.4s}, [%[src]] \n"
+ "add %[src], %[src], #64 \n"
+ "ld2 {v2.4s, v3.4s}, [%[src1]] \n"
+ "add %[src1], %[src1], #64 \n"
+ "urhadd v0.16b, v0.16b, v1.16b \n"
"urhadd v1.16b, v2.16b, v3.16b \n"
- "st2 {v0.4s,v1.4s}, [%1], #32 \n" // store 8 pixels
+ "subs %w[width], %w[width], #8 \n"
+ "st1 {v0.16b, v1.16b}, [%[dst]], #32 \n"
"b.gt 1b \n"
- : "+r"(src_argb), // %0
- "+r"(dst_argb), // %1
- "+r"(dst_width) // %2
+ : [src] "+r"(src_argb), // %[src]
+ [src1] "+r"(src_argb1), // %[src1]
+ [dst] "+r"(dst_argb), // %[dst]
+ [width] "+r"(dst_width) // %[width]
:
- : "memory", "cc", "v0", "v1", "v2", "v3" // Clobber List
- );
+ : "memory", "cc", "v0", "v1", "v2", "v3");
}
void ScaleARGBRowDown2Box_NEON(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst,
int dst_width) {
+ const uint8_t* src_ptr1 = src_ptr + src_stride;
asm volatile(
- // change the stride to row 2 pointer
- "add %1, %1, %0 \n"
- "1: \n"
- "ld4 {v0.16b,v1.16b,v2.16b,v3.16b}, [%0], #64 \n" // load 16 ARGB
- "subs %w3, %w3, #8 \n" // 8 processed per loop.
- "uaddlp v0.8h, v0.16b \n" // B 16 bytes -> 8 shorts.
- "uaddlp v1.8h, v1.16b \n" // G 16 bytes -> 8 shorts.
- "uaddlp v2.8h, v2.16b \n" // R 16 bytes -> 8 shorts.
- "uaddlp v3.8h, v3.16b \n" // A 16 bytes -> 8 shorts.
- "ld4 {v16.16b,v17.16b,v18.16b,v19.16b}, [%1], #64 \n" // load 8
- "uadalp v0.8h, v16.16b \n" // B 16 bytes -> 8 shorts.
- "uadalp v1.8h, v17.16b \n" // G 16 bytes -> 8 shorts.
- "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead
- "uadalp v2.8h, v18.16b \n" // R 16 bytes -> 8 shorts.
- "uadalp v3.8h, v19.16b \n" // A 16 bytes -> 8 shorts.
- "prfm pldl1keep, [%1, 448] \n"
- "rshrn v0.8b, v0.8h, #2 \n" // round and pack
+ "1: \n"
+ "ld2 {v0.4s, v1.4s}, [%[src]], #32 \n"
+ "ld2 {v20.4s, v21.4s}, [%[src1]], #32 \n"
+ "uaddl v2.8h, v0.8b, v1.8b \n"
+ "uaddl2 v3.8h, v0.16b, v1.16b \n"
+ "uaddl v22.8h, v20.8b, v21.8b \n"
+ "uaddl2 v23.8h, v20.16b, v21.16b \n"
+ "add v0.8h, v2.8h, v22.8h \n"
+ "add v1.8h, v3.8h, v23.8h \n"
+ "rshrn v0.8b, v0.8h, #2 \n"
"rshrn v1.8b, v1.8h, #2 \n"
- "rshrn v2.8b, v2.8h, #2 \n"
- "rshrn v3.8b, v3.8h, #2 \n"
- "st4 {v0.8b,v1.8b,v2.8b,v3.8b}, [%2], #32 \n"
+ "subs %w[width], %w[width], #4 \n"
+ "stp d0, d1, [%[dst]], #16 \n"
"b.gt 1b \n"
- : "+r"(src_ptr), // %0
- "+r"(src_stride), // %1
- "+r"(dst), // %2
- "+r"(dst_width) // %3
+ : [src] "+r"(src_ptr), [src1] "+r"(src_ptr1), [dst] "+r"(dst),
+ [width] "+r"(dst_width)
:
- : "memory", "cc", "v0", "v1", "v2", "v3", "v16", "v17", "v18", "v19");
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v20", "v21", "v22", "v23");
}
-// Reads 4 pixels at a time.
-// Alignment requirement: src_argb 4 byte aligned.
void ScaleARGBRowDownEven_NEON(const uint8_t* src_argb,
ptrdiff_t src_stride,
int src_stepx,
uint8_t* dst_argb,
int dst_width) {
+ const uint8_t* src_argb1 = src_argb + src_stepx * 4;
+ const uint8_t* src_argb2 = src_argb + src_stepx * 8;
+ const uint8_t* src_argb3 = src_argb + src_stepx * 12;
+ int64_t i = 0;
(void)src_stride;
asm volatile(
- "1: \n"
- "ld1 {v0.s}[0], [%0], %3 \n"
- "ld1 {v0.s}[1], [%0], %3 \n"
- "ld1 {v0.s}[2], [%0], %3 \n"
- "ld1 {v0.s}[3], [%0], %3 \n"
- "subs %w2, %w2, #4 \n" // 4 pixels per loop.
- "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead
- "st1 {v0.16b}, [%1], #16 \n"
+ "1: \n"
+ "ldr w10, [%[src], %[i]] \n"
+ "ldr w11, [%[src1], %[i]] \n"
+ "ldr w12, [%[src2], %[i]] \n"
+ "ldr w13, [%[src3], %[i]] \n"
+ "add %[i], %[i], %[step] \n"
+ "subs %w[width], %w[width], #4 \n"
+ "prfm pldl1keep, [%[src], 448] \n"
+ "stp w10, w11, [%[dst]], #8 \n"
+ "stp w12, w13, [%[dst]], #8 \n"
"b.gt 1b \n"
- : "+r"(src_argb), // %0
- "+r"(dst_argb), // %1
- "+r"(dst_width) // %2
- : "r"((int64_t)(src_stepx * 4)) // %3
- : "memory", "cc", "v0");
+ : [src] "+r"(src_argb), [src1] "+r"(src_argb1), [src2] "+r"(src_argb2),
+ [src3] "+r"(src_argb3), [dst] "+r"(dst_argb), [width] "+r"(dst_width),
+ [i] "+r"(i)
+ : [step] "r"((int64_t)(src_stepx * 16))
+ : "memory", "cc", "w10", "w11", "w12", "w13");
}
// Reads 4 pixels at a time.
@@ -1235,7 +1196,7 @@
int dst_width) {
asm volatile(
"add %1, %1, %0 \n"
- "1: \n"
+ "1: \n"
"ld1 {v0.8b}, [%0], %4 \n" // Read 4 2x2 -> 2x1
"ld1 {v1.8b}, [%1], %4 \n"
"ld1 {v2.8b}, [%0], %4 \n"
@@ -1249,26 +1210,24 @@
"uaddl v4.8h, v4.8b, v5.8b \n"
"uaddl v6.8h, v6.8b, v7.8b \n"
"prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead
- "mov v16.d[1], v0.d[1] \n" // ab_cd -> ac_bd
- "mov v0.d[1], v2.d[0] \n"
- "mov v2.d[0], v16.d[1] \n"
- "mov v16.d[1], v4.d[1] \n" // ef_gh -> eg_fh
- "mov v4.d[1], v6.d[0] \n"
- "mov v6.d[0], v16.d[1] \n"
+ "zip1 v1.2d, v0.2d, v2.2d \n"
+ "zip2 v2.2d, v0.2d, v2.2d \n"
+ "zip1 v5.2d, v4.2d, v6.2d \n"
+ "zip2 v6.2d, v4.2d, v6.2d \n"
"prfm pldl1keep, [%1, 448] \n"
- "add v0.8h, v0.8h, v2.8h \n" // (a+b)_(c+d)
- "add v4.8h, v4.8h, v6.8h \n" // (e+f)_(g+h)
+ "add v0.8h, v1.8h, v2.8h \n" // (a+b)_(c+d)
+ "add v4.8h, v5.8h, v6.8h \n" // (e+f)_(g+h)
"rshrn v0.8b, v0.8h, #2 \n" // first 2 pixels.
- "rshrn2 v0.16b, v4.8h, #2 \n" // next 2 pixels.
+ "rshrn v1.8b, v4.8h, #2 \n" // next 2 pixels.
"subs %w3, %w3, #4 \n" // 4 pixels per loop.
- "st1 {v0.16b}, [%2], #16 \n"
+ "stp d0, d1, [%2], #16 \n"
"b.gt 1b \n"
: "+r"(src_argb), // %0
"+r"(src_stride), // %1
"+r"(dst_argb), // %2
"+r"(dst_width) // %3
: "r"((int64_t)(src_stepx * 4)) // %4
- : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16");
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7");
}
// TODO(Yang Zhang): Investigate less load instructions for
@@ -1288,8 +1247,8 @@
int64_t x64 = (int64_t)x; // NOLINT
int64_t dx64 = (int64_t)dx; // NOLINT
int64_t tmp64;
- asm volatile(
- "1: \n"
+ asm volatile (
+ "1: \n"
// clang-format off
LOAD1_DATA32_LANE(v0, 0)
LOAD1_DATA32_LANE(v0, 1)
@@ -1317,13 +1276,14 @@
#undef LOAD1_DATA32_LANE
-// TODO(Yang Zhang): Investigate less load instructions for
-// the x/dx stepping
-#define LOAD2_DATA32_LANE(vn1, vn2, n) \
- "lsr %5, %3, #16 \n" \
- "add %6, %1, %5, lsl #2 \n" \
- "add %3, %3, %4 \n" \
- "ld2 {" #vn1 ".s, " #vn2 ".s}[" #n "], [%6] \n"
+static const uvec8 kScaleARGBFilterColsShuffleIndices = {
+ 0, 0, 0, 0, 2, 2, 2, 2, 4, 4, 4, 4, 6, 6, 6, 6,
+};
+
+#define SCALE_ARGB_FILTER_COLS_STEP_ADDR \
+ "lsr %5, %3, #16 \n" \
+ "add %6, %1, %5, lsl #2 \n" \
+ "add %3, %3, %4 \n"
void ScaleARGBFilterCols_NEON(uint8_t* dst_argb,
const uint8_t* src_argb,
@@ -1331,64 +1291,130 @@
int x,
int dx) {
int dx_offset[4] = {0, 1, 2, 3};
- int* tmp = dx_offset;
+ int64_t tmp;
const uint8_t* src_tmp = src_argb;
- int64_t x64 = (int64_t)x; // NOLINT
- int64_t dx64 = (int64_t)dx; // NOLINT
- asm volatile (
- "dup v0.4s, %w3 \n" // x
- "dup v1.4s, %w4 \n" // dx
- "ld1 {v2.4s}, [%5] \n" // 0 1 2 3
- "shl v6.4s, v1.4s, #2 \n" // 4 * dx
+ int64_t x64 = (int64_t)x;
+ int64_t dx64 = (int64_t)dx;
+ asm volatile(
+ "dup v0.4s, %w3 \n"
+ "dup v1.4s, %w4 \n"
+ "ld1 {v2.4s}, [%[kOffsets]] \n"
+ "shl v6.4s, v1.4s, #2 \n"
"mul v1.4s, v1.4s, v2.4s \n"
- "movi v3.16b, #0x7f \n" // 0x7F
- "movi v4.8h, #0x7f \n" // 0x7F
- // x , x + 1 * dx, x + 2 * dx, x + 3 * dx
+ "movi v3.16b, #0x7f \n"
+
"add v5.4s, v1.4s, v0.4s \n"
- "1: \n"
- // d0, d1: a
- // d2, d3: b
- LOAD2_DATA32_LANE(v0, v1, 0)
- LOAD2_DATA32_LANE(v0, v1, 1)
- LOAD2_DATA32_LANE(v0, v1, 2)
- LOAD2_DATA32_LANE(v0, v1, 3)
- "shrn v2.4h, v5.4s, #9 \n"
- "and v2.8b, v2.8b, v4.8b \n"
- "dup v16.8b, v2.b[0] \n"
- "dup v17.8b, v2.b[2] \n"
- "dup v18.8b, v2.b[4] \n"
- "dup v19.8b, v2.b[6] \n"
- "ext v2.8b, v16.8b, v17.8b, #4 \n"
- "ext v17.8b, v18.8b, v19.8b, #4 \n"
- "ins v2.d[1], v17.d[0] \n" // f
- "eor v7.16b, v2.16b, v3.16b \n" // 0x7f ^ f
- "umull v16.8h, v0.8b, v7.8b \n"
- "umull2 v17.8h, v0.16b, v7.16b \n"
- "umull v18.8h, v1.8b, v2.8b \n"
- "umull2 v19.8h, v1.16b, v2.16b \n"
- "prfm pldl1keep, [%1, 448] \n" // prefetch 7 lines ahead
- "add v16.8h, v16.8h, v18.8h \n"
- "add v17.8h, v17.8h, v19.8h \n"
- "shrn v0.8b, v16.8h, #7 \n"
- "shrn2 v0.16b, v17.8h, #7 \n"
- "st1 {v0.4s}, [%0], #16 \n" // store pixels
- "add v5.4s, v5.4s, v6.4s \n"
- "subs %w2, %w2, #4 \n" // 4 processed per loop
- "b.gt 1b \n"
- : "+r"(dst_argb), // %0
- "+r"(src_argb), // %1
- "+r"(dst_width), // %2
- "+r"(x64), // %3
- "+r"(dx64), // %4
- "+r"(tmp), // %5
- "+r"(src_tmp) // %6
- :
- : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5",
- "v6", "v7", "v16", "v17", "v18", "v19"
- );
+ "ldr q18, [%[kIndices]] \n"
+
+ "1: \n" //
+ SCALE_ARGB_FILTER_COLS_STEP_ADDR
+ "ldr d1, [%6] \n" //
+ SCALE_ARGB_FILTER_COLS_STEP_ADDR
+ "ldr d2, [%6] \n"
+ "shrn v4.4h, v5.4s, #9 \n" //
+ SCALE_ARGB_FILTER_COLS_STEP_ADDR
+ "ld1 {v1.d}[1], [%6] \n" //
+ SCALE_ARGB_FILTER_COLS_STEP_ADDR
+ "ld1 {v2.d}[1], [%6] \n"
+
+ "subs %w2, %w2, #4 \n" // 4 processed per loop
+ "and v4.8b, v4.8b, v3.8b \n"
+ "trn1 v0.4s, v1.4s, v2.4s \n"
+ "tbl v4.16b, {v4.16b}, v18.16b \n" // f
+ "trn2 v1.4s, v1.4s, v2.4s \n"
+ "eor v7.16b, v4.16b, v3.16b \n" // 0x7f ^ f
+
+ "umull v16.8h, v1.8b, v4.8b \n"
+ "umull2 v17.8h, v1.16b, v4.16b \n"
+ "umlal v16.8h, v0.8b, v7.8b \n"
+ "umlal2 v17.8h, v0.16b, v7.16b \n"
+
+ "prfm pldl1keep, [%1, 448] \n" // prefetch 7 lines ahead
+ "shrn v0.8b, v16.8h, #7 \n"
+ "shrn v1.8b, v17.8h, #7 \n"
+ "add v5.4s, v5.4s, v6.4s \n"
+ "stp d0, d1, [%0], #16 \n" // store pixels
+ "b.gt 1b \n"
+ : "+r"(dst_argb), // %0
+ "+r"(src_argb), // %1
+ "+r"(dst_width), // %2
+ "+r"(x64), // %3
+ "+r"(dx64), // %4
+ "=&r"(tmp), // %5
+ "+r"(src_tmp) // %6
+ : [kIndices] "r"(&kScaleARGBFilterColsShuffleIndices), // %[kIndices]
+ [kOffsets] "r"(dx_offset) // %[kOffsets]
+ : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16",
+ "v17", "v18", "v19");
}
-#undef LOAD2_DATA32_LANE
+#undef SCALE_ARGB_FILTER_COLS_STEP_ADDR
+
+void ScaleRowDown2_16_NEON(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst,
+ int dst_width) {
+ (void)src_stride;
+ asm volatile(
+ "subs %w[dst_width], %w[dst_width], #32 \n"
+ "b.lt 2f \n"
+
+ "1: \n"
+ "ldp q0, q1, [%[src_ptr]] \n"
+ "ldp q2, q3, [%[src_ptr], #32] \n"
+ "ldp q4, q5, [%[src_ptr], #64] \n"
+ "ldp q6, q7, [%[src_ptr], #96] \n"
+ "add %[src_ptr], %[src_ptr], #128 \n"
+ "uzp2 v0.8h, v0.8h, v1.8h \n"
+ "uzp2 v1.8h, v2.8h, v3.8h \n"
+ "uzp2 v2.8h, v4.8h, v5.8h \n"
+ "uzp2 v3.8h, v6.8h, v7.8h \n"
+ "subs %w[dst_width], %w[dst_width], #32 \n" // 32 elems per
+ // iteration.
+ "stp q0, q1, [%[dst_ptr]] \n"
+ "stp q2, q3, [%[dst_ptr], #32] \n"
+ "add %[dst_ptr], %[dst_ptr], #64 \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[dst_width], %w[dst_width], #32 \n"
+ "b.eq 99f \n"
+
+ "ldp q0, q1, [%[src_ptr]] \n"
+ "ldp q2, q3, [%[src_ptr], #32] \n"
+ "uzp2 v0.8h, v0.8h, v1.8h \n"
+ "uzp2 v1.8h, v2.8h, v3.8h \n"
+ "stp q0, q1, [%[dst_ptr]] \n"
+
+ "99: \n"
+ : [src_ptr] "+r"(src_ptr), // %[src_ptr]
+ [dst_ptr] "+r"(dst), // %[dst_ptr]
+ [dst_width] "+r"(dst_width) // %[dst_width]
+ :
+ : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7");
+}
+
+void ScaleRowDown2Linear_16_NEON(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst,
+ int dst_width) {
+ (void)src_stride;
+ asm volatile(
+ "1: \n"
+ "ld2 {v0.8h, v1.8h}, [%[src_ptr]], #32 \n"
+ "ld2 {v2.8h, v3.8h}, [%[src_ptr]], #32 \n"
+ "subs %w[dst_width], %w[dst_width], #16 \n"
+ "urhadd v0.8h, v0.8h, v1.8h \n"
+ "urhadd v1.8h, v2.8h, v3.8h \n"
+ "prfm pldl1keep, [%[src_ptr], 448] \n"
+ "stp q0, q1, [%[dst_ptr]], #32 \n"
+ "b.gt 1b \n"
+ : [src_ptr] "+r"(src_ptr), // %[src_ptr]
+ [dst_ptr] "+r"(dst), // %[dst_ptr]
+ [dst_width] "+r"(dst_width) // %[dst_width]
+ :
+ : "memory", "cc", "v0", "v1", "v2", "v3");
+}
// Read 16x2 average down and write 8x1.
void ScaleRowDown2Box_16_NEON(const uint16_t* src_ptr,
@@ -1398,7 +1424,7 @@
asm volatile(
// change the stride to row 2 pointer
"add %1, %0, %1, lsl #1 \n" // ptr + stide * 2
- "1: \n"
+ "1: \n"
"ld1 {v0.8h, v1.8h}, [%0], #32 \n" // load row 1 and post inc
"ld1 {v2.8h, v3.8h}, [%1], #32 \n" // load row 2 and post inc
"subs %w3, %w3, #8 \n" // 8 processed per loop
@@ -1417,59 +1443,7 @@
"+r"(dst), // %2
"+r"(dst_width) // %3
:
- : "v0", "v1", "v2", "v3" // Clobber List
- );
-}
-
-// Read 8x2 upsample with filtering and write 16x1.
-// Actually reads an extra pixel, so 9x2.
-void ScaleRowUp2_16_NEON(const uint16_t* src_ptr,
- ptrdiff_t src_stride,
- uint16_t* dst,
- int dst_width) {
- asm volatile(
- "add %1, %0, %1, lsl #1 \n" // ptr + stide * 2
- "movi v0.8h, #9 \n" // constants
- "movi v1.4s, #3 \n"
-
- "1: \n"
- "ld1 {v3.8h}, [%0], %4 \n" // TL read first 8
- "ld1 {v4.8h}, [%0], %5 \n" // TR read 8 offset by 1
- "ld1 {v5.8h}, [%1], %4 \n" // BL read 8 from next row
- "ld1 {v6.8h}, [%1], %5 \n" // BR offset by 1
- "subs %w3, %w3, #16 \n" // 16 dst pixels per loop
- "umull v16.4s, v3.4h, v0.4h \n"
- "umull2 v7.4s, v3.8h, v0.8h \n"
- "umull v18.4s, v4.4h, v0.4h \n"
- "umull2 v17.4s, v4.8h, v0.8h \n"
- "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead
- "uaddw v16.4s, v16.4s, v6.4h \n"
- "uaddl2 v19.4s, v6.8h, v3.8h \n"
- "uaddl v3.4s, v6.4h, v3.4h \n"
- "uaddw2 v6.4s, v7.4s, v6.8h \n"
- "uaddl2 v7.4s, v5.8h, v4.8h \n"
- "uaddl v4.4s, v5.4h, v4.4h \n"
- "uaddw v18.4s, v18.4s, v5.4h \n"
- "prfm pldl1keep, [%1, 448] \n"
- "mla v16.4s, v4.4s, v1.4s \n"
- "mla v18.4s, v3.4s, v1.4s \n"
- "mla v6.4s, v7.4s, v1.4s \n"
- "uaddw2 v4.4s, v17.4s, v5.8h \n"
- "uqrshrn v16.4h, v16.4s, #4 \n"
- "mla v4.4s, v19.4s, v1.4s \n"
- "uqrshrn2 v16.8h, v6.4s, #4 \n"
- "uqrshrn v17.4h, v18.4s, #4 \n"
- "uqrshrn2 v17.8h, v4.4s, #4 \n"
- "st2 {v16.8h-v17.8h}, [%2], #32 \n"
- "b.gt 1b \n"
- : "+r"(src_ptr), // %0
- "+r"(src_stride), // %1
- "+r"(dst), // %2
- "+r"(dst_width) // %3
- : "r"(2LL), // %4
- "r"(14LL) // %5
- : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16", "v17", "v18",
- "v19" // Clobber List
+ : "memory", "cc", "v0", "v1", "v2", "v3" // Clobber List
);
}
@@ -1479,7 +1453,7 @@
int dst_width) {
(void)src_stride;
asm volatile(
- "1: \n"
+ "1: \n"
"ld2 {v0.8h,v1.8h}, [%0], #32 \n" // load 16 UV
"subs %w2, %w2, #8 \n" // 8 processed per loop.
"prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead
@@ -1498,7 +1472,7 @@
int dst_width) {
(void)src_stride;
asm volatile(
- "1: \n"
+ "1: \n"
"ld2 {v0.8h,v1.8h}, [%0], #32 \n" // load 16 UV
"subs %w2, %w2, #8 \n" // 8 processed per loop.
"urhadd v0.16b, v0.16b, v1.16b \n" // rounding half add
@@ -1519,7 +1493,7 @@
asm volatile(
// change the stride to row 2 pointer
"add %1, %1, %0 \n"
- "1: \n"
+ "1: \n"
"ld2 {v0.16b,v1.16b}, [%0], #32 \n" // load 16 UV
"subs %w3, %w3, #8 \n" // 8 processed per loop.
"uaddlp v0.8h, v0.16b \n" // U 16 bytes -> 8 shorts.
@@ -1552,7 +1526,7 @@
const uint8_t* src3_ptr = src_ptr + src_stepx * 6;
(void)src_stride;
asm volatile(
- "1: \n"
+ "1: \n"
"ld1 {v0.h}[0], [%0], %6 \n"
"ld1 {v1.h}[0], [%1], %6 \n"
"ld1 {v2.h}[0], [%2], %6 \n"
diff --git a/source/scale_rgb.cc b/source/scale_rgb.cc
index 8db59b5..5e69fe3 100644
--- a/source/scale_rgb.cc
+++ b/source/scale_rgb.cc
@@ -11,6 +11,9 @@
#include "libyuv/scale.h" /* For FilterMode */
#include <assert.h>
+#include <limits.h>
+#include <stdint.h>
+#include <stdlib.h>
#include <string.h>
#include "libyuv/convert_argb.h"
@@ -38,18 +41,31 @@
int dst_height,
enum FilterMode filtering) {
int r;
- uint8_t* src_argb =
- (uint8_t*)malloc(src_width * src_height * 4 + dst_width * dst_height * 4);
- uint8_t* dst_argb = src_argb + src_width * src_height * 4;
-
- if (!src_argb) {
- return 1;
+ if (!src_rgb || !dst_rgb || src_width <= 0 || src_width > INT_MAX / 4 ||
+ src_height == 0 || dst_width <= 0 || dst_width > INT_MAX / 4 ||
+ dst_height <= 0) {
+ return -1;
}
+ const int abs_src_height = (src_height < 0) ? -src_height : src_height;
+ const uint64_t src_argb_size = (uint64_t)src_width * abs_src_height * 4;
+ const uint64_t dst_argb_size = (uint64_t)dst_width * dst_height * 4;
+ if (src_argb_size > (UINT64_MAX - dst_argb_size)) {
+ return -1; // Invalid size.
+ }
+ const uint64_t argb_size = src_argb_size + dst_argb_size;
+ if (argb_size > SIZE_MAX) {
+ return -1; // Invalid size.
+ }
+ uint8_t* src_argb = (uint8_t*)malloc((size_t)argb_size);
+ if (!src_argb) {
+ return 1; // Out of memory runtime error.
+ }
+ uint8_t* dst_argb = src_argb + (size_t)src_argb_size;
r = RGB24ToARGB(src_rgb, src_stride_rgb, src_argb, src_width * 4, src_width,
src_height);
if (!r) {
- r = ARGBScale(src_argb, src_width * 4, src_width, src_height, dst_argb,
+ r = ARGBScale(src_argb, src_width * 4, src_width, abs_src_height, dst_argb,
dst_width * 4, dst_width, dst_height, filtering);
if (!r) {
r = ARGBToRGB24(dst_argb, dst_width * 4, dst_rgb, dst_stride_rgb,
diff --git a/source/scale_rvv.cc b/source/scale_rvv.cc
index de037e4..877ccdf 100644
--- a/source/scale_rvv.cc
+++ b/source/scale_rvv.cc
@@ -28,6 +28,55 @@
extern "C" {
#endif
+#ifdef HAS_SCALEARGBFILTERCOLS_RVV
+void ScaleARGBFilterCols_RVV(uint8_t* dst_argb,
+ const uint8_t* src_argb,
+ int dst_width,
+ int x,
+ int dx) {
+ assert(x >= 0);
+
+ size_t vl = __riscv_vsetvl_e32m4(dst_width);
+ vuint32m4_t vx = __riscv_vmv_v_x_u32m4(x, vl);
+ vx = __riscv_vmacc_vx_u32m4(vx, dx, __riscv_vid_v_u32m4(vl), vl);
+ do {
+ vuint32m4_t v0_argb, v1_argb;
+ vuint32m4_t v_xf0_u32, v_xf1_u32;
+ vuint8m4_t v0_argb_u8, v1_argb_u8, v_xf0_u8, v_xf1_u8;
+ vuint16m8_t _v0_argb_u16, v_row_u16;
+ // idx is x >> 16
+ vuint32m4_t v_xi_bindex = __riscv_vsrl_vx_u32m4(vx, 14, vl);
+ v_xi_bindex = __riscv_vand_vx_u32m4(v_xi_bindex, ~3u, vl);
+ // Read Packed ARGB w/ byte index.
+ __riscv_vluxseg2ei32_v_u32m4(&v0_argb, &v1_argb, (const uint32_t*)src_argb,
+ v_xi_bindex, vl);
+ // xf = (x >> 9) & 0x7f;
+ v_xf0_u32 = __riscv_vsrl_vx_u32m4(vx, 9, vl);
+ v_xf0_u32 = __riscv_vand_vx_u32m4(v_xf0_u32, 0x7f, vl);
+ vx = __riscv_vadd_vx_u32m4(vx, vl * dx, vl);
+ // duplicate v_xf0_u32[i] from {0,0,0,f[i]} to {f[i],f[i],f[i],f[i]}
+ v_xf0_u32 = __riscv_vmul_vx_u32m4(v_xf0_u32, 0x01010101, vl);
+ // TODO(fbarchard): Replace 0x7f ^ f with 128-f. bug=607.
+ v_xf1_u32 = __riscv_vxor_vx_u32m4(v_xf0_u32, 0x7f7f7f7f, vl);
+
+ v0_argb_u8 = __riscv_vreinterpret_v_u32m4_u8m4(v0_argb);
+ v1_argb_u8 = __riscv_vreinterpret_v_u32m4_u8m4(v1_argb);
+ v_xf0_u8 = __riscv_vreinterpret_v_u32m4_u8m4(v_xf0_u32);
+ v_xf1_u8 = __riscv_vreinterpret_v_u32m4_u8m4(v_xf1_u32);
+ // ((a) * (0x7f ^ f) + (b)*f) >> 7
+ _v0_argb_u16 = __riscv_vwmulu_vv_u16m8(v0_argb_u8, v_xf1_u8, 4 * vl);
+ v_row_u16 =
+ __riscv_vwmaccu_vv_u16m8(_v0_argb_u16, v1_argb_u8, v_xf0_u8, 4 * vl);
+
+ __riscv_vse8_v_u8m4(dst_argb, __riscv_vnsrl_wx_u8m4(v_row_u16, 7, 4 * vl),
+ 4 * vl);
+ dst_width -= vl;
+ dst_argb += 4 * vl;
+ vl = __riscv_vsetvl_e32m4(dst_width);
+ } while (dst_width > 0);
+}
+#endif
+
#ifdef HAS_SCALEADDROW_RVV
void ScaleAddRow_RVV(const uint8_t* src_ptr, uint16_t* dst_ptr, int src_width) {
size_t w = (size_t)src_width;
@@ -46,6 +95,7 @@
#endif
#ifdef HAS_SCALEARGBROWDOWN2_RVV
+// TODO: Reimplement similar to linear with vlseg2 so u64 is not required
void ScaleARGBRowDown2_RVV(const uint8_t* src_argb,
ptrdiff_t src_stride,
uint8_t* dst_argb,
@@ -67,6 +117,30 @@
#endif
#ifdef HAS_SCALEARGBROWDOWN2LINEAR_RVV
+#if defined(LIBYUV_RVV_HAS_TUPLE_TYPE) && defined(LIBYUV_RVV_HAS_VXRM_ARG)
+void ScaleARGBRowDown2Linear_RVV(const uint8_t* src_argb,
+ ptrdiff_t src_stride,
+ uint8_t* dst_argb,
+ int dst_width) {
+ (void)src_stride;
+ size_t w = (size_t)dst_width;
+ const uint32_t* src = (const uint32_t*)(src_argb);
+ do {
+ size_t vl = __riscv_vsetvl_e32m4(w);
+ vuint32m4x2_t v_src = __riscv_vlseg2e32_v_u32m4x2(src, vl);
+ vuint32m4_t v_even_32 = __riscv_vget_v_u32m4x2_u32m4(v_src, 0);
+ vuint32m4_t v_odd_32 = __riscv_vget_v_u32m4x2_u32m4(v_src, 1);
+ vuint8m4_t v_even = __riscv_vreinterpret_v_u32m4_u8m4(v_even_32);
+ vuint8m4_t v_odd = __riscv_vreinterpret_v_u32m4_u8m4(v_odd_32);
+ vuint8m4_t v_dst =
+ __riscv_vaaddu_vv_u8m4(v_even, v_odd, __RISCV_VXRM_RNU, vl * 4);
+ __riscv_vse8_v_u8m4(dst_argb, v_dst, vl * 4);
+ w -= vl;
+ src += vl * 2;
+ dst_argb += vl * 4;
+ } while (w > 0);
+}
+#else
void ScaleARGBRowDown2Linear_RVV(const uint8_t* src_argb,
ptrdiff_t src_stride,
uint8_t* dst_argb,
@@ -93,8 +167,45 @@
} while (w > 0);
}
#endif
+#endif
#ifdef HAS_SCALEARGBROWDOWN2BOX_RVV
+#if defined(LIBYUV_RVV_HAS_TUPLE_TYPE) && defined(LIBYUV_RVV_HAS_VXRM_ARG)
+void ScaleARGBRowDown2Box_RVV(const uint8_t* src_argb,
+ ptrdiff_t src_stride,
+ uint8_t* dst_argb,
+ int dst_width) {
+ size_t w = (size_t)dst_width;
+ const uint32_t* src0 = (const uint32_t*)(src_argb);
+ const uint32_t* src1 = (const uint32_t*)(src_argb + src_stride);
+ do {
+ size_t vl = __riscv_vsetvl_e32m4(w);
+ vuint32m4x2_t v_src0 = __riscv_vlseg2e32_v_u32m4x2(src0, vl);
+ vuint32m4x2_t v_src1 = __riscv_vlseg2e32_v_u32m4x2(src1, vl);
+ vuint32m4_t v_row0_even_32 = __riscv_vget_v_u32m4x2_u32m4(v_src0, 0);
+ vuint32m4_t v_row0_odd_32 = __riscv_vget_v_u32m4x2_u32m4(v_src0, 1);
+ vuint32m4_t v_row1_even_32 = __riscv_vget_v_u32m4x2_u32m4(v_src1, 0);
+ vuint32m4_t v_row1_odd_32 = __riscv_vget_v_u32m4x2_u32m4(v_src1, 1);
+ vuint8m4_t v_row0_even = __riscv_vreinterpret_v_u32m4_u8m4(v_row0_even_32);
+ vuint8m4_t v_row0_odd = __riscv_vreinterpret_v_u32m4_u8m4(v_row0_odd_32);
+ vuint8m4_t v_row1_even = __riscv_vreinterpret_v_u32m4_u8m4(v_row1_even_32);
+ vuint8m4_t v_row1_odd = __riscv_vreinterpret_v_u32m4_u8m4(v_row1_odd_32);
+ vuint16m8_t v_row0_sum =
+ __riscv_vwaddu_vv_u16m8(v_row0_even, v_row0_odd, vl * 4);
+ vuint16m8_t v_row1_sum =
+ __riscv_vwaddu_vv_u16m8(v_row1_even, v_row1_odd, vl * 4);
+ vuint16m8_t v_dst_16 =
+ __riscv_vadd_vv_u16m8(v_row0_sum, v_row1_sum, vl * 4);
+ vuint8m4_t v_dst =
+ __riscv_vnclipu_wx_u8m4(v_dst_16, 2, __RISCV_VXRM_RNU, vl * 4);
+ __riscv_vse8_v_u8m4(dst_argb, v_dst, vl * 4);
+ w -= vl;
+ src0 += vl * 2;
+ src1 += vl * 2;
+ dst_argb += vl * 4;
+ } while (w > 0);
+}
+#else
void ScaleARGBRowDown2Box_RVV(const uint8_t* src_argb,
ptrdiff_t src_stride,
uint8_t* dst_argb,
@@ -129,6 +240,7 @@
} while (w > 0);
}
#endif
+#endif
#ifdef HAS_SCALEARGBROWDOWNEVEN_RVV
void ScaleARGBRowDownEven_RVV(const uint8_t* src_argb,
@@ -152,6 +264,43 @@
#endif
#ifdef HAS_SCALEARGBROWDOWNEVENBOX_RVV
+#if defined(LIBYUV_RVV_HAS_TUPLE_TYPE) && defined(LIBYUV_RVV_HAS_VXRM_ARG)
+void ScaleARGBRowDownEvenBox_RVV(const uint8_t* src_argb,
+ ptrdiff_t src_stride,
+ int src_stepx,
+ uint8_t* dst_argb,
+ int dst_width) {
+ size_t w = (size_t)dst_width;
+ const uint32_t* src0 = (const uint32_t*)(src_argb);
+ const uint32_t* src1 = (const uint32_t*)(src_argb + src_stride);
+ const int stride_byte = src_stepx * 4;
+ do {
+ size_t vl = __riscv_vsetvl_e32m4(w);
+ vuint32m4x2_t v_src0 = __riscv_vlsseg2e32_v_u32m4x2(src0, stride_byte, vl);
+ vuint32m4x2_t v_src1 = __riscv_vlsseg2e32_v_u32m4x2(src1, stride_byte, vl);
+ vuint32m4_t v_row0_low_32 = __riscv_vget_v_u32m4x2_u32m4(v_src0, 0);
+ vuint32m4_t v_row0_high_32 = __riscv_vget_v_u32m4x2_u32m4(v_src0, 1);
+ vuint32m4_t v_row1_low_32 = __riscv_vget_v_u32m4x2_u32m4(v_src1, 0);
+ vuint32m4_t v_row1_high_32 = __riscv_vget_v_u32m4x2_u32m4(v_src1, 1);
+ vuint8m4_t v_row0_low = __riscv_vreinterpret_v_u32m4_u8m4(v_row0_low_32);
+ vuint8m4_t v_row0_high = __riscv_vreinterpret_v_u32m4_u8m4(v_row0_high_32);
+ vuint8m4_t v_row1_low = __riscv_vreinterpret_v_u32m4_u8m4(v_row1_low_32);
+ vuint8m4_t v_row1_high = __riscv_vreinterpret_v_u32m4_u8m4(v_row1_high_32);
+ vuint16m8_t v_row0_sum =
+ __riscv_vwaddu_vv_u16m8(v_row0_low, v_row0_high, vl * 4);
+ vuint16m8_t v_row1_sum =
+ __riscv_vwaddu_vv_u16m8(v_row1_low, v_row1_high, vl * 4);
+ vuint16m8_t v_sum = __riscv_vadd_vv_u16m8(v_row0_sum, v_row1_sum, vl * 4);
+ vuint8m4_t v_dst =
+ __riscv_vnclipu_wx_u8m4(v_sum, 2, __RISCV_VXRM_RNU, vl * 4);
+ __riscv_vse8_v_u8m4(dst_argb, v_dst, vl * 4);
+ w -= vl;
+ src0 += vl * src_stepx;
+ src1 += vl * src_stepx;
+ dst_argb += vl * 4;
+ } while (w > 0);
+}
+#else
void ScaleARGBRowDownEvenBox_RVV(const uint8_t* src_argb,
ptrdiff_t src_stride,
int src_stepx,
@@ -190,6 +339,7 @@
} while (w > 0);
}
#endif
+#endif
#ifdef HAS_SCALEROWDOWN2_RVV
void ScaleRowDown2_RVV(const uint8_t* src_ptr,
@@ -212,6 +362,26 @@
#endif
#ifdef HAS_SCALEROWDOWN2LINEAR_RVV
+#if defined(LIBYUV_RVV_HAS_TUPLE_TYPE) && defined(LIBYUV_RVV_HAS_VXRM_ARG)
+void ScaleRowDown2Linear_RVV(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst,
+ int dst_width) {
+ size_t w = (size_t)dst_width;
+ (void)src_stride;
+ do {
+ size_t vl = __riscv_vsetvl_e8m4(w);
+ vuint8m4x2_t v_src = __riscv_vlseg2e8_v_u8m4x2(src_ptr, vl);
+ vuint8m4_t v_s0 = __riscv_vget_v_u8m4x2_u8m4(v_src, 0);
+ vuint8m4_t v_s1 = __riscv_vget_v_u8m4x2_u8m4(v_src, 1);
+ vuint8m4_t v_dst = __riscv_vaaddu_vv_u8m4(v_s0, v_s1, __RISCV_VXRM_RNU, vl);
+ __riscv_vse8_v_u8m4(dst, v_dst, vl);
+ w -= vl;
+ src_ptr += 2 * vl;
+ dst += vl;
+ } while (w > 0);
+}
+#else
void ScaleRowDown2Linear_RVV(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst,
@@ -234,8 +404,38 @@
} while (w > 0);
}
#endif
+#endif
#ifdef HAS_SCALEROWDOWN2BOX_RVV
+#if defined(LIBYUV_RVV_HAS_TUPLE_TYPE) && defined(LIBYUV_RVV_HAS_VXRM_ARG)
+void ScaleRowDown2Box_RVV(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst,
+ int dst_width) {
+ const uint8_t* s = src_ptr;
+ const uint8_t* t = src_ptr + src_stride;
+ size_t w = (size_t)dst_width;
+ do {
+ size_t vl = __riscv_vsetvl_e8m4(w);
+ vuint8m4x2_t v_s = __riscv_vlseg2e8_v_u8m4x2(s, vl);
+ vuint8m4x2_t v_t = __riscv_vlseg2e8_v_u8m4x2(t, vl);
+ vuint8m4_t v_s0 = __riscv_vget_v_u8m4x2_u8m4(v_s, 0);
+ vuint8m4_t v_s1 = __riscv_vget_v_u8m4x2_u8m4(v_s, 1);
+ vuint8m4_t v_t0 = __riscv_vget_v_u8m4x2_u8m4(v_t, 0);
+ vuint8m4_t v_t1 = __riscv_vget_v_u8m4x2_u8m4(v_t, 1);
+ vuint16m8_t v_s01 = __riscv_vwaddu_vv_u16m8(v_s0, v_s1, vl);
+ vuint16m8_t v_t01 = __riscv_vwaddu_vv_u16m8(v_t0, v_t1, vl);
+ vuint16m8_t v_st01 = __riscv_vadd_vv_u16m8(v_s01, v_t01, vl);
+ // Use round-to-nearest-up mode for vnclip
+ vuint8m4_t v_dst = __riscv_vnclipu_wx_u8m4(v_st01, 2, __RISCV_VXRM_RNU, vl);
+ __riscv_vse8_v_u8m4(dst, v_dst, vl);
+ w -= vl;
+ s += 2 * vl;
+ t += 2 * vl;
+ dst += vl;
+ } while (w > 0);
+}
+#else
void ScaleRowDown2Box_RVV(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst,
@@ -266,8 +466,27 @@
} while (w > 0);
}
#endif
+#endif
#ifdef HAS_SCALEROWDOWN4_RVV
+#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE
+void ScaleRowDown4_RVV(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ size_t w = (size_t)dst_width;
+ (void)src_stride;
+ do {
+ size_t vl = __riscv_vsetvl_e8m2(w);
+ vuint8m2x4_t v_s = __riscv_vlseg4e8_v_u8m2x4(src_ptr, vl);
+ vuint8m2_t v_s2 = __riscv_vget_v_u8m2x4_u8m2(v_s, 2);
+ __riscv_vse8_v_u8m2(dst_ptr, v_s2, vl);
+ w -= vl;
+ src_ptr += (4 * vl);
+ dst_ptr += vl;
+ } while (w > 0);
+}
+#else
void ScaleRowDown4_RVV(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
@@ -285,8 +504,70 @@
} while (w > 0);
}
#endif
+#endif
#ifdef HAS_SCALEROWDOWN4BOX_RVV
+#if defined(LIBYUV_RVV_HAS_TUPLE_TYPE) && defined(LIBYUV_RVV_HAS_VXRM_ARG)
+void ScaleRowDown4Box_RVV(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ const uint8_t* src_ptr1 = src_ptr + src_stride;
+ const uint8_t* src_ptr2 = src_ptr + src_stride * 2;
+ const uint8_t* src_ptr3 = src_ptr + src_stride * 3;
+ size_t w = (size_t)dst_width;
+ do {
+ size_t vl = __riscv_vsetvl_e8m2(w);
+ vuint8m2x4_t v_s = __riscv_vlseg4e8_v_u8m2x4(src_ptr, vl);
+ vuint8m2_t v_s0 = __riscv_vget_v_u8m2x4_u8m2(v_s, 0);
+ vuint8m2_t v_s1 = __riscv_vget_v_u8m2x4_u8m2(v_s, 1);
+ vuint8m2_t v_s2 = __riscv_vget_v_u8m2x4_u8m2(v_s, 2);
+ vuint8m2_t v_s3 = __riscv_vget_v_u8m2x4_u8m2(v_s, 3);
+ vuint16m4_t v_s01 = __riscv_vwaddu_vv_u16m4(v_s0, v_s1, vl);
+ vuint8m2x4_t v_t = __riscv_vlseg4e8_v_u8m2x4(src_ptr1, vl);
+ vuint8m2_t v_t0 = __riscv_vget_v_u8m2x4_u8m2(v_t, 0);
+ vuint8m2_t v_t1 = __riscv_vget_v_u8m2x4_u8m2(v_t, 1);
+ vuint8m2_t v_t2 = __riscv_vget_v_u8m2x4_u8m2(v_t, 2);
+ vuint8m2_t v_t3 = __riscv_vget_v_u8m2x4_u8m2(v_t, 3);
+ vuint16m4_t v_t01 = __riscv_vwaddu_vv_u16m4(v_t0, v_t1, vl);
+ vuint8m2x4_t v_u = __riscv_vlseg4e8_v_u8m2x4(src_ptr2, vl);
+ vuint8m2_t v_u0 = __riscv_vget_v_u8m2x4_u8m2(v_u, 0);
+ vuint8m2_t v_u1 = __riscv_vget_v_u8m2x4_u8m2(v_u, 1);
+ vuint8m2_t v_u2 = __riscv_vget_v_u8m2x4_u8m2(v_u, 2);
+ vuint8m2_t v_u3 = __riscv_vget_v_u8m2x4_u8m2(v_u, 3);
+ vuint16m4_t v_u01 = __riscv_vwaddu_vv_u16m4(v_u0, v_u1, vl);
+ vuint16m4_t v_u23 = __riscv_vwaddu_vv_u16m4(v_u2, v_u3, vl);
+ vuint16m4_t v_s23 = __riscv_vwaddu_vv_u16m4(v_s2, v_s3, vl);
+ vuint16m4_t v_t23 = __riscv_vwaddu_vv_u16m4(v_t2, v_t3, vl);
+ vuint16m4_t v_st01 = __riscv_vadd_vv_u16m4(v_s01, v_t01, vl);
+ vuint16m4_t v_st23 = __riscv_vadd_vv_u16m4(v_s23, v_t23, vl);
+ vuint8m2x4_t v_v = __riscv_vlseg4e8_v_u8m2x4(src_ptr3, vl);
+ vuint8m2_t v_v0 = __riscv_vget_v_u8m2x4_u8m2(v_v, 0);
+ vuint8m2_t v_v1 = __riscv_vget_v_u8m2x4_u8m2(v_v, 1);
+ vuint8m2_t v_v2 = __riscv_vget_v_u8m2x4_u8m2(v_v, 2);
+ vuint8m2_t v_v3 = __riscv_vget_v_u8m2x4_u8m2(v_v, 3);
+
+ vuint16m4_t v_v01 = __riscv_vwaddu_vv_u16m4(v_v0, v_v1, vl);
+ vuint16m4_t v_v23 = __riscv_vwaddu_vv_u16m4(v_v2, v_v3, vl);
+
+ vuint16m4_t v_uv01 = __riscv_vadd_vv_u16m4(v_u01, v_v01, vl);
+ vuint16m4_t v_uv23 = __riscv_vadd_vv_u16m4(v_u23, v_v23, vl);
+
+ vuint16m4_t v_st0123 = __riscv_vadd_vv_u16m4(v_st01, v_st23, vl);
+ vuint16m4_t v_uv0123 = __riscv_vadd_vv_u16m4(v_uv01, v_uv23, vl);
+ vuint16m4_t v_stuv0123 = __riscv_vadd_vv_u16m4(v_st0123, v_uv0123, vl);
+ vuint8m2_t v_dst =
+ __riscv_vnclipu_wx_u8m2(v_stuv0123, 4, __RISCV_VXRM_RNU, vl);
+ __riscv_vse8_v_u8m2(dst_ptr, v_dst, vl);
+ w -= vl;
+ src_ptr += 4 * vl;
+ src_ptr1 += 4 * vl;
+ src_ptr2 += 4 * vl;
+ src_ptr3 += 4 * vl;
+ dst_ptr += vl;
+ } while (w > 0);
+}
+#else
void ScaleRowDown4Box_RVV(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
@@ -348,8 +629,29 @@
} while (w > 0);
}
#endif
+#endif
#ifdef HAS_SCALEROWDOWN34_RVV
+#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE
+void ScaleRowDown34_RVV(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ size_t w = (size_t)dst_width / 3u;
+ do {
+ size_t vl = __riscv_vsetvl_e8m2(w);
+ vuint8m2x4_t v_src = __riscv_vlseg4e8_v_u8m2x4(src_ptr, vl);
+ vuint8m2_t v_0 = __riscv_vget_v_u8m2x4_u8m2(v_src, 0);
+ vuint8m2_t v_1 = __riscv_vget_v_u8m2x4_u8m2(v_src, 1);
+ vuint8m2_t v_3 = __riscv_vget_v_u8m2x4_u8m2(v_src, 3);
+ vuint8m2x3_t v_dst = __riscv_vcreate_v_u8m2x3(v_0, v_1, v_3);
+ __riscv_vsseg3e8_v_u8m2x3(dst_ptr, v_dst, vl);
+ w -= vl;
+ src_ptr += 4 * vl;
+ dst_ptr += 3 * vl;
+ } while (w > 0);
+}
+#else
void ScaleRowDown34_RVV(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
@@ -366,8 +668,77 @@
} while (w > 0);
}
#endif
+#endif
#ifdef HAS_SCALEROWDOWN34_0_BOX_RVV
+#if defined(LIBYUV_RVV_HAS_TUPLE_TYPE) && defined(LIBYUV_RVV_HAS_VXRM_ARG)
+void ScaleRowDown34_0_Box_RVV(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ size_t w = (size_t)dst_width / 3u;
+ const uint8_t* s = src_ptr;
+ const uint8_t* t = src_ptr + src_stride;
+ do {
+ vuint16m4_t v_t0_u16, v_t1_u16, v_t2_u16, v_t3_u16;
+ vuint8m2_t v_u0, v_u1, v_u2, v_u3;
+ vuint16m4_t v_u1_u16;
+ vuint8m2_t v_a0, v_a1, v_a2;
+ vuint8m2x3_t v_dst;
+ size_t vl = __riscv_vsetvl_e8m2(w);
+ vuint8m2x4_t v_s = __riscv_vlseg4e8_v_u8m2x4(s, vl);
+ vuint8m2_t v_s0 = __riscv_vget_v_u8m2x4_u8m2(v_s, 0);
+ vuint8m2_t v_s1 = __riscv_vget_v_u8m2x4_u8m2(v_s, 1);
+ vuint8m2_t v_s2 = __riscv_vget_v_u8m2x4_u8m2(v_s, 2);
+ vuint8m2_t v_s3 = __riscv_vget_v_u8m2x4_u8m2(v_s, 3);
+
+ if (src_stride == 0) {
+ v_t0_u16 = __riscv_vwaddu_vx_u16m4(v_s0, 2, vl);
+ v_t1_u16 = __riscv_vwaddu_vx_u16m4(v_s1, 2, vl);
+ v_t2_u16 = __riscv_vwaddu_vx_u16m4(v_s2, 2, vl);
+ v_t3_u16 = __riscv_vwaddu_vx_u16m4(v_s3, 2, vl);
+ } else {
+ vuint8m2x4_t v_t = __riscv_vlseg4e8_v_u8m2x4(t, vl);
+ vuint8m2_t v_t0 = __riscv_vget_v_u8m2x4_u8m2(v_t, 0);
+ vuint8m2_t v_t1 = __riscv_vget_v_u8m2x4_u8m2(v_t, 1);
+ vuint8m2_t v_t2 = __riscv_vget_v_u8m2x4_u8m2(v_t, 2);
+ vuint8m2_t v_t3 = __riscv_vget_v_u8m2x4_u8m2(v_t, 3);
+ v_t0_u16 = __riscv_vwaddu_vx_u16m4(v_t0, 0, vl);
+ v_t1_u16 = __riscv_vwaddu_vx_u16m4(v_t1, 0, vl);
+ v_t2_u16 = __riscv_vwaddu_vx_u16m4(v_t2, 0, vl);
+ v_t3_u16 = __riscv_vwaddu_vx_u16m4(v_t3, 0, vl);
+ t += 4 * vl;
+ }
+
+ v_t0_u16 = __riscv_vwmaccu_vx_u16m4(v_t0_u16, 3, v_s0, vl);
+ v_t1_u16 = __riscv_vwmaccu_vx_u16m4(v_t1_u16, 3, v_s1, vl);
+ v_t2_u16 = __riscv_vwmaccu_vx_u16m4(v_t2_u16, 3, v_s2, vl);
+ v_t3_u16 = __riscv_vwmaccu_vx_u16m4(v_t3_u16, 3, v_s3, vl);
+
+ v_u0 = __riscv_vnclipu_wx_u8m2(v_t0_u16, 2, __RISCV_VXRM_RNU, vl);
+ v_u1 = __riscv_vnclipu_wx_u8m2(v_t1_u16, 2, __RISCV_VXRM_RNU, vl);
+ v_u2 = __riscv_vnclipu_wx_u8m2(v_t2_u16, 2, __RISCV_VXRM_RNU, vl);
+ v_u3 = __riscv_vnclipu_wx_u8m2(v_t3_u16, 2, __RISCV_VXRM_RNU, vl);
+ // a0 = (src[0] * 3 + s[1] * 1 + 2) >> 2
+ v_u1_u16 = __riscv_vwaddu_vx_u16m4(v_u1, 0, vl);
+ v_u1_u16 = __riscv_vwmaccu_vx_u16m4(v_u1_u16, 3, v_u0, vl);
+ v_a0 = __riscv_vnclipu_wx_u8m2(v_u1_u16, 2, __RISCV_VXRM_RNU, vl);
+ // a1 = (src[1] * 1 + s[2] * 1 + 1) >> 1
+ v_a1 = __riscv_vaaddu_vv_u8m2(v_u1, v_u2, __RISCV_VXRM_RNU, vl);
+ // a2 = (src[2] * 1 + s[3] * 3 + 2) >> 2
+ v_u1_u16 = __riscv_vwaddu_vx_u16m4(v_u2, 0, vl);
+ v_u1_u16 = __riscv_vwmaccu_vx_u16m4(v_u1_u16, 3, v_u3, vl);
+ v_a2 = __riscv_vnclipu_wx_u8m2(v_u1_u16, 2, __RISCV_VXRM_RNU, vl);
+
+ v_dst = __riscv_vcreate_v_u8m2x3(v_a0, v_a1, v_a2);
+ __riscv_vsseg3e8_v_u8m2x3(dst_ptr, v_dst, vl);
+
+ w -= vl;
+ s += 4 * vl;
+ dst_ptr += 3 * vl;
+ } while (w > 0);
+}
+#else
void ScaleRowDown34_0_Box_RVV(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
@@ -434,8 +805,69 @@
} while (w > 0);
}
#endif
+#endif
#ifdef HAS_SCALEROWDOWN34_1_BOX_RVV
+#if defined(LIBYUV_RVV_HAS_TUPLE_TYPE) && defined(LIBYUV_RVV_HAS_VXRM_ARG)
+void ScaleRowDown34_1_Box_RVV(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ size_t w = (size_t)dst_width / 3u;
+ const uint8_t* s = src_ptr;
+ const uint8_t* t = src_ptr + src_stride;
+ do {
+ vuint8m2_t v_ave0, v_ave1, v_ave2, v_ave3;
+ vuint16m4_t v_u1_u16;
+ vuint8m2_t v_a0, v_a1, v_a2;
+ vuint8m2x3_t v_dst;
+ size_t vl = __riscv_vsetvl_e8m2(w);
+ vuint8m2x4_t v_s = __riscv_vlseg4e8_v_u8m2x4(s, vl);
+ vuint8m2_t v_s0 = __riscv_vget_v_u8m2x4_u8m2(v_s, 0);
+ vuint8m2_t v_s1 = __riscv_vget_v_u8m2x4_u8m2(v_s, 1);
+ vuint8m2_t v_s2 = __riscv_vget_v_u8m2x4_u8m2(v_s, 2);
+ vuint8m2_t v_s3 = __riscv_vget_v_u8m2x4_u8m2(v_s, 3);
+
+ // Use round-to-nearest-up mode for vnclip & averaging add
+ if (src_stride == 0) {
+ v_ave0 = __riscv_vaaddu_vv_u8m2(v_s0, v_s0, __RISCV_VXRM_RNU, vl);
+ v_ave1 = __riscv_vaaddu_vv_u8m2(v_s1, v_s1, __RISCV_VXRM_RNU, vl);
+ v_ave2 = __riscv_vaaddu_vv_u8m2(v_s2, v_s2, __RISCV_VXRM_RNU, vl);
+ v_ave3 = __riscv_vaaddu_vv_u8m2(v_s3, v_s3, __RISCV_VXRM_RNU, vl);
+ } else {
+ vuint8m2x4_t v_t = __riscv_vlseg4e8_v_u8m2x4(t, vl);
+ vuint8m2_t v_t0 = __riscv_vget_v_u8m2x4_u8m2(v_t, 0);
+ vuint8m2_t v_t1 = __riscv_vget_v_u8m2x4_u8m2(v_t, 1);
+ vuint8m2_t v_t2 = __riscv_vget_v_u8m2x4_u8m2(v_t, 2);
+ vuint8m2_t v_t3 = __riscv_vget_v_u8m2x4_u8m2(v_t, 3);
+ v_ave0 = __riscv_vaaddu_vv_u8m2(v_s0, v_t0, __RISCV_VXRM_RNU, vl);
+ v_ave1 = __riscv_vaaddu_vv_u8m2(v_s1, v_t1, __RISCV_VXRM_RNU, vl);
+ v_ave2 = __riscv_vaaddu_vv_u8m2(v_s2, v_t2, __RISCV_VXRM_RNU, vl);
+ v_ave3 = __riscv_vaaddu_vv_u8m2(v_s3, v_t3, __RISCV_VXRM_RNU, vl);
+ t += 4 * vl;
+ }
+ // a0 = (src[0] * 3 + s[1] * 1 + 2) >> 2
+ v_u1_u16 = __riscv_vwaddu_vx_u16m4(v_ave1, 0, vl);
+ v_u1_u16 = __riscv_vwmaccu_vx_u16m4(v_u1_u16, 3, v_ave0, vl);
+ v_a0 = __riscv_vnclipu_wx_u8m2(v_u1_u16, 2, __RISCV_VXRM_RNU, vl);
+
+ // a1 = (src[1] * 1 + s[2] * 1 + 1) >> 1
+ v_a1 = __riscv_vaaddu_vv_u8m2(v_ave1, v_ave2, __RISCV_VXRM_RNU, vl);
+
+ // a2 = (src[2] * 1 + s[3] * 3 + 2) >> 2
+ v_u1_u16 = __riscv_vwaddu_vx_u16m4(v_ave2, 0, vl);
+ v_u1_u16 = __riscv_vwmaccu_vx_u16m4(v_u1_u16, 3, v_ave3, vl);
+ v_a2 = __riscv_vnclipu_wx_u8m2(v_u1_u16, 2, __RISCV_VXRM_RNU, vl);
+
+ v_dst = __riscv_vcreate_v_u8m2x3(v_a0, v_a1, v_a2);
+ __riscv_vsseg3e8_v_u8m2x3(dst_ptr, v_dst, vl);
+
+ w -= vl;
+ s += 4 * vl;
+ dst_ptr += 3 * vl;
+ } while (w > 0);
+}
+#else
void ScaleRowDown34_1_Box_RVV(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
@@ -490,8 +922,31 @@
} while (w > 0);
}
#endif
+#endif
#ifdef HAS_SCALEROWDOWN38_RVV
+#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE
+void ScaleRowDown38_RVV(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ size_t w = (size_t)dst_width / 3u;
+ (void)src_stride;
+ assert(dst_width % 3 == 0);
+ do {
+ size_t vl = __riscv_vsetvl_e8m1(w);
+ vuint8m1x8_t v_src = __riscv_vlseg8e8_v_u8m1x8(src_ptr, vl);
+ vuint8m1_t v_s0 = __riscv_vget_v_u8m1x8_u8m1(v_src, 0);
+ vuint8m1_t v_s3 = __riscv_vget_v_u8m1x8_u8m1(v_src, 3);
+ vuint8m1_t v_s6 = __riscv_vget_v_u8m1x8_u8m1(v_src, 6);
+ vuint8m1x3_t v_dst = __riscv_vcreate_v_u8m1x3(v_s0, v_s3, v_s6);
+ __riscv_vsseg3e8_v_u8m1x3(dst_ptr, v_dst, vl);
+ w -= vl;
+ src_ptr += 8 * vl;
+ dst_ptr += 3 * vl;
+ } while (w > 0);
+}
+#else
void ScaleRowDown38_RVV(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
@@ -511,8 +966,77 @@
} while (w > 0);
}
#endif
+#endif
#ifdef HAS_SCALEROWDOWN38_2_BOX_RVV
+#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE
+void ScaleRowDown38_2_Box_RVV(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ size_t w = (size_t)dst_width / 3u;
+ const uint16_t coeff_a = (65536u / 6u);
+ const uint16_t coeff_b = (65536u / 4u);
+ assert((dst_width % 3 == 0) && (dst_width > 0));
+ do {
+ vuint16m2_t v_e, v_f, v_g;
+ vuint8m1_t v_dst_e, v_dst_f, v_dst_g;
+ vuint8m1x3_t v_dst;
+ size_t vl = __riscv_vsetvl_e8m1(w);
+ // s: e00, e10, e20, f00, f10, f20, g00, g10
+ vuint8m1x8_t v_s = __riscv_vlseg8e8_v_u8m1x8(src_ptr, vl);
+ vuint8m1_t v_s0 = __riscv_vget_v_u8m1x8_u8m1(v_s, 0);
+ vuint8m1_t v_s1 = __riscv_vget_v_u8m1x8_u8m1(v_s, 1);
+ vuint8m1_t v_s2 = __riscv_vget_v_u8m1x8_u8m1(v_s, 2);
+ vuint8m1_t v_s3 = __riscv_vget_v_u8m1x8_u8m1(v_s, 3);
+ vuint8m1_t v_s4 = __riscv_vget_v_u8m1x8_u8m1(v_s, 4);
+ vuint8m1_t v_s5 = __riscv_vget_v_u8m1x8_u8m1(v_s, 5);
+ vuint8m1_t v_s6 = __riscv_vget_v_u8m1x8_u8m1(v_s, 6);
+ vuint8m1_t v_s7 = __riscv_vget_v_u8m1x8_u8m1(v_s, 7);
+ // t: e01, e11, e21, f01, f11, f21, g01, g11
+ vuint8m1x8_t v_t = __riscv_vlseg8e8_v_u8m1x8(src_ptr + src_stride, vl);
+ vuint8m1_t v_t0 = __riscv_vget_v_u8m1x8_u8m1(v_t, 0);
+ vuint8m1_t v_t1 = __riscv_vget_v_u8m1x8_u8m1(v_t, 1);
+ vuint8m1_t v_t2 = __riscv_vget_v_u8m1x8_u8m1(v_t, 2);
+ vuint8m1_t v_t3 = __riscv_vget_v_u8m1x8_u8m1(v_t, 3);
+ vuint8m1_t v_t4 = __riscv_vget_v_u8m1x8_u8m1(v_t, 4);
+ vuint8m1_t v_t5 = __riscv_vget_v_u8m1x8_u8m1(v_t, 5);
+ vuint8m1_t v_t6 = __riscv_vget_v_u8m1x8_u8m1(v_t, 6);
+ vuint8m1_t v_t7 = __riscv_vget_v_u8m1x8_u8m1(v_t, 7);
+ // Calculate sum of [e00, e21] to v_e
+ // Calculate sum of [f00, f21] to v_f
+ // Calculate sum of [g00, g11] to v_g
+ vuint16m2_t v_e0 = __riscv_vwaddu_vv_u16m2(v_s0, v_t0, vl);
+ vuint16m2_t v_e1 = __riscv_vwaddu_vv_u16m2(v_s1, v_t1, vl);
+ vuint16m2_t v_e2 = __riscv_vwaddu_vv_u16m2(v_s2, v_t2, vl);
+ vuint16m2_t v_f0 = __riscv_vwaddu_vv_u16m2(v_s3, v_t3, vl);
+ vuint16m2_t v_f1 = __riscv_vwaddu_vv_u16m2(v_s4, v_t4, vl);
+ vuint16m2_t v_f2 = __riscv_vwaddu_vv_u16m2(v_s5, v_t5, vl);
+ vuint16m2_t v_g0 = __riscv_vwaddu_vv_u16m2(v_s6, v_t6, vl);
+ vuint16m2_t v_g1 = __riscv_vwaddu_vv_u16m2(v_s7, v_t7, vl);
+
+ v_e0 = __riscv_vadd_vv_u16m2(v_e0, v_e1, vl);
+ v_f0 = __riscv_vadd_vv_u16m2(v_f0, v_f1, vl);
+ v_e = __riscv_vadd_vv_u16m2(v_e0, v_e2, vl);
+ v_f = __riscv_vadd_vv_u16m2(v_f0, v_f2, vl);
+ v_g = __riscv_vadd_vv_u16m2(v_g0, v_g1, vl);
+
+ // Average in 16-bit fixed-point
+ v_e = __riscv_vmulhu_vx_u16m2(v_e, coeff_a, vl);
+ v_f = __riscv_vmulhu_vx_u16m2(v_f, coeff_a, vl);
+ v_g = __riscv_vmulhu_vx_u16m2(v_g, coeff_b, vl);
+ v_dst_e = __riscv_vnsrl_wx_u8m1(v_e, 0, vl);
+ v_dst_f = __riscv_vnsrl_wx_u8m1(v_f, 0, vl);
+ v_dst_g = __riscv_vnsrl_wx_u8m1(v_g, 0, vl);
+
+ v_dst = __riscv_vcreate_v_u8m1x3(v_dst_e, v_dst_f, v_dst_g);
+ __riscv_vsseg3e8_v_u8m1x3(dst_ptr, v_dst, vl);
+ w -= vl;
+ src_ptr += 8 * vl;
+ dst_ptr += 3 * vl;
+ } while (w > 0);
+}
+#else
void ScaleRowDown38_2_Box_RVV(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
@@ -569,8 +1093,101 @@
} while (w > 0);
}
#endif
+#endif
#ifdef HAS_SCALEROWDOWN38_3_BOX_RVV
+#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE
+void ScaleRowDown38_3_Box_RVV(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ size_t w = (size_t)dst_width / 3u;
+ const uint16_t coeff_a = (65536u / 9u);
+ const uint16_t coeff_b = (65536u / 6u);
+ assert((dst_width % 3 == 0) && (dst_width > 0));
+ do {
+ vuint16m2_t v_e0, v_e1, v_e2, v_e3, v_e4, v_e;
+ vuint16m2_t v_f0, v_f1, v_f2, v_f3, v_f4, v_f;
+ vuint16m2_t v_g0, v_g1, v_g2, v_g;
+ vuint8m1_t v_dst_e, v_dst_f, v_dst_g;
+ vuint8m1x3_t v_dst;
+ size_t vl = __riscv_vsetvl_e8m1(w);
+ // s: e00, e10, e20, f00, f10, f20, g00, g10
+ vuint8m1x8_t v_s = __riscv_vlseg8e8_v_u8m1x8(src_ptr, vl);
+ vuint8m1_t v_s0 = __riscv_vget_v_u8m1x8_u8m1(v_s, 0);
+ vuint8m1_t v_s1 = __riscv_vget_v_u8m1x8_u8m1(v_s, 1);
+ vuint8m1_t v_s2 = __riscv_vget_v_u8m1x8_u8m1(v_s, 2);
+ vuint8m1_t v_s3 = __riscv_vget_v_u8m1x8_u8m1(v_s, 3);
+ vuint8m1_t v_s4 = __riscv_vget_v_u8m1x8_u8m1(v_s, 4);
+ vuint8m1_t v_s5 = __riscv_vget_v_u8m1x8_u8m1(v_s, 5);
+ vuint8m1_t v_s6 = __riscv_vget_v_u8m1x8_u8m1(v_s, 6);
+ vuint8m1_t v_s7 = __riscv_vget_v_u8m1x8_u8m1(v_s, 7);
+ // t: e01, e11, e21, f01, f11, f21, g01, g11
+ vuint8m1x8_t v_t = __riscv_vlseg8e8_v_u8m1x8(src_ptr + src_stride, vl);
+ vuint8m1_t v_t0 = __riscv_vget_v_u8m1x8_u8m1(v_t, 0);
+ vuint8m1_t v_t1 = __riscv_vget_v_u8m1x8_u8m1(v_t, 1);
+ vuint8m1_t v_t2 = __riscv_vget_v_u8m1x8_u8m1(v_t, 2);
+ vuint8m1_t v_t3 = __riscv_vget_v_u8m1x8_u8m1(v_t, 3);
+ vuint8m1_t v_t4 = __riscv_vget_v_u8m1x8_u8m1(v_t, 4);
+ vuint8m1_t v_t5 = __riscv_vget_v_u8m1x8_u8m1(v_t, 5);
+ vuint8m1_t v_t6 = __riscv_vget_v_u8m1x8_u8m1(v_t, 6);
+ vuint8m1_t v_t7 = __riscv_vget_v_u8m1x8_u8m1(v_t, 7);
+ // u: e02, e12, e22, f02, f12, f22, g02, g12
+ vuint8m1x8_t v_u = __riscv_vlseg8e8_v_u8m1x8(src_ptr + 2 * src_stride, vl);
+ vuint8m1_t v_u0 = __riscv_vget_v_u8m1x8_u8m1(v_u, 0);
+ vuint8m1_t v_u1 = __riscv_vget_v_u8m1x8_u8m1(v_u, 1);
+ vuint8m1_t v_u2 = __riscv_vget_v_u8m1x8_u8m1(v_u, 2);
+ vuint8m1_t v_u3 = __riscv_vget_v_u8m1x8_u8m1(v_u, 3);
+ vuint8m1_t v_u4 = __riscv_vget_v_u8m1x8_u8m1(v_u, 4);
+ vuint8m1_t v_u5 = __riscv_vget_v_u8m1x8_u8m1(v_u, 5);
+ vuint8m1_t v_u6 = __riscv_vget_v_u8m1x8_u8m1(v_u, 6);
+ vuint8m1_t v_u7 = __riscv_vget_v_u8m1x8_u8m1(v_u, 7);
+ // Calculate sum of [e00, e22]
+ v_e0 = __riscv_vwaddu_vv_u16m2(v_s0, v_t0, vl);
+ v_e1 = __riscv_vwaddu_vv_u16m2(v_s1, v_t1, vl);
+ v_e2 = __riscv_vwaddu_vv_u16m2(v_s2, v_t2, vl);
+ v_e3 = __riscv_vwaddu_vv_u16m2(v_u0, v_u1, vl);
+ v_e4 = __riscv_vwaddu_vx_u16m2(v_u2, 0, vl);
+
+ v_e0 = __riscv_vadd_vv_u16m2(v_e0, v_e1, vl);
+ v_e2 = __riscv_vadd_vv_u16m2(v_e2, v_e3, vl);
+ v_e0 = __riscv_vadd_vv_u16m2(v_e0, v_e4, vl);
+ v_e = __riscv_vadd_vv_u16m2(v_e0, v_e2, vl);
+ // Calculate sum of [f00, f22]
+ v_f0 = __riscv_vwaddu_vv_u16m2(v_s3, v_t3, vl);
+ v_f1 = __riscv_vwaddu_vv_u16m2(v_s4, v_t4, vl);
+ v_f2 = __riscv_vwaddu_vv_u16m2(v_s5, v_t5, vl);
+ v_f3 = __riscv_vwaddu_vv_u16m2(v_u3, v_u4, vl);
+ v_f4 = __riscv_vwaddu_vx_u16m2(v_u5, 0, vl);
+
+ v_f0 = __riscv_vadd_vv_u16m2(v_f0, v_f1, vl);
+ v_f2 = __riscv_vadd_vv_u16m2(v_f2, v_f3, vl);
+ v_f0 = __riscv_vadd_vv_u16m2(v_f0, v_f4, vl);
+ v_f = __riscv_vadd_vv_u16m2(v_f0, v_f2, vl);
+ // Calculate sum of [g00, g12]
+ v_g0 = __riscv_vwaddu_vv_u16m2(v_s6, v_t6, vl);
+ v_g1 = __riscv_vwaddu_vv_u16m2(v_s7, v_t7, vl);
+ v_g2 = __riscv_vwaddu_vv_u16m2(v_u6, v_u7, vl);
+
+ v_g = __riscv_vadd_vv_u16m2(v_g0, v_g1, vl);
+ v_g = __riscv_vadd_vv_u16m2(v_g, v_g2, vl);
+
+ // Average in 16-bit fixed-point
+ v_e = __riscv_vmulhu_vx_u16m2(v_e, coeff_a, vl);
+ v_f = __riscv_vmulhu_vx_u16m2(v_f, coeff_a, vl);
+ v_g = __riscv_vmulhu_vx_u16m2(v_g, coeff_b, vl);
+ v_dst_e = __riscv_vnsrl_wx_u8m1(v_e, 0, vl);
+ v_dst_f = __riscv_vnsrl_wx_u8m1(v_f, 0, vl);
+ v_dst_g = __riscv_vnsrl_wx_u8m1(v_g, 0, vl);
+
+ v_dst = __riscv_vcreate_v_u8m1x3(v_dst_e, v_dst_f, v_dst_g);
+ __riscv_vsseg3e8_v_u8m1x3(dst_ptr, v_dst, vl);
+ w -= vl;
+ src_ptr += 8 * vl;
+ dst_ptr += 3 * vl;
+ } while (w > 0);
+}
+#else
void ScaleRowDown38_3_Box_RVV(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
@@ -642,12 +1259,50 @@
} while (w > 0);
}
#endif
+#endif
// ScaleUVRowUp2_(Bi)linear_RVV function is equal to other platforms'
// ScaleRowUp2_(Bi)linear_Any_XXX. We process entire row in this function. Other
// platforms only implement non-edge part of image and process edge with scalar.
#ifdef HAS_SCALEROWUP2_LINEAR_RVV
+#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE
+void ScaleRowUp2_Linear_RVV(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ size_t work_width = (size_t)dst_width - 1u;
+ size_t src_width = work_width >> 1u;
+ const uint8_t* work_src_ptr = src_ptr;
+ uint8_t* work_dst_ptr = dst_ptr + 1;
+ size_t vl = __riscv_vsetvlmax_e8m4();
+ vuint8m4_t v_3 = __riscv_vmv_v_x_u8m4(3, vl);
+ dst_ptr[0] = src_ptr[0];
+ while (src_width > 0) {
+ vuint8m4_t v_src0, v_src1, v_dst_odd, v_dst_even;
+ vuint16m8_t v_src0_u16, v_src1_u16;
+ vuint8m4x2_t v_dst;
+ size_t vl = __riscv_vsetvl_e8m4(src_width);
+ v_src0 = __riscv_vle8_v_u8m4(work_src_ptr, vl);
+ v_src1 = __riscv_vle8_v_u8m4(work_src_ptr + 1, vl);
+
+ v_src0_u16 = __riscv_vwaddu_vx_u16m8(v_src0, 2, vl);
+ v_src1_u16 = __riscv_vwaddu_vx_u16m8(v_src1, 2, vl);
+ v_src0_u16 = __riscv_vwmaccu_vv_u16m8(v_src0_u16, v_3, v_src1, vl);
+ v_src1_u16 = __riscv_vwmaccu_vv_u16m8(v_src1_u16, v_3, v_src0, vl);
+
+ v_dst_odd = __riscv_vnsrl_wx_u8m4(v_src0_u16, 2, vl);
+ v_dst_even = __riscv_vnsrl_wx_u8m4(v_src1_u16, 2, vl);
+
+ v_dst = __riscv_vcreate_v_u8m4x2(v_dst_even, v_dst_odd);
+ __riscv_vsseg2e8_v_u8m4x2(work_dst_ptr, v_dst, vl);
+
+ src_width -= vl;
+ work_src_ptr += vl;
+ work_dst_ptr += 2 * vl;
+ }
+ dst_ptr[dst_width - 1] = src_ptr[(dst_width - 1) / 2];
+}
+#else
void ScaleRowUp2_Linear_RVV(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int dst_width) {
@@ -682,8 +1337,82 @@
dst_ptr[dst_width - 1] = src_ptr[(dst_width - 1) / 2];
}
#endif
+#endif
#ifdef HAS_SCALEROWUP2_BILINEAR_RVV
+#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE
+void ScaleRowUp2_Bilinear_RVV(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width) {
+ size_t work_width = ((size_t)dst_width - 1u) & ~1u;
+ size_t src_width = work_width >> 1u;
+ const uint8_t* work_s = src_ptr;
+ const uint8_t* work_t = src_ptr + src_stride;
+ const uint8_t* s = work_s;
+ const uint8_t* t = work_t;
+ uint8_t* d = dst_ptr;
+ uint8_t* e = dst_ptr + dst_stride;
+ uint8_t* work_d = d + 1;
+ uint8_t* work_e = e + 1;
+ size_t vl = __riscv_vsetvlmax_e16m4();
+ vuint16m4_t v_3_u16 = __riscv_vmv_v_x_u16m4(3, vl);
+ vuint8m2_t v_3_u8 = __riscv_vmv_v_x_u8m2(3, vl);
+ d[0] = (3 * s[0] + t[0] + 2) >> 2;
+ e[0] = (s[0] + 3 * t[0] + 2) >> 2;
+ while (src_width > 0) {
+ vuint8m2_t v_s0, v_s1, v_t0, v_t1;
+ vuint16m4_t v_s0_u16, v_s1_u16, v_t0_u16, v_t1_u16;
+ vuint16m4_t v_t0_u16_, v_t1_u16_;
+ vuint8m2_t v_dst0_even, v_dst0_odd, v_dst1_even, v_dst1_odd;
+ vuint8m2x2_t v_dst0, v_dst1;
+ size_t vl = __riscv_vsetvl_e8m2(src_width);
+ v_s0 = __riscv_vle8_v_u8m2(work_s, vl);
+ v_s1 = __riscv_vle8_v_u8m2(work_s + 1, vl);
+
+ v_s0_u16 = __riscv_vwaddu_vx_u16m4(v_s0, 2, vl);
+ v_s1_u16 = __riscv_vwaddu_vx_u16m4(v_s1, 2, vl);
+ v_s0_u16 = __riscv_vwmaccu_vv_u16m4(v_s0_u16, v_3_u8, v_s1, vl);
+ v_s1_u16 = __riscv_vwmaccu_vv_u16m4(v_s1_u16, v_3_u8, v_s0, vl);
+
+ v_t0 = __riscv_vle8_v_u8m2(work_t, vl);
+ v_t1 = __riscv_vle8_v_u8m2(work_t + 1, vl);
+
+ v_t0_u16 = __riscv_vwaddu_vx_u16m4(v_t0, 2, vl);
+ v_t1_u16 = __riscv_vwaddu_vx_u16m4(v_t1, 2, vl);
+ v_t0_u16 = __riscv_vwmaccu_vv_u16m4(v_t0_u16, v_3_u8, v_t1, vl);
+ v_t1_u16 = __riscv_vwmaccu_vv_u16m4(v_t1_u16, v_3_u8, v_t0, vl);
+
+ v_t0_u16_ = __riscv_vmv_v_v_u16m4(v_t0_u16, vl);
+ v_t1_u16_ = __riscv_vmv_v_v_u16m4(v_t1_u16, vl);
+
+ v_t0_u16 = __riscv_vmacc_vv_u16m4(v_t0_u16, v_3_u16, v_s0_u16, vl);
+ v_t1_u16 = __riscv_vmacc_vv_u16m4(v_t1_u16, v_3_u16, v_s1_u16, vl);
+ v_s0_u16 = __riscv_vmacc_vv_u16m4(v_s0_u16, v_3_u16, v_t0_u16_, vl);
+ v_s1_u16 = __riscv_vmacc_vv_u16m4(v_s1_u16, v_3_u16, v_t1_u16_, vl);
+
+ v_dst0_odd = __riscv_vnsrl_wx_u8m2(v_t0_u16, 4, vl);
+ v_dst0_even = __riscv_vnsrl_wx_u8m2(v_t1_u16, 4, vl);
+ v_dst1_odd = __riscv_vnsrl_wx_u8m2(v_s0_u16, 4, vl);
+ v_dst1_even = __riscv_vnsrl_wx_u8m2(v_s1_u16, 4, vl);
+
+ v_dst0 = __riscv_vcreate_v_u8m2x2(v_dst0_even, v_dst0_odd);
+ __riscv_vsseg2e8_v_u8m2x2(work_d, v_dst0, vl);
+ v_dst1 = __riscv_vcreate_v_u8m2x2(v_dst1_even, v_dst1_odd);
+ __riscv_vsseg2e8_v_u8m2x2(work_e, v_dst1, vl);
+ src_width -= vl;
+ work_s += vl;
+ work_t += vl;
+ work_d += 2 * vl;
+ work_e += 2 * vl;
+ }
+ d[dst_width - 1] =
+ (3 * s[(dst_width - 1) / 2] + t[(dst_width - 1) / 2] + 2) >> 2;
+ e[dst_width - 1] =
+ (s[(dst_width - 1) / 2] + 3 * t[(dst_width - 1) / 2] + 2) >> 2;
+}
+#else
void ScaleRowUp2_Bilinear_RVV(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
@@ -754,6 +1483,7 @@
(s[(dst_width - 1) / 2] + 3 * t[(dst_width - 1) / 2] + 2) >> 2;
}
#endif
+#endif
#ifdef HAS_SCALEUVROWDOWN2_RVV
void ScaleUVRowDown2_RVV(const uint8_t* src_uv,
@@ -777,6 +1507,30 @@
#endif
#ifdef HAS_SCALEUVROWDOWN2LINEAR_RVV
+#if defined(LIBYUV_RVV_HAS_TUPLE_TYPE) && defined(LIBYUV_RVV_HAS_VXRM_ARG)
+void ScaleUVRowDown2Linear_RVV(const uint8_t* src_uv,
+ ptrdiff_t src_stride,
+ uint8_t* dst_uv,
+ int dst_width) {
+ size_t w = (size_t)dst_width;
+ const uint16_t* src = (const uint16_t*)src_uv;
+ (void)src_stride;
+ do {
+ size_t vl = __riscv_vsetvl_e16m4(w);
+ vuint16m4x2_t v_src = __riscv_vlseg2e16_v_u16m4x2(src, vl);
+ vuint16m4_t v_u0v0_16 = __riscv_vget_v_u16m4x2_u16m4(v_src, 0);
+ vuint16m4_t v_u1v1_16 = __riscv_vget_v_u16m4x2_u16m4(v_src, 1);
+ vuint8m4_t v_u0v0 = __riscv_vreinterpret_v_u16m4_u8m4(v_u0v0_16);
+ vuint8m4_t v_u1v1 = __riscv_vreinterpret_v_u16m4_u8m4(v_u1v1_16);
+ vuint8m4_t v_avg =
+ __riscv_vaaddu_vv_u8m4(v_u0v0, v_u1v1, __RISCV_VXRM_RNU, vl * 2);
+ __riscv_vse8_v_u8m4(dst_uv, v_avg, vl * 2);
+ w -= vl;
+ src += vl * 2;
+ dst_uv += vl * 2;
+ } while (w > 0);
+}
+#else
void ScaleUVRowDown2Linear_RVV(const uint8_t* src_uv,
ptrdiff_t src_stride,
uint8_t* dst_uv,
@@ -803,8 +1557,50 @@
} while (w > 0);
}
#endif
+#endif
#ifdef HAS_SCALEUVROWDOWN2BOX_RVV
+#if defined(LIBYUV_RVV_HAS_TUPLE_TYPE) && defined(LIBYUV_RVV_HAS_VXRM_ARG)
+void ScaleUVRowDown2Box_RVV(const uint8_t* src_uv,
+ ptrdiff_t src_stride,
+ uint8_t* dst_uv,
+ int dst_width) {
+ const uint8_t* src_uv_row1 = src_uv + src_stride;
+ size_t w = (size_t)dst_width;
+ do {
+ size_t vl = __riscv_vsetvl_e8m2(w);
+ vuint8m2x4_t v_s = __riscv_vlseg4e8_v_u8m2x4(src_uv, vl);
+ vuint8m2_t v_u0_row0 = __riscv_vget_v_u8m2x4_u8m2(v_s, 0);
+ vuint8m2_t v_v0_row0 = __riscv_vget_v_u8m2x4_u8m2(v_s, 1);
+ vuint8m2_t v_u1_row0 = __riscv_vget_v_u8m2x4_u8m2(v_s, 2);
+ vuint8m2_t v_v1_row0 = __riscv_vget_v_u8m2x4_u8m2(v_s, 3);
+ vuint8m2x4_t v_t = __riscv_vlseg4e8_v_u8m2x4(src_uv_row1, vl);
+ vuint8m2_t v_u0_row1 = __riscv_vget_v_u8m2x4_u8m2(v_t, 0);
+ vuint8m2_t v_v0_row1 = __riscv_vget_v_u8m2x4_u8m2(v_t, 1);
+ vuint8m2_t v_u1_row1 = __riscv_vget_v_u8m2x4_u8m2(v_t, 2);
+ vuint8m2_t v_v1_row1 = __riscv_vget_v_u8m2x4_u8m2(v_t, 3);
+
+ vuint16m4_t v_u0u1_row0 = __riscv_vwaddu_vv_u16m4(v_u0_row0, v_u1_row0, vl);
+ vuint16m4_t v_u0u1_row1 = __riscv_vwaddu_vv_u16m4(v_u0_row1, v_u1_row1, vl);
+ vuint16m4_t v_v0v1_row0 = __riscv_vwaddu_vv_u16m4(v_v0_row0, v_v1_row0, vl);
+ vuint16m4_t v_v0v1_row1 = __riscv_vwaddu_vv_u16m4(v_v0_row1, v_v1_row1, vl);
+ vuint16m4_t v_sum0 = __riscv_vadd_vv_u16m4(v_u0u1_row0, v_u0u1_row1, vl);
+ vuint16m4_t v_sum1 = __riscv_vadd_vv_u16m4(v_v0v1_row0, v_v0v1_row1, vl);
+ vuint8m2_t v_dst_u =
+ __riscv_vnclipu_wx_u8m2(v_sum0, 2, __RISCV_VXRM_RNU, vl);
+ vuint8m2_t v_dst_v =
+ __riscv_vnclipu_wx_u8m2(v_sum1, 2, __RISCV_VXRM_RNU, vl);
+
+ vuint8m2x2_t v_dst_uv = __riscv_vcreate_v_u8m2x2(v_dst_u, v_dst_v);
+ __riscv_vsseg2e8_v_u8m2x2(dst_uv, v_dst_uv, vl);
+
+ dst_uv += 2 * vl;
+ src_uv += 4 * vl;
+ w -= vl;
+ src_uv_row1 += 4 * vl;
+ } while (w > 0);
+}
+#else
void ScaleUVRowDown2Box_RVV(const uint8_t* src_uv,
ptrdiff_t src_stride,
uint8_t* dst_uv,
@@ -847,6 +1643,7 @@
} while (w > 0);
}
#endif
+#endif
#ifdef HAS_SCALEUVROWDOWN4_RVV
void ScaleUVRowDown4_RVV(const uint8_t* src_uv,
@@ -903,6 +1700,49 @@
// scalar.
#ifdef HAS_SCALEUVROWUP2_LINEAR_RVV
+#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE
+void ScaleUVRowUp2_Linear_RVV(const uint8_t* src_ptr,
+ uint8_t* dst_ptr,
+ int dst_width) {
+ size_t work_width = ((size_t)dst_width - 1u) & ~1u;
+ uint16_t* work_dst_ptr = (uint16_t*)dst_ptr + 1;
+ const uint8_t* work_src_ptr = src_ptr;
+ size_t vl = __riscv_vsetvlmax_e8m4();
+ vuint8m4_t v_3_u8 = __riscv_vmv_v_x_u8m4(3, vl);
+ dst_ptr[0] = src_ptr[0];
+ dst_ptr[1] = src_ptr[1];
+ while (work_width > 0) {
+ vuint8m4_t v_uv0, v_uv1, v_dst_odd_u8, v_dst_even_u8;
+ vuint16m4_t v_dst_odd, v_dst_even;
+ vuint16m8_t v_uv0_u16, v_uv1_u16;
+ vuint16m4x2_t v_dst;
+ size_t vl = __riscv_vsetvl_e8m4(work_width);
+ v_uv0 = __riscv_vle8_v_u8m4(work_src_ptr, vl);
+ v_uv1 = __riscv_vle8_v_u8m4(work_src_ptr + 2, vl);
+
+ v_uv0_u16 = __riscv_vwaddu_vx_u16m8(v_uv0, 2, vl);
+ v_uv1_u16 = __riscv_vwaddu_vx_u16m8(v_uv1, 2, vl);
+
+ v_uv0_u16 = __riscv_vwmaccu_vv_u16m8(v_uv0_u16, v_3_u8, v_uv1, vl);
+ v_uv1_u16 = __riscv_vwmaccu_vv_u16m8(v_uv1_u16, v_3_u8, v_uv0, vl);
+
+ v_dst_odd_u8 = __riscv_vnsrl_wx_u8m4(v_uv0_u16, 2, vl);
+ v_dst_even_u8 = __riscv_vnsrl_wx_u8m4(v_uv1_u16, 2, vl);
+
+ v_dst_even = __riscv_vreinterpret_v_u8m4_u16m4(v_dst_even_u8);
+ v_dst_odd = __riscv_vreinterpret_v_u8m4_u16m4(v_dst_odd_u8);
+
+ v_dst = __riscv_vcreate_v_u16m4x2(v_dst_even, v_dst_odd);
+ __riscv_vsseg2e16_v_u16m4x2(work_dst_ptr, v_dst, vl / 2);
+
+ work_width -= vl;
+ work_src_ptr += vl;
+ work_dst_ptr += vl;
+ }
+ dst_ptr[2 * dst_width - 2] = src_ptr[((dst_width + 1) & ~1) - 2];
+ dst_ptr[2 * dst_width - 1] = src_ptr[((dst_width + 1) & ~1) - 1];
+}
+#else
void ScaleUVRowUp2_Linear_RVV(const uint8_t* src_ptr,
uint8_t* dst_ptr,
int dst_width) {
@@ -943,8 +1783,98 @@
dst_ptr[2 * dst_width - 1] = src_ptr[((dst_width + 1) & ~1) - 1];
}
#endif
+#endif
#ifdef HAS_SCALEUVROWUP2_BILINEAR_RVV
+#ifdef LIBYUV_RVV_HAS_TUPLE_TYPE
+void ScaleUVRowUp2_Bilinear_RVV(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst_ptr,
+ ptrdiff_t dst_stride,
+ int dst_width) {
+ size_t work_width = ((size_t)dst_width - 1u) & ~1u;
+ const uint8_t* work_s = src_ptr;
+ const uint8_t* work_t = src_ptr + src_stride;
+ const uint8_t* s = work_s;
+ const uint8_t* t = work_t;
+ uint8_t* d = dst_ptr;
+ uint8_t* e = dst_ptr + dst_stride;
+ uint16_t* work_d = (uint16_t*)d + 1;
+ uint16_t* work_e = (uint16_t*)e + 1;
+ size_t vl = __riscv_vsetvlmax_e16m4();
+ vuint16m4_t v_3_u16 = __riscv_vmv_v_x_u16m4(3, vl);
+ vuint8m2_t v_3_u8 = __riscv_vmv_v_x_u8m2(3, vl);
+ d[0] = (3 * s[0] + t[0] + 2) >> 2;
+ e[0] = (s[0] + 3 * t[0] + 2) >> 2;
+ d[1] = (3 * s[1] + t[1] + 2) >> 2;
+ e[1] = (s[1] + 3 * t[1] + 2) >> 2;
+ while (work_width > 0) {
+ vuint8m2_t v_s0, v_s1, v_t0, v_t1;
+ vuint16m4_t v_s0_u16, v_s1_u16, v_t0_u16, v_t1_u16;
+ vuint16m4_t v_t0_u16_, v_t1_u16_;
+ vuint8m2_t v_dst0_odd_u8, v_dst0_even_u8, v_dst1_odd_u8, v_dst1_even_u8;
+ vuint16m2_t v_dst0_even, v_dst0_odd, v_dst1_even, v_dst1_odd;
+ vuint16m2x2_t v_dst0, v_dst1;
+ size_t vl = __riscv_vsetvl_e8m2(work_width);
+ v_s0 = __riscv_vle8_v_u8m2(work_s, vl);
+ v_s1 = __riscv_vle8_v_u8m2(work_s + 2, vl);
+
+ v_s0_u16 = __riscv_vwaddu_vx_u16m4(v_s0, 2, vl);
+ v_s1_u16 = __riscv_vwaddu_vx_u16m4(v_s1, 2, vl);
+ v_s0_u16 = __riscv_vwmaccu_vv_u16m4(v_s0_u16, v_3_u8, v_s1, vl);
+ v_s1_u16 = __riscv_vwmaccu_vv_u16m4(v_s1_u16, v_3_u8, v_s0, vl);
+
+ v_t0 = __riscv_vle8_v_u8m2(work_t, vl);
+ v_t1 = __riscv_vle8_v_u8m2(work_t + 2, vl);
+
+ v_t0_u16 = __riscv_vwaddu_vx_u16m4(v_t0, 2, vl);
+ v_t1_u16 = __riscv_vwaddu_vx_u16m4(v_t1, 2, vl);
+ v_t0_u16 = __riscv_vwmaccu_vv_u16m4(v_t0_u16, v_3_u8, v_t1, vl);
+ v_t1_u16 = __riscv_vwmaccu_vv_u16m4(v_t1_u16, v_3_u8, v_t0, vl);
+
+ v_t0_u16_ = __riscv_vmv_v_v_u16m4(v_t0_u16, vl);
+ v_t1_u16_ = __riscv_vmv_v_v_u16m4(v_t1_u16, vl);
+
+ v_t0_u16 = __riscv_vmacc_vv_u16m4(v_t0_u16, v_3_u16, v_s0_u16, vl);
+ v_t1_u16 = __riscv_vmacc_vv_u16m4(v_t1_u16, v_3_u16, v_s1_u16, vl);
+ v_s0_u16 = __riscv_vmacc_vv_u16m4(v_s0_u16, v_3_u16, v_t0_u16_, vl);
+ v_s1_u16 = __riscv_vmacc_vv_u16m4(v_s1_u16, v_3_u16, v_t1_u16_, vl);
+
+ v_dst0_odd_u8 = __riscv_vnsrl_wx_u8m2(v_t0_u16, 4, vl);
+ v_dst0_even_u8 = __riscv_vnsrl_wx_u8m2(v_t1_u16, 4, vl);
+ v_dst1_odd_u8 = __riscv_vnsrl_wx_u8m2(v_s0_u16, 4, vl);
+ v_dst1_even_u8 = __riscv_vnsrl_wx_u8m2(v_s1_u16, 4, vl);
+
+ v_dst0_even = __riscv_vreinterpret_v_u8m2_u16m2(v_dst0_even_u8);
+ v_dst0_odd = __riscv_vreinterpret_v_u8m2_u16m2(v_dst0_odd_u8);
+ v_dst1_even = __riscv_vreinterpret_v_u8m2_u16m2(v_dst1_even_u8);
+ v_dst1_odd = __riscv_vreinterpret_v_u8m2_u16m2(v_dst1_odd_u8);
+
+ v_dst0 = __riscv_vcreate_v_u16m2x2(v_dst0_even, v_dst0_odd);
+ __riscv_vsseg2e16_v_u16m2x2(work_d, v_dst0, vl / 2);
+ v_dst1 = __riscv_vcreate_v_u16m2x2(v_dst1_even, v_dst1_odd);
+ __riscv_vsseg2e16_v_u16m2x2(work_e, v_dst1, vl / 2);
+
+ work_width -= vl;
+ work_s += vl;
+ work_t += vl;
+ work_d += vl;
+ work_e += vl;
+ }
+ d[2 * dst_width - 2] =
+ (3 * s[((dst_width + 1) & ~1) - 2] + t[((dst_width + 1) & ~1) - 2] + 2) >>
+ 2;
+ e[2 * dst_width - 2] =
+ (s[((dst_width + 1) & ~1) - 2] + 3 * t[((dst_width + 1) & ~1) - 2] + 2) >>
+ 2;
+ d[2 * dst_width - 1] =
+ (3 * s[((dst_width + 1) & ~1) - 1] + t[((dst_width + 1) & ~1) - 1] + 2) >>
+ 2;
+ e[2 * dst_width - 1] =
+ (s[((dst_width + 1) & ~1) - 1] + 3 * t[((dst_width + 1) & ~1) - 1] + 2) >>
+ 2;
+}
+#else
void ScaleUVRowUp2_Bilinear_RVV(const uint8_t* src_ptr,
ptrdiff_t src_stride,
uint8_t* dst_ptr,
@@ -1030,6 +1960,7 @@
2;
}
#endif
+#endif
#ifdef __cplusplus
} // extern "C"
diff --git a/source/scale_sme.cc b/source/scale_sme.cc
new file mode 100644
index 0000000..fa74569
--- /dev/null
+++ b/source/scale_sme.cc
@@ -0,0 +1,555 @@
+/*
+ * Copyright 2024 The LibYuv Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "libyuv/scale_row.h"
+
+#ifdef __cplusplus
+namespace libyuv {
+extern "C" {
+#endif
+
+#if !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && \
+ defined(__aarch64__)
+
+__arm_locally_streaming void ScaleRowDown2_SME(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst,
+ int dst_width) {
+ // Streaming-SVE only, no use of ZA tile.
+ (void)src_stride;
+ int vl;
+ asm volatile(
+ "cntb %x[vl] \n"
+ "subs %w[dst_width], %w[dst_width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ "1: \n"
+ "ptrue p0.b \n"
+ "ld2b {z0.b, z1.b}, p0/z, [%[src_ptr]] \n"
+ "incb %[src_ptr], all, mul #2 \n"
+ "subs %w[dst_width], %w[dst_width], %w[vl] \n"
+ "st1b {z1.b}, p0, [%[dst_ptr]] \n"
+ "incb %[dst_ptr] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[dst_width], %w[dst_width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ "whilelt p0.b, wzr, %w[dst_width] \n"
+ "ld2b {z0.b, z1.b}, p0/z, [%[src_ptr]] \n"
+ "st1b {z1.b}, p0, [%[dst_ptr]] \n"
+
+ "99: \n"
+ : [src_ptr] "+r"(src_ptr), // %[src_ptr]
+ [dst_ptr] "+r"(dst), // %[dst_ptr]
+ [dst_width] "+r"(dst_width), // %[dst_width]
+ [vl] "=r"(vl) // %[vl]
+ :
+ : "memory", "cc", "z0", "z1", "p0");
+}
+
+__arm_locally_streaming void ScaleRowDown2_16_SME(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst,
+ int dst_width) {
+ // Streaming-SVE only, no use of ZA tile.
+ (void)src_stride;
+ int vl;
+ asm volatile(
+ "cnth %x[vl] \n"
+ "subs %w[dst_width], %w[dst_width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ "1: \n"
+ "ptrue p0.h \n"
+ "ld2h {z0.h, z1.h}, p0/z, [%[src_ptr]] \n"
+ "incb %[src_ptr], all, mul #2 \n"
+ "subs %w[dst_width], %w[dst_width], %w[vl] \n"
+ "st1h {z1.h}, p0, [%[dst_ptr]] \n"
+ "incb %[dst_ptr] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[dst_width], %w[dst_width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ "whilelt p0.h, wzr, %w[dst_width] \n"
+ "ld2h {z0.h, z1.h}, p0/z, [%[src_ptr]] \n"
+ "st1h {z1.h}, p0, [%[dst_ptr]] \n"
+
+ "99: \n"
+ : [src_ptr] "+r"(src_ptr), // %[src_ptr]
+ [dst_ptr] "+r"(dst), // %[dst_ptr]
+ [dst_width] "+r"(dst_width), // %[dst_width]
+ [vl] "=r"(vl) // %[vl]
+ :
+ : "memory", "cc", "z0", "z1", "p0");
+}
+
+__arm_locally_streaming void ScaleRowDown2Linear_SME(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst,
+ int dst_width) {
+ // Streaming-SVE only, no use of ZA tile.
+ (void)src_stride;
+ int vl;
+ asm volatile(
+ "cntb %x[vl] \n"
+ "subs %w[dst_width], %w[dst_width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ "1: \n"
+ "ptrue p0.b \n"
+ "ld2b {z0.b, z1.b}, p0/z, [%[src_ptr]] \n"
+ "incb %[src_ptr], all, mul #2 \n"
+ "urhadd z0.b, p0/m, z0.b, z1.b \n"
+ "subs %w[dst_width], %w[dst_width], %w[vl] \n"
+ "st1b {z0.b}, p0, [%[dst_ptr]] \n"
+ "incb %[dst_ptr] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[dst_width], %w[dst_width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ "whilelt p0.b, wzr, %w[dst_width] \n"
+ "ld2b {z0.b, z1.b}, p0/z, [%[src_ptr]] \n"
+ "urhadd z0.b, p0/m, z0.b, z1.b \n"
+ "st1b {z0.b}, p0, [%[dst_ptr]] \n"
+
+ "99: \n"
+ : [src_ptr] "+r"(src_ptr), // %[src_ptr]
+ [dst_ptr] "+r"(dst), // %[dst_ptr]
+ [dst_width] "+r"(dst_width), // %[dst_width]
+ [vl] "=r"(vl) // %[vl]
+ :
+ : "memory", "cc", "z0", "z1", "p0");
+}
+
+__arm_locally_streaming void ScaleRowDown2Linear_16_SME(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst,
+ int dst_width) {
+ // Streaming-SVE only, no use of ZA tile.
+ (void)src_stride;
+ int vl;
+ asm volatile(
+ "cnth %x[vl] \n"
+ "subs %w[dst_width], %w[dst_width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ "1: \n"
+ "ptrue p0.h \n"
+ "ld2h {z0.h, z1.h}, p0/z, [%[src_ptr]] \n"
+ "incb %[src_ptr], all, mul #2 \n"
+ "urhadd z0.h, p0/m, z0.h, z1.h \n"
+ "subs %w[dst_width], %w[dst_width], %w[vl] \n"
+ "st1h {z0.h}, p0, [%[dst_ptr]] \n"
+ "incb %[dst_ptr] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[dst_width], %w[dst_width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ "whilelt p0.h, wzr, %w[dst_width] \n"
+ "ld2h {z0.h, z1.h}, p0/z, [%[src_ptr]] \n"
+ "urhadd z0.h, p0/m, z0.h, z1.h \n"
+ "st1h {z0.h}, p0, [%[dst_ptr]] \n"
+
+ "99: \n"
+ : [src_ptr] "+r"(src_ptr), // %[src_ptr]
+ [dst_ptr] "+r"(dst), // %[dst_ptr]
+ [dst_width] "+r"(dst_width), // %[dst_width]
+ [vl] "=r"(vl) // %[vl]
+ :
+ : "memory", "cc", "z0", "z1", "p0");
+}
+
+#define SCALEROWDOWN2BOX_SVE \
+ "ld2b {z0.b, z1.b}, p0/z, [%[src_ptr]] \n" \
+ "ld2b {z2.b, z3.b}, p0/z, [%[src2_ptr]] \n" \
+ "incb %[src_ptr], all, mul #2 \n" \
+ "incb %[src2_ptr], all, mul #2 \n" \
+ "uaddlb z4.h, z0.b, z1.b \n" \
+ "uaddlt z5.h, z0.b, z1.b \n" \
+ "uaddlb z6.h, z2.b, z3.b \n" \
+ "uaddlt z7.h, z2.b, z3.b \n" \
+ "add z4.h, z4.h, z6.h \n" \
+ "add z5.h, z5.h, z7.h \n" \
+ "rshrnb z0.b, z4.h, #2 \n" \
+ "rshrnt z0.b, z5.h, #2 \n" \
+ "subs %w[dst_width], %w[dst_width], %w[vl] \n" \
+ "st1b {z0.b}, p0, [%[dst_ptr]] \n" \
+ "incb %[dst_ptr] \n"
+
+__arm_locally_streaming void ScaleRowDown2Box_SME(const uint8_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint8_t* dst,
+ int dst_width) {
+ // Streaming-SVE only, no use of ZA tile.
+ const uint8_t* src2_ptr = src_ptr + src_stride;
+ int vl;
+ asm volatile(
+ "cntb %x[vl] \n"
+ "subs %w[dst_width], %w[dst_width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ "ptrue p0.b \n"
+ "1: \n" //
+ SCALEROWDOWN2BOX_SVE
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[dst_width], %w[dst_width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ "whilelt p0.b, wzr, %w[dst_width] \n" //
+ SCALEROWDOWN2BOX_SVE
+
+ "99: \n"
+ : [src_ptr] "+r"(src_ptr), // %[src_ptr]
+ [src2_ptr] "+r"(src2_ptr), // %[src2_ptr]
+ [dst_ptr] "+r"(dst), // %[dst_ptr]
+ [dst_width] "+r"(dst_width), // %[dst_width]
+ [vl] "=r"(vl) // %[vl]
+ :
+ : "memory", "cc", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "p0");
+}
+
+#undef SCALEROWDOWN2BOX_SVE
+
+#define SCALEROWDOWN2BOX_16_SVE \
+ "ld2h {z0.h, z1.h}, p0/z, [%[src_ptr]] \n" \
+ "ld2h {z2.h, z3.h}, p0/z, [%[src2_ptr]] \n" \
+ "incb %[src_ptr], all, mul #2 \n" \
+ "incb %[src2_ptr], all, mul #2 \n" \
+ "uaddlb z4.s, z0.h, z1.h \n" \
+ "uaddlt z5.s, z0.h, z1.h \n" \
+ "uaddlb z6.s, z2.h, z3.h \n" \
+ "uaddlt z7.s, z2.h, z3.h \n" \
+ "add z4.s, z4.s, z6.s \n" \
+ "add z5.s, z5.s, z7.s \n" \
+ "rshrnb z0.h, z4.s, #2 \n" \
+ "rshrnt z0.h, z5.s, #2 \n" \
+ "subs %w[dst_width], %w[dst_width], %w[vl] \n" \
+ "st1h {z0.h}, p0, [%[dst_ptr]] \n" \
+ "incb %[dst_ptr] \n"
+
+__arm_locally_streaming void ScaleRowDown2Box_16_SME(const uint16_t* src_ptr,
+ ptrdiff_t src_stride,
+ uint16_t* dst,
+ int dst_width) {
+ // Streaming-SVE only, no use of ZA tile.
+ const uint16_t* src2_ptr = src_ptr + src_stride;
+ int vl;
+ asm volatile(
+ "cnth %x[vl] \n"
+ "subs %w[dst_width], %w[dst_width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ "ptrue p0.h \n"
+ "1: \n" //
+ SCALEROWDOWN2BOX_16_SVE
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[dst_width], %w[dst_width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ "whilelt p0.h, wzr, %w[dst_width] \n" //
+ SCALEROWDOWN2BOX_16_SVE
+
+ "99: \n"
+ : [src_ptr] "+r"(src_ptr), // %[src_ptr]
+ [src2_ptr] "+r"(src2_ptr), // %[src2_ptr]
+ [dst_ptr] "+r"(dst), // %[dst_ptr]
+ [dst_width] "+r"(dst_width), // %[dst_width]
+ [vl] "=r"(vl) // %[vl]
+ :
+ : "memory", "cc", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "p0");
+}
+
+#undef SCALEROWDOWN2BOX_16_SVE
+
+__arm_locally_streaming void ScaleUVRowDown2_SME(const uint8_t* src_uv,
+ ptrdiff_t src_stride,
+ uint8_t* dst_uv,
+ int dst_width) {
+ // Streaming-SVE only, no use of ZA tile.
+ (void)src_stride;
+ int vl;
+ asm volatile(
+ "cnth %x[vl] \n"
+ "subs %w[dst_width], %w[dst_width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ "1: \n"
+ "ptrue p0.b \n"
+ "ld2h {z0.h, z1.h}, p0/z, [%[src_uv]] \n"
+ "incb %[src_uv], all, mul #2 \n"
+ "subs %w[dst_width], %w[dst_width], %w[vl] \n"
+ "st1h {z1.h}, p0, [%[dst_uv]] \n"
+ "incb %[dst_uv] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[dst_width], %w[dst_width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ "whilelt p0.h, wzr, %w[dst_width] \n"
+ "ld2h {z0.h, z1.h}, p0/z, [%[src_uv]] \n"
+ "st1h {z1.h}, p0, [%[dst_uv]] \n"
+
+ "99: \n"
+ : [src_uv] "+r"(src_uv), // %[src_uv]
+ [dst_uv] "+r"(dst_uv), // %[dst_uv]
+ [dst_width] "+r"(dst_width), // %[dst_width]
+ [vl] "=r"(vl) // %[vl]
+ :
+ : "memory", "cc", "z0", "z1", "p0");
+}
+
+__arm_locally_streaming void ScaleUVRowDown2Linear_SME(const uint8_t* src_uv,
+ ptrdiff_t src_stride,
+ uint8_t* dst_uv,
+ int dst_width) {
+ // Streaming-SVE only, no use of ZA tile.
+ (void)src_stride;
+ int vl;
+ asm volatile(
+ "cnth %x[vl] \n"
+ "ptrue p1.b \n"
+ "subs %w[dst_width], %w[dst_width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ "ptrue p0.h \n"
+ "1: \n"
+ "ld2h {z0.h, z1.h}, p0/z, [%[src_uv]] \n"
+ "incb %[src_uv], all, mul #2 \n"
+ "urhadd z0.b, p1/m, z0.b, z1.b \n"
+ "st1h {z0.h}, p0, [%[dst_uv]] \n"
+ "incb %[dst_uv], all, mul #1 \n"
+ "subs %w[dst_width], %w[dst_width], %w[vl] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[dst_width], %w[dst_width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ "whilelt p0.h, wzr, %w[dst_width] \n"
+ "ld2h {z0.h, z1.h}, p0/z, [%[src_uv]] \n"
+ "urhadd z0.b, p1/m, z0.b, z1.b \n"
+ "st1h {z0.h}, p0, [%[dst_uv]] \n"
+
+ "99: \n"
+ : [src_uv] "+r"(src_uv), // %[src_uv]
+ [dst_uv] "+r"(dst_uv), // %[dst_uv]
+ [dst_width] "+r"(dst_width), // %[dst_width]
+ [vl] "=r"(vl) // %[vl]
+ :
+ : "z0", "z1", "p0", "p1");
+}
+
+#define SCALEUVROWDOWN2BOX_SVE \
+ "ld2h {z0.h, z1.h}, p0/z, [%[src_uv]] \n" \
+ "ld2h {z2.h, z3.h}, p0/z, [%[src2_uv]] \n" \
+ "incb %[src_uv], all, mul #2 \n" \
+ "incb %[src2_uv], all, mul #2 \n" \
+ "uaddlb z4.h, z0.b, z1.b \n" \
+ "uaddlt z5.h, z0.b, z1.b \n" \
+ "uaddlb z6.h, z2.b, z3.b \n" \
+ "uaddlt z7.h, z2.b, z3.b \n" \
+ "add z4.h, z4.h, z6.h \n" \
+ "add z5.h, z5.h, z7.h \n" \
+ "rshrnb z0.b, z4.h, #2 \n" \
+ "rshrnt z0.b, z5.h, #2 \n" \
+ "st1h {z0.h}, p0, [%[dst_uv]] \n" \
+ "incb %[dst_uv], all, mul #1 \n"
+
+__arm_locally_streaming void ScaleUVRowDown2Box_SME(const uint8_t* src_uv,
+ ptrdiff_t src_stride,
+ uint8_t* dst_uv,
+ int dst_width) {
+ // Streaming-SVE only, no use of ZA tile.
+ const uint8_t* src2_uv = src_uv + src_stride;
+ int vl;
+ asm volatile(
+ "cnth %x[vl] \n"
+ "ptrue p1.b \n"
+ "subs %w[dst_width], %w[dst_width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ "ptrue p0.h \n"
+ "1: \n" //
+ SCALEUVROWDOWN2BOX_SVE
+ "subs %w[dst_width], %w[dst_width], %w[vl] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[dst_width], %w[dst_width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ "whilelt p0.h, wzr, %w[dst_width] \n" //
+ SCALEUVROWDOWN2BOX_SVE
+
+ "99: \n"
+ : [src_uv] "+r"(src_uv), // %[src_uv]
+ [src2_uv] "+r"(src2_uv), // %[src2_uv]
+ [dst_uv] "+r"(dst_uv), // %[dst_uv]
+ [dst_width] "+r"(dst_width), // %[dst_width]
+ [vl] "=r"(vl) // %[vl]
+ :
+ : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "p0", "p1");
+}
+
+#undef SCALEUVROWDOWN2BOX_SVE
+
+__arm_locally_streaming void ScaleARGBRowDown2_SME(const uint8_t* src_argb,
+ ptrdiff_t src_stride,
+ uint8_t* dst_argb,
+ int dst_width) {
+ // Streaming-SVE only, no use of ZA tile.
+ (void)src_stride;
+ int vl;
+ asm volatile(
+ "cntw %x[vl] \n"
+ "subs %w[dst_width], %w[dst_width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ "1: \n"
+ "ptrue p0.b \n"
+ "ld2w {z0.s, z1.s}, p0/z, [%[src_argb]] \n"
+ "incb %[src_argb], all, mul #2 \n"
+ "subs %w[dst_width], %w[dst_width], %w[vl] \n"
+ "st1w {z1.s}, p0, [%[dst_argb]] \n"
+ "incb %[dst_argb] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[dst_width], %w[dst_width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ "whilelt p0.s, wzr, %w[dst_width] \n"
+ "ld2w {z0.s, z1.s}, p0/z, [%[src_argb]] \n"
+ "st1w {z1.s}, p0, [%[dst_argb]] \n"
+
+ "99: \n"
+ : [src_argb] "+r"(src_argb), // %[src_argb]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [dst_width] "+r"(dst_width), // %[dst_width]
+ [vl] "=r"(vl) // %[vl]
+ :
+ : "memory", "cc", "z0", "z1", "p0");
+}
+
+__arm_locally_streaming void ScaleARGBRowDown2Linear_SME(
+ const uint8_t* src_argb,
+ ptrdiff_t src_stride,
+ uint8_t* dst_argb,
+ int dst_width) {
+ // Streaming-SVE only, no use of ZA tile.
+ (void)src_stride;
+ int vl;
+ asm volatile(
+ "cntw %x[vl] \n"
+ "ptrue p1.b \n"
+ "subs %w[dst_width], %w[dst_width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ "ptrue p0.s \n"
+ "1: \n"
+ "ld2w {z0.s, z1.s}, p0/z, [%[src_argb]] \n"
+ "incb %[src_argb], all, mul #2 \n"
+ "urhadd z0.b, p1/m, z0.b, z1.b \n"
+ "st1w {z0.s}, p0, [%[dst_argb]] \n"
+ "incb %[dst_argb], all, mul #1 \n"
+ "subs %w[dst_width], %w[dst_width], %w[vl] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[dst_width], %w[dst_width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ "whilelt p0.s, wzr, %w[dst_width] \n"
+ "ld2w {z0.s, z1.s}, p0/z, [%[src_argb]] \n"
+ "urhadd z0.b, p1/m, z0.b, z1.b \n"
+ "st1w {z0.s}, p0, [%[dst_argb]] \n"
+
+ "99: \n"
+ : [src_argb] "+r"(src_argb), // %[src_argb]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [dst_width] "+r"(dst_width), // %[dst_width]
+ [vl] "=r"(vl) // %[vl]
+ :
+ : "memory", "cc", "z0", "z1", "p0", "p1");
+}
+
+#define SCALEARGBROWDOWN2BOX_SVE \
+ "ld2w {z0.s, z1.s}, p0/z, [%[src_argb]] \n" \
+ "ld2w {z2.s, z3.s}, p0/z, [%[src2_argb]] \n" \
+ "incb %[src_argb], all, mul #2 \n" \
+ "incb %[src2_argb], all, mul #2 \n" \
+ "uaddlb z4.h, z0.b, z1.b \n" \
+ "uaddlt z5.h, z0.b, z1.b \n" \
+ "uaddlb z6.h, z2.b, z3.b \n" \
+ "uaddlt z7.h, z2.b, z3.b \n" \
+ "add z4.h, z4.h, z6.h \n" \
+ "add z5.h, z5.h, z7.h \n" \
+ "rshrnb z0.b, z4.h, #2 \n" \
+ "rshrnt z0.b, z5.h, #2 \n" \
+ "st1w {z0.s}, p0, [%[dst_argb]] \n" \
+ "incb %[dst_argb], all, mul #1 \n"
+
+__arm_locally_streaming void ScaleARGBRowDown2Box_SME(const uint8_t* src_argb,
+ ptrdiff_t src_stride,
+ uint8_t* dst_argb,
+ int dst_width) {
+ // Streaming-SVE only, no use of ZA tile.
+ const uint8_t* src2_argb = src_argb + src_stride;
+ int vl;
+ asm volatile(
+ "cntw %x[vl] \n"
+ "ptrue p1.b \n"
+ "subs %w[dst_width], %w[dst_width], %w[vl] \n"
+ "b.lt 2f \n"
+
+ "ptrue p0.s \n"
+ "1: \n" //
+ SCALEARGBROWDOWN2BOX_SVE
+ "subs %w[dst_width], %w[dst_width], %w[vl] \n"
+ "b.ge 1b \n"
+
+ "2: \n"
+ "adds %w[dst_width], %w[dst_width], %w[vl] \n"
+ "b.eq 99f \n"
+
+ "whilelt p0.s, wzr, %w[dst_width] \n" //
+ SCALEARGBROWDOWN2BOX_SVE
+
+ "99: \n"
+ : [src_argb] "+r"(src_argb), // %[src_argb]
+ [src2_argb] "+r"(src2_argb), // %[src2_argb]
+ [dst_argb] "+r"(dst_argb), // %[dst_argb]
+ [dst_width] "+r"(dst_width), // %[dst_width]
+ [vl] "=r"(vl) // %[vl]
+ :
+ : "memory", "cc", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "p0",
+ "p1");
+}
+
+#endif // !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) &&
+ // defined(__aarch64__)
+
+#ifdef __cplusplus
+} // extern "C"
+} // namespace libyuv
+#endif
diff --git a/source/scale_uv.cc b/source/scale_uv.cc
index 0931c89..700d1b2 100644
--- a/source/scale_uv.cc
+++ b/source/scale_uv.cc
@@ -8,7 +8,7 @@
* be found in the AUTHORS file in the root of the source tree.
*/
-#include "libyuv/scale.h"
+#include "libyuv/scale_uv.h"
#include <assert.h>
#include <string.h>
@@ -104,14 +104,6 @@
}
}
#endif
-#if defined(HAS_SCALEUVROWDOWN2BOX_NEON)
- if (TestCpuFlag(kCpuHasNEON) && filtering) {
- ScaleUVRowDown2 = ScaleUVRowDown2Box_Any_NEON;
- if (IS_ALIGNED(dst_width, 8)) {
- ScaleUVRowDown2 = ScaleUVRowDown2Box_NEON;
- }
- }
-#endif
#if defined(HAS_SCALEUVROWDOWN2_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
ScaleUVRowDown2 =
@@ -128,6 +120,13 @@
}
}
#endif
+#if defined(HAS_SCALEUVROWDOWN2_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ ScaleUVRowDown2 = filtering == kFilterNone ? ScaleUVRowDown2_SME
+ : filtering == kFilterLinear ? ScaleUVRowDown2Linear_SME
+ : ScaleUVRowDown2Box_SME;
+ }
+#endif
#if defined(HAS_SCALEUVROWDOWN2_RVV)
if (TestCpuFlag(kCpuHasRVV)) {
ScaleUVRowDown2 =
@@ -242,6 +241,11 @@
}
}
#endif
+#if defined(HAS_SCALEUVROWDOWN2BOX_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ ScaleUVRowDown2 = ScaleUVRowDown2Box_SME;
+ }
+#endif
#if defined(HAS_SCALEUVROWDOWN2BOX_RVV)
if (TestCpuFlag(kCpuHasRVV)) {
ScaleUVRowDown2 = ScaleUVRowDown2Box_RVV;
@@ -327,10 +331,16 @@
}
}
#endif
-#if defined(HAS_SCALEUVROWDOWNEVEN_RVV)
+#if defined(HAS_SCALEUVROWDOWNEVEN_RVV) || defined(HAS_SCALEUVROWDOWN4_RVV)
if (TestCpuFlag(kCpuHasRVV) && !filtering) {
- ScaleUVRowDownEven =
- (col_step == 4) ? ScaleUVRowDown4_RVV : ScaleUVRowDownEven_RVV;
+#if defined(HAS_SCALEUVROWDOWNEVEN_RVV)
+ ScaleUVRowDownEven = ScaleUVRowDownEven_RVV;
+#endif
+#if defined(HAS_SCALEUVROWDOWN4_RVV)
+ if (col_step == 4) {
+ ScaleUVRowDownEven = ScaleUVRowDown4_RVV;
+ }
+#endif
}
#endif
@@ -404,6 +414,11 @@
}
}
#endif
+#if defined(HAS_INTERPOLATEROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ InterpolateRow = InterpolateRow_SME;
+ }
+#endif
#if defined(HAS_INTERPOLATEROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
InterpolateRow = InterpolateRow_Any_MSA;
@@ -525,6 +540,11 @@
}
}
#endif
+#if defined(HAS_INTERPOLATEROW_SME)
+ if (TestCpuFlag(kCpuHasSME)) {
+ InterpolateRow = InterpolateRow_SME;
+ }
+#endif
#if defined(HAS_INTERPOLATEROW_MSA)
if (TestCpuFlag(kCpuHasMSA)) {
InterpolateRow = InterpolateRow_Any_MSA;
@@ -680,6 +700,7 @@
int dy;
// This function can only scale up by 2 times horizontally.
+ (void)src_width;
assert(src_width == ((dst_width + 1) / 2));
#ifdef HAS_SCALEUVROWUP2_LINEAR_SSSE3
@@ -738,6 +759,7 @@
int x;
// This function can only scale up by 2 times.
+ (void)src_width;
assert(src_width == ((dst_width + 1) / 2));
assert(src_height == ((dst_height + 1) / 2));
@@ -799,6 +821,7 @@
int dy;
// This function can only scale up by 2 times horizontally.
+ (void)src_width;
assert(src_width == ((dst_width + 1) / 2));
#ifdef HAS_SCALEUVROWUP2_LINEAR_16_SSE41
@@ -851,6 +874,7 @@
int x;
// This function can only scale up by 2 times.
+ (void)src_width;
assert(src_width == ((dst_width + 1) / 2));
assert(src_height == ((dst_height + 1) / 2));
@@ -1045,7 +1069,7 @@
// Optimized even scale down. ie 2, 4, 6, 8, 10x.
if (!(dx & 0x10000) && !(dy & 0x10000)) {
#if HAS_SCALEUVDOWN2
- if (dx == 0x20000) {
+ if (dx == 0x20000 && dy == 0x20000) {
// Optimized 1/2 downsample.
ScaleUVDown2(src_width, src_height, clip_width, clip_height,
src_stride, dst_stride, src, dst, x, dx, y, dy,
@@ -1054,7 +1078,7 @@
}
#endif
#if HAS_SCALEUVDOWN4BOX
- if (dx == 0x40000 && filtering == kFilterBox) {
+ if (dx == 0x40000 && dy == 0x40000 && filtering == kFilterBox) {
// Optimized 1/4 box downsample.
return ScaleUVDown4Box(src_width, src_height, clip_width, clip_height,
src_stride, dst_stride, src, dst, x, dx, y,
diff --git a/source/scale_win.cc b/source/scale_win.cc
index ea1f95c..32c0506 100644
--- a/source/scale_win.cc
+++ b/source/scale_win.cc
@@ -17,8 +17,8 @@
#endif
// This module is for 32 bit Visual C x86
-#if !defined(LIBYUV_DISABLE_X86) && defined(_MSC_VER) && \
- !defined(__clang__) && defined(_M_IX86)
+#if !defined(LIBYUV_DISABLE_X86) && defined(_MSC_VER) && defined(_M_IX86) && \
+ (!defined(__clang__) || defined(LIBYUV_ENABLE_ROWWIN))
// Offsets for source bytes 0 to 9
static const uvec8 kShuf0 = {0, 1, 3, 4, 5, 7, 8, 9,
diff --git a/tools_libyuv/autoroller/roll_deps.py b/tools_libyuv/autoroller/roll_deps.py
index d5c1089..472cbe6 100755
--- a/tools_libyuv/autoroller/roll_deps.py
+++ b/tools_libyuv/autoroller/roll_deps.py
@@ -20,13 +20,20 @@
import sys
import urllib.request
+def FindRootPath():
+ """Returns the absolute path to the highest level repo root.
-def FindSrcDirPath():
- """Returns the abs path to the src/ dir of the project."""
- src_dir = os.path.dirname(os.path.abspath(__file__))
- while os.path.basename(src_dir) != 'src':
- src_dir = os.path.normpath(os.path.join(src_dir, os.pardir))
- return src_dir
+ If this repo is checked out as a submodule of the chromium/src
+ superproject, this returns the superproect root. Otherwise, it returns the
+ webrtc/src repo root.
+ """
+ root_dir = os.path.dirname(os.path.abspath(__file__))
+ while os.path.basename(root_dir) not in ('src', 'chromium'):
+ par_dir = os.path.normpath(os.path.join(root_dir, os.pardir))
+ if par_dir == root_dir:
+ raise RuntimeError('Could not find the repo root.')
+ root_dir = par_dir
+ return root_dir
# Skip these dependencies (list without solution name prefix).
@@ -41,15 +48,12 @@
# but we pull it through a subtree mirror, so therefore it isn't listed in
# Chromium's deps but it is in ours.
LIBYUV_ONLY_DEPS = [
- 'src/base',
'src/build',
'src/buildtools',
'src/ios',
'src/testing',
'src/third_party',
'src/third_party/android_support_test_runner',
- 'src/third_party/bazel',
- 'src/third_party/bouncycastle',
'src/third_party/errorprone/lib',
'src/third_party/findbugs',
'src/third_party/gson',
@@ -75,8 +79,8 @@
ROLL_BRANCH_NAME = 'roll_chromium_revision'
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
-CHECKOUT_SRC_DIR = FindSrcDirPath()
-CHECKOUT_ROOT_DIR = os.path.realpath(os.path.join(CHECKOUT_SRC_DIR, os.pardir))
+CHECKOUT_ROOT_DIR = FindRootPath()
+GCLIENT_ROOT_DIR = os.path.realpath(os.path.join(CHECKOUT_ROOT_DIR, os.pardir))
# Copied from tools/android/roll/android_deps/.../BuildConfigGenerator.groovy.
ANDROID_DEPS_START = r'=== ANDROID_DEPS Generated Code Start ==='
@@ -84,172 +88,188 @@
# Location of automically gathered android deps.
ANDROID_DEPS_PATH = 'src/third_party/android_deps/'
-sys.path.append(os.path.join(CHECKOUT_SRC_DIR, 'build'))
-import find_depot_tools
+sys.path.append(os.path.join(CHECKOUT_ROOT_DIR, 'build'))
+import find_depot_tools # pylint: disable=wrong-import-position
find_depot_tools.add_depot_tools_to_path()
CLANG_UPDATE_SCRIPT_URL_PATH = 'tools/clang/scripts/update.py'
-CLANG_UPDATE_SCRIPT_LOCAL_PATH = os.path.join(CHECKOUT_SRC_DIR, 'tools',
- 'clang', 'scripts', 'update.py')
+CLANG_UPDATE_SCRIPT_LOCAL_PATH = os.path.join(
+ CHECKOUT_ROOT_DIR, 'tools', 'clang', 'scripts', 'update.py'
+)
DepsEntry = collections.namedtuple('DepsEntry', 'path url revision')
-ChangedDep = collections.namedtuple('ChangedDep',
- 'path url current_rev new_rev')
+ChangedDep = collections.namedtuple(
+ 'ChangedDep', 'path url current_rev new_rev'
+)
CipdDepsEntry = collections.namedtuple('CipdDepsEntry', 'path packages')
+GcsDepsEntry = collections.namedtuple('GcsDepsEntry', 'path bucket objects')
VersionEntry = collections.namedtuple('VersionEntry', 'version')
ChangedCipdPackage = collections.namedtuple(
- 'ChangedCipdPackage', 'path package current_version new_version')
+ 'ChangedCipdPackage', 'path package current_version new_version'
+)
ChangedVersionEntry = collections.namedtuple(
- 'ChangedVersionEntry', 'path current_version new_version')
+ 'ChangedVersionEntry', 'path current_version new_version'
+)
-ChromiumRevisionUpdate = collections.namedtuple('ChromiumRevisionUpdate',
- ('current_chromium_rev '
- 'new_chromium_rev '))
+ChromiumRevisionUpdate = collections.namedtuple(
+ 'ChromiumRevisionUpdate', 'current_chromium_rev new_chromium_rev '
+)
class RollError(Exception):
- pass
+ pass
def StrExpansion():
- return lambda str_value: str_value
+ return lambda str_value: str_value
def VarLookup(local_scope):
- return lambda var_name: local_scope['vars'][var_name]
+ return lambda var_name: local_scope['vars'][var_name]
def ParseDepsDict(deps_content):
- local_scope = {}
- global_scope = {
- 'Str': StrExpansion(),
- 'Var': VarLookup(local_scope),
- 'deps_os': {},
- }
- exec(deps_content, global_scope, local_scope)
- return local_scope
+ local_scope = {}
+ global_scope = {
+ 'Str': StrExpansion(),
+ 'Var': VarLookup(local_scope),
+ 'deps_os': {},
+ }
+ exec(deps_content, global_scope, local_scope)
+ return local_scope
def ParseLocalDepsFile(filename):
- with open(filename, 'rb') as f:
- deps_content = f.read().decode('utf-8')
- return ParseDepsDict(deps_content)
+ with open(filename, 'rb') as f:
+ deps_content = f.read().decode('utf-8')
+ return ParseDepsDict(deps_content)
def ParseCommitPosition(commit_message):
- for line in reversed(commit_message.splitlines()):
- m = COMMIT_POSITION_RE.match(line.strip())
- if m:
- return int(m.group(1))
- logging.error('Failed to parse commit position id from:\n%s\n',
- commit_message)
- sys.exit(-1)
+ for line in reversed(commit_message.splitlines()):
+ m = COMMIT_POSITION_RE.match(line.strip())
+ if m:
+ return int(m.group(1))
+ logging.error(
+ 'Failed to parse commit position id from:\n%s\n', commit_message
+ )
+ sys.exit(-1)
-def _RunCommand(command,
- working_dir=None,
- ignore_exit_code=False,
- extra_env=None,
- input_data=None):
- """Runs a command and returns the output from that command.
+def _RunCommand(
+ command,
+ working_dir=None,
+ ignore_exit_code=False,
+ extra_env=None,
+ input_data=None,
+):
+ """Runs a command and returns the output from that command.
If the command fails (exit code != 0), the function will exit the process.
Returns:
A tuple containing the stdout and stderr outputs as strings.
"""
- working_dir = working_dir or CHECKOUT_SRC_DIR
- logging.debug('CMD: %s CWD: %s', ' '.join(command), working_dir)
- env = os.environ.copy()
- if extra_env:
- assert all(isinstance(value, str) for value in extra_env.values())
- logging.debug('extra env: %s', extra_env)
- env.update(extra_env)
- p = subprocess.Popen(command,
- stdin=subprocess.PIPE,
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE,
- env=env,
- cwd=working_dir,
- universal_newlines=True)
- std_output, err_output = p.communicate(input_data)
- p.stdout.close()
- p.stderr.close()
- if not ignore_exit_code and p.returncode != 0:
- logging.error('Command failed: %s\n'
- 'stdout:\n%s\n'
- 'stderr:\n%s\n', ' '.join(command), std_output, err_output)
- sys.exit(p.returncode)
- return std_output, err_output
+ working_dir = working_dir or CHECKOUT_ROOT_DIR
+ logging.debug('CMD: %s CWD: %s', ' '.join(command), working_dir)
+ env = os.environ.copy()
+ if extra_env:
+ assert all(isinstance(value, str) for value in extra_env.values())
+ logging.debug('extra env: %s', extra_env)
+ env.update(extra_env)
+ p = subprocess.Popen(
+ command,
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ env=env,
+ cwd=working_dir,
+ universal_newlines=True,
+ )
+ std_output, err_output = p.communicate(input_data)
+ p.stdout.close()
+ p.stderr.close()
+ if not ignore_exit_code and p.returncode != 0:
+ logging.error(
+ 'Command failed: %s\nstdout:\n%s\nstderr:\n%s\n',
+ ' '.join(command),
+ std_output,
+ err_output,
+ )
+ sys.exit(p.returncode)
+ return std_output, err_output
def _GetBranches():
- """Returns a tuple of active,branches.
+ """Returns a tuple of active,branches.
The 'active' is the name of the currently active branch and 'branches' is a
list of all branches.
"""
- lines = _RunCommand(['git', 'branch'])[0].split('\n')
- branches = []
- active = ''
- for line in lines:
- if '*' in line:
- # The assumption is that the first char will always be the '*'.
- active = line[1:].strip()
- branches.append(active)
- else:
- branch = line.strip()
- if branch:
- branches.append(branch)
- return active, branches
+ lines = _RunCommand(['git', 'branch'])[0].split('\n')
+ branches = []
+ active = ''
+ for line in lines:
+ if '*' in line:
+ # The assumption is that the first char will always be the '*'.
+ active = line[1:].strip()
+ branches.append(active)
+ else:
+ branch = line.strip()
+ if branch:
+ branches.append(branch)
+ return active, branches
def _ReadGitilesContent(url):
- # Download and decode BASE64 content until
- # https://code.google.com/p/gitiles/issues/detail?id=7 is fixed.
- base64_content = ReadUrlContent(url + '?format=TEXT')
- return base64.b64decode(base64_content[0]).decode('utf-8')
+ # Download and decode BASE64 content until
+ # https://code.google.com/p/gitiles/issues/detail?id=7 is fixed.
+ base64_content = ReadUrlContent(url + '?format=TEXT')
+ return base64.b64decode(base64_content[0]).decode('utf-8')
def ReadRemoteCrFile(path_below_src, revision):
- """Reads a remote Chromium file of a specific revision.
+ """Reads a remote Chromium file of a specific revision.
Args:
path_below_src: A path to the target file relative to src dir.
revision: Revision to read.
+
Returns:
A string with file content.
"""
- return _ReadGitilesContent(CHROMIUM_FILE_TEMPLATE %
- (revision, path_below_src))
+ return _ReadGitilesContent(
+ CHROMIUM_FILE_TEMPLATE % (revision, path_below_src)
+ )
def ReadRemoteCrCommit(revision):
- """Reads a remote Chromium commit message. Returns a string."""
- return _ReadGitilesContent(CHROMIUM_COMMIT_TEMPLATE % revision)
+ """Reads a remote Chromium commit message. Returns a string."""
+ return _ReadGitilesContent(CHROMIUM_COMMIT_TEMPLATE % revision)
def ReadUrlContent(url):
- """Connect to a remote host and read the contents.
+ """Connect to a remote host and read the contents.
Args:
url: URL to connect to.
+
Returns:
A list of lines.
"""
- conn = urllib.request.urlopen(url)
- try:
- return conn.readlines()
- except IOError as e:
- logging.exception('Error connecting to %s. Error: %s', url, e)
- raise
- finally:
- conn.close()
+ conn = urllib.request.urlopen(url)
+ try:
+ return conn.readlines()
+ except IOError as e:
+ logging.exception('Error connecting to %s. Error: %s', url, e)
+ raise
+ finally:
+ conn.close()
def GetMatchingDepsEntries(depsentry_dict, dir_path):
- """Gets all deps entries matching the provided path.
+ """Gets all deps entries matching the provided path.
This list may contain more than one DepsEntry object.
Example: dir_path='src/testing' would give results containing both
@@ -261,94 +281,104 @@
Returns:
A list of DepsEntry objects.
"""
- result = []
- for path, depsentry in depsentry_dict.items():
- if path == dir_path:
- result.append(depsentry)
- else:
- parts = path.split('/')
- if all(part == parts[i] for i, part in enumerate(dir_path.split('/'))):
- result.append(depsentry)
- return result
+ result = []
+ for path, depsentry in depsentry_dict.items():
+ if path == dir_path:
+ result.append(depsentry)
+ else:
+ parts = path.split('/')
+ if all(
+ part == parts[i] for i, part in enumerate(dir_path.split('/'))
+ ):
+ result.append(depsentry)
+ return result
def BuildDepsentryDict(deps_dict):
- """Builds a dict of paths to DepsEntry objects from a raw deps dict."""
- result = {}
+ """Builds a dict of paths to DepsEntry objects from a raw deps dict."""
+ result = {}
- def AddDepsEntries(deps_subdict):
- for path, dep in deps_subdict.items():
- if path in result:
- continue
- if not isinstance(dep, dict):
- dep = {'url': dep}
- if dep.get('dep_type') == 'cipd':
- result[path] = CipdDepsEntry(path, dep['packages'])
- else:
- if '@' not in dep['url']:
- continue
- url, revision = dep['url'].split('@')
- result[path] = DepsEntry(path, url, revision)
+ def AddDepsEntries(deps_subdict):
+ for path, dep in deps_subdict.items():
+ if path in result:
+ continue
+ if not isinstance(dep, dict):
+ dep = {'url': dep}
+ if dep.get('dep_type') == 'cipd':
+ result[path] = CipdDepsEntry(path, dep['packages'])
+ elif dep.get('dep_type') == 'gcs':
+ result[path] = GcsDepsEntry(path, dep['bucket'],
+ dep['objects'])
+ else:
+ if '@' not in dep['url']:
+ url, revision = dep['url'], 'HEAD'
+ else:
+ url, revision = dep['url'].split('@')
+ result[path] = DepsEntry(path, url, revision)
- def AddVersionEntry(vars_subdict):
- for key, value in vars_subdict.items():
- if key in result:
- continue
- if not key.endswith('_version'):
- continue
- key = re.sub('_version$', '', key)
- result[key] = VersionEntry(value)
+ def AddVersionEntry(vars_subdict):
+ for key, value in vars_subdict.items():
+ if key in result:
+ continue
+ if not key.endswith('_version'):
+ continue
+ key = re.sub('_version$', '', key)
+ result[key] = VersionEntry(value)
- AddDepsEntries(deps_dict['deps'])
- for deps_os in ['win', 'mac', 'unix', 'android', 'ios', 'unix']:
- AddDepsEntries(deps_dict.get('deps_os', {}).get(deps_os, {}))
- AddVersionEntry(deps_dict.get('vars', {}))
- return result
+ AddDepsEntries(deps_dict['deps'])
+ for deps_os in ['win', 'mac', 'unix', 'android', 'ios', 'unix']:
+ AddDepsEntries(deps_dict.get('deps_os', {}).get(deps_os, {}))
+ AddVersionEntry(deps_dict.get('vars', {}))
+ return result
def _FindChangedCipdPackages(path, old_pkgs, new_pkgs):
- old_pkgs_names = {p['package'] for p in old_pkgs}
- new_pkgs_names = {p['package'] for p in new_pkgs}
- pkgs_equal = (old_pkgs_names == new_pkgs_names)
- added_pkgs = [p for p in new_pkgs_names if p not in old_pkgs_names]
- removed_pkgs = [p for p in old_pkgs_names if p not in new_pkgs_names]
+ old_pkgs_names = {p['package'] for p in old_pkgs}
+ new_pkgs_names = {p['package'] for p in new_pkgs}
+ pkgs_equal = old_pkgs_names == new_pkgs_names
+ added_pkgs = [p for p in new_pkgs_names if p not in old_pkgs_names]
+ removed_pkgs = [p for p in old_pkgs_names if p not in new_pkgs_names]
- assert pkgs_equal, ('Old: %s\n New: %s.\nYou need to do a manual roll '
- 'and remove/add entries in DEPS so the old and new '
- 'list match.\nMost likely, you should add \"%s\" and '
- 'remove \"%s\"' %
- (old_pkgs, new_pkgs, added_pkgs, removed_pkgs))
+ assert pkgs_equal, (
+ 'Old: %s\n New: %s.\nYou need to do a manual roll '
+ 'and remove/add entries in DEPS so the old and new '
+ 'list match.\nMost likely, you should add "%s" and '
+ 'remove "%s"' % (old_pkgs, new_pkgs, added_pkgs, removed_pkgs)
+ )
- for old_pkg in old_pkgs:
- for new_pkg in new_pkgs:
- old_version = old_pkg['version']
- new_version = new_pkg['version']
- if (old_pkg['package'] == new_pkg['package']
- and old_version != new_version):
- logging.debug('Roll dependency %s to %s', path, new_version)
- yield ChangedCipdPackage(path, old_pkg['package'], old_version,
- new_version)
+ for old_pkg in old_pkgs:
+ for new_pkg in new_pkgs:
+ old_version = old_pkg['version']
+ new_version = new_pkg['version']
+ if (
+ old_pkg['package'] == new_pkg['package']
+ and old_version != new_version
+ ):
+ logging.debug('Roll dependency %s to %s', path, new_version)
+ yield ChangedCipdPackage(
+ path, old_pkg['package'], old_version, new_version
+ )
def _FindChangedVars(name, old_version, new_version):
- if old_version != new_version:
- logging.debug('Roll dependency %s to %s', name, new_version)
- yield ChangedVersionEntry(name, old_version, new_version)
+ if old_version != new_version:
+ logging.debug('Roll dependency %s to %s', name, new_version)
+ yield ChangedVersionEntry(name, old_version, new_version)
def _FindNewDeps(old, new):
- """ Gather dependencies only in `new` and return corresponding paths. """
- old_entries = set(BuildDepsentryDict(old))
- new_entries = set(BuildDepsentryDict(new))
- return [
- path for path in new_entries - old_entries
- if path not in DONT_AUTOROLL_THESE
- ]
+ """Gather dependencies only in `new` and return corresponding paths."""
+ old_entries = set(BuildDepsentryDict(old))
+ new_entries = set(BuildDepsentryDict(new))
+ return [
+ path
+ for path in new_entries - old_entries
+ if path not in DONT_AUTOROLL_THESE
+ ]
def FindAddedDeps(libyuv_deps, new_cr_deps):
- """
- Calculate new deps entries of interest.
+ """Calculate new deps entries of interest.
Ideally, that would mean: only appearing in chromium DEPS
but transitively used in LibYUV.
@@ -368,19 +398,18 @@
A list of paths added dependencies sitting in `ANDROID_DEPS_PATH`.
A list of paths for other added dependencies.
"""
- all_added_deps = _FindNewDeps(libyuv_deps, new_cr_deps)
- generated_android_deps = [
- path for path in all_added_deps if path.startswith(ANDROID_DEPS_PATH)
- ]
- other_deps = [
- path for path in all_added_deps if path not in generated_android_deps
- ]
- return generated_android_deps, other_deps
+ all_added_deps = _FindNewDeps(libyuv_deps, new_cr_deps)
+ generated_android_deps = [
+ path for path in all_added_deps if path.startswith(ANDROID_DEPS_PATH)
+ ]
+ other_deps = [
+ path for path in all_added_deps if path not in generated_android_deps
+ ]
+ return generated_android_deps, other_deps
def FindRemovedDeps(libyuv_deps, new_cr_deps):
- """
- Calculate obsolete deps entries.
+ """Calculate obsolete deps entries.
Ideally, that would mean: no more appearing in chromium DEPS
and not used in LibYUV.
@@ -402,20 +431,24 @@
A list of paths of dependencies removed from `ANDROID_DEPS_PATH`.
A list of paths of unexpected disappearing dependencies.
"""
- all_removed_deps = _FindNewDeps(new_cr_deps, libyuv_deps)
- generated_android_deps = sorted(
- [path for path in all_removed_deps if path.startswith(ANDROID_DEPS_PATH)])
- # Webrtc-only dependencies are handled in CalculateChangedDeps.
- other_deps = sorted([
- path for path in all_removed_deps
- if path not in generated_android_deps and path not in LIBYUV_ONLY_DEPS
- ])
- return generated_android_deps, other_deps
+ all_removed_deps = _FindNewDeps(new_cr_deps, libyuv_deps)
+ generated_android_deps = sorted([
+ path for path in all_removed_deps if path.startswith(ANDROID_DEPS_PATH)
+ ])
+ # Webrtc-only dependencies are handled in CalculateChangedDeps.
+ other_deps = sorted([
+ path
+ for path in all_removed_deps
+ if path not in generated_android_deps and path not in LIBYUV_ONLY_DEPS
+ ])
+ return generated_android_deps, other_deps
def CalculateChangedDeps(libyuv_deps, new_cr_deps):
- """
- Calculate changed deps entries based on entries defined in the LibYUV DEPS
+ """Calculate changed deps entries based on entries defined in the LibYUV
+
+ DEPS
+
file:
- If a shared dependency with the Chromium DEPS file: roll it to the same
revision as Chromium (i.e. entry in the new_cr_deps dict)
@@ -428,254 +461,298 @@
Returns:
A list of ChangedDep objects representing the changed deps.
"""
- result = []
- libyuv_entries = BuildDepsentryDict(libyuv_deps)
- new_cr_entries = BuildDepsentryDict(new_cr_deps)
- for path, libyuv_deps_entry in libyuv_entries.items():
- if path in DONT_AUTOROLL_THESE:
- continue
- cr_deps_entry = new_cr_entries.get(path)
- if cr_deps_entry:
- assert type(cr_deps_entry) is type(libyuv_deps_entry)
+ result = []
+ libyuv_entries = BuildDepsentryDict(libyuv_deps)
+ new_cr_entries = BuildDepsentryDict(new_cr_deps)
+ for path, libyuv_deps_entry in libyuv_entries.items():
+ if path in DONT_AUTOROLL_THESE:
+ continue
+ cr_deps_entry = new_cr_entries.get(path)
+ if cr_deps_entry:
+ assert type(cr_deps_entry) is type(libyuv_deps_entry)
- if isinstance(cr_deps_entry, CipdDepsEntry):
- result.extend(
- _FindChangedCipdPackages(path, libyuv_deps_entry.packages,
- cr_deps_entry.packages))
- continue
+ if isinstance(cr_deps_entry, CipdDepsEntry):
+ result.extend(
+ _FindChangedCipdPackages(path, libyuv_deps_entry.packages,
+ cr_deps_entry.packages))
+ continue
- if isinstance(cr_deps_entry, VersionEntry):
- result.extend(
- _FindChangedVars(path, libyuv_deps_entry.version,
- cr_deps_entry.version))
- continue
+ if isinstance(cr_deps_entry, GcsDepsEntry):
+ result.extend(
+ _FindChangedVars(
+ path, ','.join(x['object_name']
+ for x in libyuv_deps_entry.objects),
+ ','.join(x['object_name']
+ for x in cr_deps_entry.objects)))
+ continue
- # Use the revision from Chromium's DEPS file.
- new_rev = cr_deps_entry.revision
- assert libyuv_deps_entry.url == cr_deps_entry.url, (
- 'LibYUV DEPS entry %s has a different URL %s than Chromium %s.' %
- (path, libyuv_deps_entry.url, cr_deps_entry.url))
- else:
- if isinstance(libyuv_deps_entry, DepsEntry):
- # Use the HEAD of the deps repo.
- stdout, _ = _RunCommand(
- ['git', 'ls-remote', libyuv_deps_entry.url, 'HEAD'])
- new_rev = stdout.strip().split('\t')[0]
- else:
- # The dependency has been removed from chromium.
- # This is handled by FindRemovedDeps.
- continue
+ if isinstance(cr_deps_entry, VersionEntry):
+ result.extend(
+ _FindChangedVars(
+ path, libyuv_deps_entry.version, cr_deps_entry.version
+ )
+ )
+ continue
- # Check if an update is necessary.
- if libyuv_deps_entry.revision != new_rev:
- logging.debug('Roll dependency %s to %s', path, new_rev)
- result.append(
- ChangedDep(path, libyuv_deps_entry.url, libyuv_deps_entry.revision,
- new_rev))
- return sorted(result)
+ # Use the revision from Chromium's DEPS file.
+ new_rev = cr_deps_entry.revision
+ assert libyuv_deps_entry.url == cr_deps_entry.url, (
+ 'LibYUV DEPS entry %s has a different URL %s than Chromium %s.'
+ % (
+ path,
+ libyuv_deps_entry.url,
+ cr_deps_entry.url,
+ )
+ )
+ else:
+ if isinstance(libyuv_deps_entry, DepsEntry):
+ # Use the HEAD of the deps repo.
+ stdout, _ = _RunCommand(
+ ['git', 'ls-remote', libyuv_deps_entry.url, 'HEAD']
+ )
+ new_rev = stdout.strip().split('\t')[0]
+ else:
+ # The dependency has been removed from chromium.
+ # This is handled by FindRemovedDeps.
+ continue
+
+ # Check if an update is necessary.
+ if libyuv_deps_entry.revision != new_rev:
+ logging.debug('Roll dependency %s to %s', path, new_rev)
+ result.append(
+ ChangedDep(
+ path,
+ libyuv_deps_entry.url,
+ libyuv_deps_entry.revision,
+ new_rev,
+ )
+ )
+ return sorted(result)
def CalculateChangedClang(new_cr_rev):
- def GetClangRev(lines):
- for line in lines:
- match = CLANG_REVISION_RE.match(line)
- if match:
- return match.group(1)
- raise RollError('Could not parse Clang revision!')
+ def GetClangRev(lines):
+ for line in lines:
+ match = CLANG_REVISION_RE.match(line)
+ if match:
+ return match.group(1)
+ raise RollError('Could not parse Clang revision!')
- with open(CLANG_UPDATE_SCRIPT_LOCAL_PATH, 'r') as f:
- current_lines = f.readlines()
- current_rev = GetClangRev(current_lines)
+ with open(CLANG_UPDATE_SCRIPT_LOCAL_PATH, 'r') as f:
+ current_lines = f.readlines()
+ current_rev = GetClangRev(current_lines)
- new_clang_update_py = ReadRemoteCrFile(CLANG_UPDATE_SCRIPT_URL_PATH,
- new_cr_rev).splitlines()
- new_rev = GetClangRev(new_clang_update_py)
- return ChangedDep(CLANG_UPDATE_SCRIPT_LOCAL_PATH, None, current_rev, new_rev)
+ new_clang_update_py = ReadRemoteCrFile(
+ CLANG_UPDATE_SCRIPT_URL_PATH, new_cr_rev
+ ).splitlines()
+ new_rev = GetClangRev(new_clang_update_py)
+ return ChangedDep(
+ CLANG_UPDATE_SCRIPT_LOCAL_PATH, None, current_rev, new_rev
+ )
def GenerateCommitMessage(
- rev_update,
- current_commit_pos,
- new_commit_pos,
- changed_deps_list,
- added_deps_paths=None,
- removed_deps_paths=None,
- clang_change=None,
+ rev_update,
+ current_commit_pos,
+ new_commit_pos,
+ changed_deps_list,
+ added_deps_paths=None,
+ removed_deps_paths=None,
+ clang_change=None,
):
- current_cr_rev = rev_update.current_chromium_rev[0:10]
- new_cr_rev = rev_update.new_chromium_rev[0:10]
- rev_interval = '%s..%s' % (current_cr_rev, new_cr_rev)
- git_number_interval = '%s:%s' % (current_commit_pos, new_commit_pos)
+ current_cr_rev = rev_update.current_chromium_rev[0:10]
+ new_cr_rev = rev_update.new_chromium_rev[0:10]
+ rev_interval = '%s..%s' % (current_cr_rev, new_cr_rev)
+ git_number_interval = '%s:%s' % (current_commit_pos, new_commit_pos)
- commit_msg = [
- 'Roll chromium_revision %s (%s)\n' % (rev_interval, git_number_interval),
- 'Change log: %s' % (CHROMIUM_LOG_TEMPLATE % rev_interval),
- 'Full diff: %s\n' % (CHROMIUM_COMMIT_TEMPLATE % rev_interval)
- ]
+ commit_msg = [
+ 'Roll chromium_revision %s (%s)\n'
+ % (rev_interval, git_number_interval),
+ 'Change log: %s' % (CHROMIUM_LOG_TEMPLATE % rev_interval),
+ 'Full diff: %s\n' % (CHROMIUM_COMMIT_TEMPLATE % rev_interval),
+ ]
- def Section(adjective, deps):
- noun = 'dependency' if len(deps) == 1 else 'dependencies'
- commit_msg.append('%s %s' % (adjective, noun))
+ def Section(adjective, deps):
+ noun = 'dependency' if len(deps) == 1 else 'dependencies'
+ commit_msg.append('%s %s' % (adjective, noun))
- if changed_deps_list:
- Section('Changed', changed_deps_list)
+ if changed_deps_list:
+ Section('Changed', changed_deps_list)
- for c in changed_deps_list:
- if isinstance(c, ChangedCipdPackage):
- commit_msg.append('* %s: %s..%s' %
- (c.path, c.current_version, c.new_version))
- elif isinstance(c, ChangedVersionEntry):
- commit_msg.append('* %s_vesion: %s..%s' %
- (c.path, c.current_version, c.new_version))
- else:
- commit_msg.append('* %s: %s/+log/%s..%s' %
- (c.path, c.url, c.current_rev[0:10], c.new_rev[0:10]))
+ for c in changed_deps_list:
+ if isinstance(c, ChangedCipdPackage):
+ commit_msg.append(
+ '* %s: %s..%s' % (c.path, c.current_version, c.new_version)
+ )
+ elif isinstance(c, ChangedVersionEntry):
+ commit_msg.append(
+ '* %s_vesion: %s..%s'
+ % (c.path, c.current_version, c.new_version)
+ )
+ else:
+ commit_msg.append(
+ '* %s: %s/+log/%s..%s'
+ % (c.path, c.url, c.current_rev[0:10], c.new_rev[0:10])
+ )
- if added_deps_paths:
- Section('Added', added_deps_paths)
- commit_msg.extend('* %s' % p for p in added_deps_paths)
+ if added_deps_paths:
+ Section('Added', added_deps_paths)
+ commit_msg.extend('* %s' % p for p in added_deps_paths)
- if removed_deps_paths:
- Section('Removed', removed_deps_paths)
- commit_msg.extend('* %s' % p for p in removed_deps_paths)
+ if removed_deps_paths:
+ Section('Removed', removed_deps_paths)
+ commit_msg.extend('* %s' % p for p in removed_deps_paths)
- if any([changed_deps_list, added_deps_paths, removed_deps_paths]):
- change_url = CHROMIUM_FILE_TEMPLATE % (rev_interval, 'DEPS')
- commit_msg.append('DEPS diff: %s\n' % change_url)
- else:
- commit_msg.append('No dependencies changed.')
+ if any([changed_deps_list, added_deps_paths, removed_deps_paths]):
+ change_url = CHROMIUM_FILE_TEMPLATE % (rev_interval, 'DEPS')
+ commit_msg.append('DEPS diff: %s\n' % change_url)
+ else:
+ commit_msg.append('No dependencies changed.')
- if clang_change and clang_change.current_rev != clang_change.new_rev:
- commit_msg.append('Clang version changed %s:%s' %
- (clang_change.current_rev, clang_change.new_rev))
- change_url = CHROMIUM_FILE_TEMPLATE % (rev_interval,
- CLANG_UPDATE_SCRIPT_URL_PATH)
- commit_msg.append('Details: %s\n' % change_url)
- else:
- commit_msg.append('No update to Clang.\n')
+ if clang_change and clang_change.current_rev != clang_change.new_rev:
+ commit_msg.append(
+ 'Clang version changed %s:%s'
+ % (clang_change.current_rev, clang_change.new_rev)
+ )
+ change_url = CHROMIUM_FILE_TEMPLATE % (
+ rev_interval,
+ CLANG_UPDATE_SCRIPT_URL_PATH,
+ )
+ commit_msg.append('Details: %s\n' % change_url)
+ else:
+ commit_msg.append('No update to Clang.\n')
- commit_msg.append('BUG=None')
- return '\n'.join(commit_msg)
+ commit_msg.append('BUG=None')
+ return '\n'.join(commit_msg)
def UpdateDepsFile(deps_filename, rev_update, changed_deps, new_cr_content):
- """Update the DEPS file with the new revision."""
+ """Update the DEPS file with the new revision."""
- with open(deps_filename, 'rb') as deps_file:
- deps_content = deps_file.read().decode('utf-8')
+ with open(deps_filename, 'rb') as deps_file:
+ deps_content = deps_file.read().decode('utf-8')
- # Update the chromium_revision variable.
- deps_content = deps_content.replace(rev_update.current_chromium_rev,
- rev_update.new_chromium_rev)
+ # Update the chromium_revision variable.
+ deps_content = deps_content.replace(
+ rev_update.current_chromium_rev, rev_update.new_chromium_rev
+ )
- # Add and remove dependencies. For now: only generated android deps.
- # Since gclient cannot add or remove deps, we on the fact that
- # these android deps are located in one place we can copy/paste.
- deps_re = re.compile(ANDROID_DEPS_START + '.*' + ANDROID_DEPS_END, re.DOTALL)
- new_deps = deps_re.search(new_cr_content)
- old_deps = deps_re.search(deps_content)
- if not new_deps or not old_deps:
- faulty = 'Chromium' if not new_deps else 'LibYUV'
- raise RollError('Was expecting to find "%s" and "%s"\n'
- 'in %s DEPS' %
- (ANDROID_DEPS_START, ANDROID_DEPS_END, faulty))
- deps_content = deps_re.sub(new_deps.group(0), deps_content)
+ # Add and remove dependencies. For now: only generated android deps.
+ # Since gclient cannot add or remove deps, we on the fact that
+ # these android deps are located in one place we can copy/paste.
+ deps_re = re.compile(
+ ANDROID_DEPS_START + '.*' + ANDROID_DEPS_END, re.DOTALL
+ )
+ new_deps = deps_re.search(new_cr_content)
+ old_deps = deps_re.search(deps_content)
+ if not new_deps or not old_deps:
+ faulty = 'Chromium' if not new_deps else 'LibYUV'
+ raise RollError(
+ 'Was expecting to find "%s" and "%s"\nin %s DEPS'
+ % (ANDROID_DEPS_START, ANDROID_DEPS_END, faulty)
+ )
+ deps_content = deps_re.sub(new_deps.group(0), deps_content)
- for dep in changed_deps:
- if isinstance(dep, ChangedVersionEntry):
- deps_content = deps_content.replace(dep.current_version, dep.new_version)
+ for dep in changed_deps:
+ if isinstance(dep, ChangedVersionEntry):
+ deps_content = deps_content.replace(
+ dep.current_version, dep.new_version
+ )
- with open(deps_filename, 'wb') as deps_file:
- deps_file.write(deps_content.encode('utf-8'))
+ with open(deps_filename, 'wb') as deps_file:
+ deps_file.write(deps_content.encode('utf-8'))
- # Update each individual DEPS entry.
- for dep in changed_deps:
- # ChangedVersionEntry types are already been processed.
- if isinstance(dep, ChangedVersionEntry):
- continue
- local_dep_dir = os.path.join(CHECKOUT_ROOT_DIR, dep.path)
- if not os.path.isdir(local_dep_dir):
- raise RollError(
- 'Cannot find local directory %s. Either run\n'
- 'gclient sync --deps=all\n'
- 'or make sure the .gclient file for your solution contains all '
- 'platforms in the target_os list, i.e.\n'
- 'target_os = ["android", "unix", "mac", "ios", "win"];\n'
- 'Then run "gclient sync" again.' % local_dep_dir)
- if isinstance(dep, ChangedCipdPackage):
- package = dep.package.format() # Eliminate double curly brackets
- update = '%s:%s@%s' % (dep.path, package, dep.new_version)
- else:
- update = '%s@%s' % (dep.path, dep.new_rev)
- _RunCommand(['gclient', 'setdep', '--revision', update],
- working_dir=CHECKOUT_SRC_DIR)
+ # Update each individual DEPS entry.
+ for dep in changed_deps:
+ # ChangedVersionEntry types are already been processed.
+ if isinstance(dep, ChangedVersionEntry):
+ continue
+ local_dep_dir = os.path.join(GCLIENT_ROOT_DIR, dep.path)
+ if not os.path.isdir(local_dep_dir):
+ raise RollError(
+ 'Cannot find local directory %s. Either run\n'
+ 'gclient sync --deps=all\n'
+ 'or make sure the .gclient file for your solution contains '
+ 'all platforms in the target_os list, i.e.\n'
+ 'target_os = ["android", "unix", "mac", "ios", "win"];\n'
+ 'Then run "gclient sync" again.' % local_dep_dir
+ )
+ if isinstance(dep, ChangedCipdPackage):
+ package = dep.package.format() # Eliminate double curly brackets
+ update = '%s:%s@%s' % (dep.path, package, dep.new_version)
+ else:
+ update = '%s@%s' % (dep.path, dep.new_rev)
+ _RunCommand(
+ ['gclient', 'setdep', '--revision', update],
+ working_dir=CHECKOUT_ROOT_DIR,
+ )
def _IsTreeClean():
- stdout, _ = _RunCommand(['git', 'status', '--porcelain'])
- if len(stdout) == 0:
- return True
+ stdout, _ = _RunCommand(['git', 'status', '--porcelain'])
+ if len(stdout) == 0:
+ return True
- logging.error('Dirty/unversioned files:\n%s', stdout)
- return False
+ logging.error('Dirty/unversioned files:\n%s', stdout)
+ return False
def _EnsureUpdatedMainBranch(dry_run):
- current_branch = _RunCommand(['git', 'rev-parse', '--abbrev-ref',
- 'HEAD'])[0].splitlines()[0]
- if current_branch != 'main':
- logging.error('Please checkout the main branch and re-run this script.')
- if not dry_run:
- sys.exit(-1)
+ current_branch = _RunCommand(['git', 'rev-parse', '--abbrev-ref', 'HEAD'])[
+ 0
+ ].splitlines()[0]
+ if current_branch != 'main':
+ logging.error('Please checkout the main branch and re-run this script.') # pylint: disable=line-too-long
+ if not dry_run:
+ sys.exit(-1)
- logging.info('Updating main branch...')
- _RunCommand(['git', 'pull'])
+ logging.info('Updating main branch...')
+ _RunCommand(['git', 'pull'])
def _CreateRollBranch(dry_run):
- logging.info('Creating roll branch: %s', ROLL_BRANCH_NAME)
- if not dry_run:
- _RunCommand(['git', 'checkout', '-b', ROLL_BRANCH_NAME])
+ logging.info('Creating roll branch: %s', ROLL_BRANCH_NAME)
+ if not dry_run:
+ _RunCommand(['git', 'checkout', '-b', ROLL_BRANCH_NAME])
def _RemovePreviousRollBranch(dry_run):
- active_branch, branches = _GetBranches()
- if active_branch == ROLL_BRANCH_NAME:
- active_branch = 'main'
- if ROLL_BRANCH_NAME in branches:
- logging.info('Removing previous roll branch (%s)', ROLL_BRANCH_NAME)
- if not dry_run:
- _RunCommand(['git', 'checkout', active_branch])
- _RunCommand(['git', 'branch', '-D', ROLL_BRANCH_NAME])
+ active_branch, branches = _GetBranches()
+ if active_branch == ROLL_BRANCH_NAME:
+ active_branch = 'main'
+ if ROLL_BRANCH_NAME in branches:
+ logging.info('Removing previous roll branch (%s)', ROLL_BRANCH_NAME)
+ if not dry_run:
+ _RunCommand(['git', 'checkout', active_branch])
+ _RunCommand(['git', 'branch', '-D', ROLL_BRANCH_NAME])
def _LocalCommit(commit_msg, dry_run):
- logging.info('Committing changes locally.')
- if not dry_run:
- _RunCommand(['git', 'add', '--update', '.'])
- _RunCommand(['git', 'commit', '-m', commit_msg])
+ logging.info('Committing changes locally.')
+ if not dry_run:
+ _RunCommand(['git', 'add', '--update', '.'])
+ _RunCommand(['git', 'commit', '-m', commit_msg])
def ChooseCQMode(skip_cq, cq_over, current_commit_pos, new_commit_pos):
- if skip_cq:
- return 0
- if (new_commit_pos - current_commit_pos) < cq_over:
- return 1
- return 2
+ if skip_cq:
+ return 0
+ if (new_commit_pos - current_commit_pos) < cq_over:
+ return 1
+ return 2
def _GetCcRecipients(changed_deps_list):
- """Returns a list of emails to notify based on the changed deps list.
- """
- cc_recipients = []
- for c in changed_deps_list:
- pass
- return cc_recipients
+ """Returns a list of emails to notify based on the changed deps list."""
+ cc_recipients = []
+ for _ in changed_deps_list:
+ pass
+ return cc_recipients
def _UploadCL(commit_queue_mode, add_cc=None):
- """Upload the committed changes as a changelist to Gerrit.
+ """Upload the committed changes as a changelist to Gerrit.
commit_queue_mode:
- 2: Submit to commit queue.
@@ -684,139 +761,171 @@
add_cc: A list of email addresses to add as CC recipients.
"""
- cc_recipients = []
- if add_cc:
- cc_recipients.extend(add_cc)
- cmd = ['git', 'cl', 'upload', '--force', '--bypass-hooks']
- if commit_queue_mode >= 2:
- logging.info('Sending the CL to the CQ...')
- cmd.extend(['-o', 'label=Bot-Commit+1'])
- cmd.extend(['-o', 'label=Commit-Queue+2'])
- cmd.extend(['--send-mail', '--cc', ','.join(cc_recipients)])
- elif commit_queue_mode >= 1:
- logging.info('Starting CQ dry run...')
- cmd.extend(['-o', 'label=Commit-Queue+1'])
- extra_env = {
- 'EDITOR': 'true',
- 'SKIP_GCE_AUTH_FOR_GIT': '1',
- }
- stdout, stderr = _RunCommand(cmd, extra_env=extra_env)
- logging.debug('Output from "git cl upload":\nstdout:\n%s\n\nstderr:\n%s',
- stdout, stderr)
+ cc_recipients = []
+ if add_cc:
+ cc_recipients.extend(add_cc)
+ cmd = ['git', 'cl', 'upload', '--force', '--bypass-hooks']
+ if commit_queue_mode >= 2:
+ logging.info('Sending the CL to the CQ...')
+ cmd.extend(['-o', 'label=Bot-Commit+1'])
+ cmd.extend(['-o', 'label=Commit-Queue+2'])
+ cmd.extend(['--send-mail', '--cc', ','.join(cc_recipients)])
+ elif commit_queue_mode >= 1:
+ logging.info('Starting CQ dry run...')
+ cmd.extend(['-o', 'label=Commit-Queue+1'])
+ extra_env = {
+ 'EDITOR': 'true',
+ 'SKIP_GCE_AUTH_FOR_GIT': '1',
+ }
+ stdout, stderr = _RunCommand(cmd, extra_env=extra_env)
+ logging.debug(
+ 'Output from "git cl upload":\nstdout:\n%s\n\nstderr:\n%s',
+ stdout,
+ stderr,
+ )
def GetRollRevisionRanges(opts, libyuv_deps):
- current_cr_rev = libyuv_deps['vars']['chromium_revision']
- new_cr_rev = opts.revision
- if not new_cr_rev:
- stdout, _ = _RunCommand(['git', 'ls-remote', CHROMIUM_SRC_URL, 'HEAD'])
- head_rev = stdout.strip().split('\t')[0]
- logging.info('No revision specified. Using HEAD: %s', head_rev)
- new_cr_rev = head_rev
+ current_cr_rev = libyuv_deps['vars']['chromium_revision']
+ new_cr_rev = opts.revision
+ if not new_cr_rev:
+ stdout, _ = _RunCommand(['git', 'ls-remote', CHROMIUM_SRC_URL, 'HEAD'])
+ head_rev = stdout.strip().split('\t')[0]
+ logging.info('No revision specified. Using HEAD: %s', head_rev)
+ new_cr_rev = head_rev
- return ChromiumRevisionUpdate(current_cr_rev, new_cr_rev)
+ return ChromiumRevisionUpdate(current_cr_rev, new_cr_rev)
def main():
- p = argparse.ArgumentParser()
- p.add_argument('--clean',
- action='store_true',
- default=False,
- help='Removes any previous local roll branch.')
- p.add_argument('-r',
- '--revision',
- help=('Chromium Git revision to roll to. Defaults to the '
- 'Chromium HEAD revision if omitted.'))
- p.add_argument('--dry-run',
- action='store_true',
- default=False,
- help=('Calculate changes and modify DEPS, but don\'t create '
- 'any local branch, commit, upload CL or send any '
- 'tryjobs.'))
- p.add_argument('-i',
- '--ignore-unclean-workdir',
- action='store_true',
- default=False,
- help=('Ignore if the current branch is not main or if there '
- 'are uncommitted changes (default: %(default)s).'))
- grp = p.add_mutually_exclusive_group()
- grp.add_argument('--skip-cq',
- action='store_true',
- default=False,
- help='Skip sending the CL to the CQ (default: %(default)s)')
- grp.add_argument('--cq-over',
- type=int,
- default=1,
- help=('Commit queue dry run if the revision difference '
- 'is below this number (default: %(default)s)'))
- p.add_argument('-v',
- '--verbose',
- action='store_true',
- default=False,
- help='Be extra verbose in printing of log messages.')
- opts = p.parse_args()
+ p = argparse.ArgumentParser()
+ p.add_argument(
+ '--clean',
+ action='store_true',
+ default=False,
+ help='Removes any previous local roll branch.',
+ )
+ p.add_argument(
+ '-r',
+ '--revision',
+ help=(
+ 'Chromium Git revision to roll to. Defaults to the '
+ 'Chromium HEAD revision if omitted.'
+ ),
+ )
+ p.add_argument(
+ '--dry-run',
+ action='store_true',
+ default=False,
+ help=(
+ "Calculate changes and modify DEPS, but don't create "
+ 'any local branch, commit, upload CL or send any '
+ 'tryjobs.'
+ ),
+ )
+ p.add_argument(
+ '-i',
+ '--ignore-unclean-workdir',
+ action='store_true',
+ default=False,
+ help=(
+ 'Ignore if the current branch is not main or if there '
+ 'are uncommitted changes (default: %(default)s).'
+ ),
+ )
+ grp = p.add_mutually_exclusive_group()
+ grp.add_argument(
+ '--skip-cq',
+ action='store_true',
+ default=False,
+ help='Skip sending the CL to the CQ (default: %(default)s)',
+ )
+ grp.add_argument(
+ '--cq-over',
+ type=int,
+ default=1,
+ help=(
+ 'Commit queue dry run if the revision difference '
+ 'is below this number (default: %(default)s)'
+ ),
+ )
+ p.add_argument(
+ '-v',
+ '--verbose',
+ action='store_true',
+ default=False,
+ help='Be extra verbose in printing of log messages.',
+ )
+ opts = p.parse_args()
- if opts.verbose:
- logging.basicConfig(level=logging.DEBUG)
- else:
- logging.basicConfig(level=logging.INFO)
+ if opts.verbose:
+ logging.basicConfig(level=logging.DEBUG)
+ else:
+ logging.basicConfig(level=logging.INFO)
- if not opts.ignore_unclean_workdir and not _IsTreeClean():
- logging.error('Please clean your local checkout first.')
- return 1
+ if not opts.ignore_unclean_workdir and not _IsTreeClean():
+ logging.error('Please clean your local checkout first.')
+ return 1
- if opts.clean:
- _RemovePreviousRollBranch(opts.dry_run)
+ if opts.clean:
+ _RemovePreviousRollBranch(opts.dry_run)
- if not opts.ignore_unclean_workdir:
- _EnsureUpdatedMainBranch(opts.dry_run)
+ if not opts.ignore_unclean_workdir:
+ _EnsureUpdatedMainBranch(opts.dry_run)
- deps_filename = os.path.join(CHECKOUT_SRC_DIR, 'DEPS')
- libyuv_deps = ParseLocalDepsFile(deps_filename)
+ deps_filename = os.path.join(CHECKOUT_ROOT_DIR, 'DEPS')
+ libyuv_deps = ParseLocalDepsFile(deps_filename)
- rev_update = GetRollRevisionRanges(opts, libyuv_deps)
+ rev_update = GetRollRevisionRanges(opts, libyuv_deps)
- current_commit_pos = ParseCommitPosition(
- ReadRemoteCrCommit(rev_update.current_chromium_rev))
- new_commit_pos = ParseCommitPosition(
- ReadRemoteCrCommit(rev_update.new_chromium_rev))
+ current_commit_pos = ParseCommitPosition(
+ ReadRemoteCrCommit(rev_update.current_chromium_rev)
+ )
+ new_commit_pos = ParseCommitPosition(
+ ReadRemoteCrCommit(rev_update.new_chromium_rev)
+ )
- new_cr_content = ReadRemoteCrFile('DEPS', rev_update.new_chromium_rev)
- new_cr_deps = ParseDepsDict(new_cr_content)
- changed_deps = CalculateChangedDeps(libyuv_deps, new_cr_deps)
- # Discard other deps, assumed to be chromium-only dependencies.
- new_generated_android_deps, _ = FindAddedDeps(libyuv_deps, new_cr_deps)
- removed_generated_android_deps, other_deps = FindRemovedDeps(
- libyuv_deps, new_cr_deps)
- if other_deps:
- raise RollError('LibYUV DEPS entries are missing from Chromium: %s.\n'
- 'Remove them or add them to either '
- 'LIBYUV_ONLY_DEPS or DONT_AUTOROLL_THESE.' % other_deps)
- clang_change = CalculateChangedClang(rev_update.new_chromium_rev)
- commit_msg = GenerateCommitMessage(
- rev_update,
- current_commit_pos,
- new_commit_pos,
- changed_deps,
- added_deps_paths=new_generated_android_deps,
- removed_deps_paths=removed_generated_android_deps,
- clang_change=clang_change)
- logging.debug('Commit message:\n%s', commit_msg)
+ new_cr_content = ReadRemoteCrFile('DEPS', rev_update.new_chromium_rev)
+ new_cr_deps = ParseDepsDict(new_cr_content)
+ changed_deps = CalculateChangedDeps(libyuv_deps, new_cr_deps)
+ # Discard other deps, assumed to be chromium-only dependencies.
+ new_generated_android_deps, _ = FindAddedDeps(libyuv_deps, new_cr_deps)
+ removed_generated_android_deps, other_deps = FindRemovedDeps(
+ libyuv_deps, new_cr_deps
+ )
+ if other_deps:
+ raise RollError(
+ 'LibYUV DEPS entries are missing from Chromium: %s.\n'
+ 'Remove them or add them to either '
+ 'LIBYUV_ONLY_DEPS or DONT_AUTOROLL_THESE.' % other_deps
+ )
+ clang_change = CalculateChangedClang(rev_update.new_chromium_rev)
+ commit_msg = GenerateCommitMessage(
+ rev_update,
+ current_commit_pos,
+ new_commit_pos,
+ changed_deps,
+ added_deps_paths=new_generated_android_deps,
+ removed_deps_paths=removed_generated_android_deps,
+ clang_change=clang_change,
+ )
+ logging.debug('Commit message:\n%s', commit_msg)
- _CreateRollBranch(opts.dry_run)
- if not opts.dry_run:
- UpdateDepsFile(deps_filename, rev_update, changed_deps, new_cr_content)
- if _IsTreeClean():
- logging.info("No DEPS changes detected, skipping CL creation.")
- else:
- _LocalCommit(commit_msg, opts.dry_run)
- commit_queue_mode = ChooseCQMode(opts.skip_cq, opts.cq_over,
- current_commit_pos, new_commit_pos)
- logging.info('Uploading CL...')
+ _CreateRollBranch(opts.dry_run)
if not opts.dry_run:
- _UploadCL(commit_queue_mode, _GetCcRecipients(changed_deps))
- return 0
+ UpdateDepsFile(deps_filename, rev_update, changed_deps, new_cr_content)
+ if _IsTreeClean():
+ logging.info('No DEPS changes detected, skipping CL creation.')
+ else:
+ _LocalCommit(commit_msg, opts.dry_run)
+ commit_queue_mode = ChooseCQMode(
+ opts.skip_cq, opts.cq_over, current_commit_pos, new_commit_pos
+ )
+ logging.info('Uploading CL...')
+ if not opts.dry_run:
+ _UploadCL(commit_queue_mode, _GetCcRecipients(changed_deps))
+ return 0
if __name__ == '__main__':
- sys.exit(main())
+ sys.exit(main())
diff --git a/tools_libyuv/autoroller/unittests/roll_deps_test.py b/tools_libyuv/autoroller/unittests/roll_deps_test.py
index af86bdd..bd38c94 100755
--- a/tools_libyuv/autoroller/unittests/roll_deps_test.py
+++ b/tools_libyuv/autoroller/unittests/roll_deps_test.py
@@ -16,23 +16,22 @@
import unittest
import roll_deps
-from roll_deps import CalculateChangedDeps, GetMatchingDepsEntries, \
- ParseDepsDict, ParseLocalDepsFile, UpdateDepsFile
+from roll_deps import CalculateChangedDeps, GetMatchingDepsEntries, ParseDepsDict, ParseLocalDepsFile, UpdateDepsFile # pylint: disable=line-too-long
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
PARENT_DIR = os.path.join(SCRIPT_DIR, os.pardir)
sys.path.append(PARENT_DIR)
TEST_DATA_VARS = {
- 'chromium_git': 'https://chromium.googlesource.com',
- 'chromium_revision': '1b9c098a08e40114e44b6c1ec33ddf95c40b901d',
+ 'chromium_git': 'https://chromium.googlesource.com',
+ 'chromium_revision': '1b9c098a08e40114e44b6c1ec33ddf95c40b901d',
}
DEPS_ENTRIES = {
- 'src/build': 'https://build.com',
- 'src/buildtools': 'https://buildtools.com',
- 'src/testing/gtest': 'https://gtest.com',
- 'src/testing/gmock': 'https://gmock.com',
+ 'src/build': 'https://build.com',
+ 'src/buildtools': 'https://buildtools.com',
+ 'src/testing/gtest': 'https://gtest.com',
+ 'src/testing/gmock': 'https://gmock.com',
}
BUILD_OLD_REV = '52f7afeca991d96d68cf0507e20dbdd5b845691f'
@@ -42,107 +41,124 @@
class TestError(Exception):
- pass
+ pass
-class FakeCmd():
- def __init__(self):
- self.expectations = []
+class FakeCmd:
- def add_expectation(self, *args, **kwargs):
- returns = kwargs.pop('_returns', None)
- self.expectations.append((args, kwargs, returns))
+ def __init__(self):
+ self.expectations = []
- def __call__(self, *args, **kwargs):
- if not self.expectations:
- raise TestError('Got unexpected\n%s\n%s' % (args, kwargs))
- exp_args, exp_kwargs, exp_returns = self.expectations.pop(0)
- if args != exp_args or kwargs != exp_kwargs:
- message = 'Expected:\n args: %s\n kwargs: %s\n' % (exp_args, exp_kwargs)
- message += 'Got:\n args: %s\n kwargs: %s\n' % (args, kwargs)
- raise TestError(message)
- return exp_returns
+ def add_expectation(self, *args, **kwargs):
+ returns = kwargs.pop('_returns', None)
+ self.expectations.append((args, kwargs, returns))
+
+ def __call__(self, *args, **kwargs):
+ if not self.expectations:
+ raise TestError('Got unexpected\n%s\n%s' % (args, kwargs))
+ exp_args, exp_kwargs, exp_returns = self.expectations.pop(0)
+ if args != exp_args or kwargs != exp_kwargs:
+ message = 'Expected:\n args: %s\n kwargs: %s\n' % (
+ exp_args,
+ exp_kwargs,
+ )
+ message += 'Got:\n args: %s\n kwargs: %s\n' % (args, kwargs)
+ raise TestError(message)
+ return exp_returns
class TestRollChromiumRevision(unittest.TestCase):
- def setUp(self):
- self._output_dir = tempfile.mkdtemp()
- for test_file in glob.glob(os.path.join(SCRIPT_DIR, 'testdata', '*')):
- shutil.copy(test_file, self._output_dir)
- self._libyuv_depsfile = os.path.join(self._output_dir, 'DEPS')
- self._old_cr_depsfile = os.path.join(self._output_dir, 'DEPS.chromium.old')
- self._new_cr_depsfile = os.path.join(self._output_dir, 'DEPS.chromium.new')
- self.fake = FakeCmd()
- self.old_RunCommand = getattr(roll_deps, '_RunCommand')
- setattr(roll_deps, '_RunCommand', self.fake)
+ def setUp(self):
+ self._output_dir = tempfile.mkdtemp()
+ for test_file in glob.glob(os.path.join(SCRIPT_DIR, 'testdata', '*')):
+ shutil.copy(test_file, self._output_dir)
+ self._libyuv_depsfile = os.path.join(self._output_dir, 'DEPS')
+ self._old_cr_depsfile = os.path.join(
+ self._output_dir, 'DEPS.chromium.old'
+ )
+ self._new_cr_depsfile = os.path.join(
+ self._output_dir, 'DEPS.chromium.new'
+ )
- def tearDown(self):
- shutil.rmtree(self._output_dir, ignore_errors=True)
- self.assertEqual(self.fake.expectations, [])
- setattr(roll_deps, '_RunCommand', self.old_RunCommand)
+ self.fake = FakeCmd()
+ self.old_RunCommand = getattr(roll_deps, '_RunCommand')
+ setattr(roll_deps, '_RunCommand', self.fake)
- def testVarLookup(self):
- local_scope = {'foo': 'wrong', 'vars': {'foo': 'bar'}}
- lookup = roll_deps.VarLookup(local_scope)
- self.assertEqual(lookup('foo'), 'bar')
+ def tearDown(self):
+ shutil.rmtree(self._output_dir, ignore_errors=True)
+ self.assertEqual(self.fake.expectations, [])
+ setattr(roll_deps, '_RunCommand', self.old_RunCommand)
- def testUpdateDepsFile(self):
- new_rev = 'aaaaabbbbbcccccdddddeeeeefffff0000011111'
+ def testVarLookup(self):
+ local_scope = {'foo': 'wrong', 'vars': {'foo': 'bar'}}
+ lookup = roll_deps.VarLookup(local_scope)
+ self.assertEqual(lookup('foo'), 'bar')
- current_rev = TEST_DATA_VARS['chromium_revision']
- UpdateDepsFile(self._libyuv_depsfile, current_rev, new_rev, [])
- with open(self._libyuv_depsfile, 'r') as deps_file:
- deps_contents = deps_file.read()
- self.assertTrue(new_rev in deps_contents,
- 'Failed to find %s in\n%s' % (new_rev, deps_contents))
+ def testUpdateDepsFile(self):
+ new_rev = 'aaaaabbbbbcccccdddddeeeeefffff0000011111'
- def testParseDepsDict(self):
- with open(self._libyuv_depsfile, 'r') as deps_file:
- deps_contents = deps_file.read()
- local_scope = ParseDepsDict(deps_contents)
- vars_dict = local_scope['vars']
+ current_rev = TEST_DATA_VARS['chromium_revision']
+ UpdateDepsFile(self._libyuv_depsfile, current_rev, new_rev, [])
+ with open(self._libyuv_depsfile, 'r') as deps_file:
+ deps_contents = deps_file.read()
+ self.assertTrue(
+ new_rev in deps_contents,
+ 'Failed to find %s in\n%s' % (new_rev, deps_contents),
+ )
- def assertVar(variable_name):
- self.assertEqual(vars_dict[variable_name], TEST_DATA_VARS[variable_name])
- assertVar('chromium_git')
- assertVar('chromium_revision')
- self.assertEqual(len(local_scope['deps']), 3)
+ def testParseDepsDict(self):
+ with open(self._libyuv_depsfile, 'r') as deps_file:
+ deps_contents = deps_file.read()
+ local_scope = ParseDepsDict(deps_contents)
+ vars_dict = local_scope['vars']
- def testGetMatchingDepsEntriesReturnsPathInSimpleCase(self):
- entries = GetMatchingDepsEntries(DEPS_ENTRIES, 'src/testing/gtest')
- self.assertEqual(len(entries), 1)
- self.assertEqual(entries[0], DEPS_ENTRIES['src/testing/gtest'])
+ def assertVar(variable_name):
+ self.assertEqual(
+ vars_dict[variable_name], TEST_DATA_VARS[variable_name]
+ )
- def testGetMatchingDepsEntriesHandlesSimilarStartingPaths(self):
- entries = GetMatchingDepsEntries(DEPS_ENTRIES, 'src/testing')
- self.assertEqual(len(entries), 2)
+ assertVar('chromium_git')
+ assertVar('chromium_revision')
+ self.assertEqual(len(local_scope['deps']), 3)
- def testGetMatchingDepsEntriesHandlesTwoPathsWithIdenticalFirstParts(self):
- entries = GetMatchingDepsEntries(DEPS_ENTRIES, 'src/build')
- self.assertEqual(len(entries), 1)
- self.assertEqual(entries[0], DEPS_ENTRIES['src/build'])
+ def testGetMatchingDepsEntriesReturnsPathInSimpleCase(self):
+ entries = GetMatchingDepsEntries(DEPS_ENTRIES, 'src/testing/gtest')
+ self.assertEqual(len(entries), 1)
+ self.assertEqual(entries[0], DEPS_ENTRIES['src/testing/gtest'])
- def testCalculateChangedDeps(self):
- _SetupGitLsRemoteCall(self.fake,
- 'https://chromium.googlesource.com/chromium/src/build', BUILD_NEW_REV)
- libyuv_deps = ParseLocalDepsFile(self._libyuv_depsfile)
- new_cr_deps = ParseLocalDepsFile(self._new_cr_depsfile)
- changed_deps = CalculateChangedDeps(libyuv_deps, new_cr_deps)
- self.assertEqual(len(changed_deps), 2)
- self.assertEqual(changed_deps[0].path, 'src/build')
- self.assertEqual(changed_deps[0].current_rev, BUILD_OLD_REV)
- self.assertEqual(changed_deps[0].new_rev, BUILD_NEW_REV)
+ def testGetMatchingDepsEntriesHandlesSimilarStartingPaths(self):
+ entries = GetMatchingDepsEntries(DEPS_ENTRIES, 'src/testing')
+ self.assertEqual(len(entries), 2)
- self.assertEqual(changed_deps[1].path, 'src/buildtools')
- self.assertEqual(changed_deps[1].current_rev, BUILDTOOLS_OLD_REV)
- self.assertEqual(changed_deps[1].new_rev, BUILDTOOLS_NEW_REV)
+ def testGetMatchingDepsEntriesHandlesTwoPathsWithIdenticalFirstParts(self):
+ entries = GetMatchingDepsEntries(DEPS_ENTRIES, 'src/build')
+ self.assertEqual(len(entries), 1)
+ self.assertEqual(entries[0], DEPS_ENTRIES['src/build'])
+
+ def testCalculateChangedDeps(self):
+ _SetupGitLsRemoteCall(
+ self.fake,
+ 'https://chromium.googlesource.com/chromium/src/build',
+ BUILD_NEW_REV,
+ )
+ libyuv_deps = ParseLocalDepsFile(self._libyuv_depsfile)
+ new_cr_deps = ParseLocalDepsFile(self._new_cr_depsfile)
+ changed_deps = CalculateChangedDeps(libyuv_deps, new_cr_deps)
+ self.assertEqual(len(changed_deps), 2)
+ self.assertEqual(changed_deps[0].path, 'src/build')
+ self.assertEqual(changed_deps[0].current_rev, BUILD_OLD_REV)
+ self.assertEqual(changed_deps[0].new_rev, BUILD_NEW_REV)
+
+ self.assertEqual(changed_deps[1].path, 'src/buildtools')
+ self.assertEqual(changed_deps[1].current_rev, BUILDTOOLS_OLD_REV)
+ self.assertEqual(changed_deps[1].new_rev, BUILDTOOLS_NEW_REV)
def _SetupGitLsRemoteCall(cmd_fake, url, revision):
- cmd = ['git', 'ls-remote', url, revision]
- cmd_fake.add_expectation(cmd, _returns=(revision, None))
+ cmd = ['git', 'ls-remote', url, revision]
+ cmd_fake.add_expectation(cmd, _returns=(revision, None))
if __name__ == '__main__':
- unittest.main()
+ unittest.main()
diff --git a/tools_libyuv/get_landmines.py b/tools_libyuv/get_landmines.py
index 8b33483..ee0dbec 100755
--- a/tools_libyuv/get_landmines.py
+++ b/tools_libyuv/get_landmines.py
@@ -9,31 +9,30 @@
# be found in the AUTHORS file in the root of the source tree.
"""
-This file emits the list of reasons why a particular build needs to be clobbered
-(or a list of 'landmines').
+This file emits the list of reasons why a particular build needs to be
+clobbered (or a list of 'landmines').
"""
import sys
def print_landmines():
- """
- ALL LANDMINES ARE EMITTED FROM HERE.
- """
- # DO NOT add landmines as part of a regular CL. Landmines are a last-effort
- # bandaid fix if a CL that got landed has a build dependency bug and all bots
- # need to be cleaned up. If you're writing a new CL that causes build
- # dependency problems, fix the dependency problems instead of adding a
- # landmine.
- # See the Chromium version in src/build/get_landmines.py for usage examples.
- print('Clobber to remove GYP artifacts after switching bots to GN.')
- print('Another try to remove GYP artifacts after switching bots to GN.')
+ """ALL LANDMINES ARE EMITTED FROM HERE."""
+ # DO NOT add landmines as part of a regular CL. Landmines are a last-effort
+ # bandaid fix if a CL that got landed has a build dependency bug and all
+ # bots need to be cleaned up. If you're writing a new CL that causes build
+ # dependency problems, fix the dependency problems instead of adding a
+ # landmine.
+ # See the Chromium version in src/build/get_landmines.py for usage
+ # examples.
+ print('Clobber to remove GYP artifacts after switching bots to GN.')
+ print('Another try to remove GYP artifacts after switching bots to GN.')
def main():
- print_landmines()
- return 0
+ print_landmines()
+ return 0
if __name__ == '__main__':
- sys.exit(main())
+ sys.exit(main())
diff --git a/unit_test/convert_argb_test.cc b/unit_test/convert_argb_test.cc
index aeee8a7..718afec 100644
--- a/unit_test/convert_argb_test.cc
+++ b/unit_test/convert_argb_test.cc
@@ -12,6 +12,7 @@
#include <stdlib.h>
#include <time.h>
+#include "../unit_test/unit_test.h"
#include "libyuv/basic_types.h"
#include "libyuv/compare.h"
#include "libyuv/convert.h"
@@ -19,10 +20,6 @@
#include "libyuv/convert_from.h"
#include "libyuv/convert_from_argb.h"
#include "libyuv/cpu_id.h"
-#ifdef HAVE_JPEG
-#include "libyuv/mjpeg_decoder.h"
-#endif
-#include "../unit_test/unit_test.h"
#include "libyuv/planar_functions.h"
#include "libyuv/rotate.h"
#include "libyuv/video_common.h"
@@ -2682,19 +2679,47 @@
free_aligned_buffer_page_end(dest_rgb24);
}
-TEST_F(LibYUVConvertTest, Test565) {
+TEST_F(LibYUVConvertTest, TestARGBToRGB565) {
SIMD_ALIGNED(uint8_t orig_pixels[256][4]);
- SIMD_ALIGNED(uint8_t pixels565[256][2]);
+ SIMD_ALIGNED(uint8_t dest_rgb565[256][2]);
for (int i = 0; i < 256; ++i) {
for (int j = 0; j < 4; ++j) {
orig_pixels[i][j] = i;
}
}
- ARGBToRGB565(&orig_pixels[0][0], 0, &pixels565[0][0], 0, 256, 1);
- uint32_t checksum = HashDjb2(&pixels565[0][0], sizeof(pixels565), 5381);
+ ARGBToRGB565(&orig_pixels[0][0], 0, &dest_rgb565[0][0], 0, 256, 1);
+ uint32_t checksum = HashDjb2(&dest_rgb565[0][0], sizeof(dest_rgb565), 5381);
EXPECT_EQ(610919429u, checksum);
}
+
+TEST_F(LibYUVConvertTest, TestYUY2ToARGB) {
+ SIMD_ALIGNED(uint8_t orig_pixels[256][2]);
+ SIMD_ALIGNED(uint8_t dest_argb[256][4]);
+
+ for (int i = 0; i < 256; ++i) {
+ for (int j = 0; j < 2; ++j) {
+ orig_pixels[i][j] = i;
+ }
+ }
+ YUY2ToARGB(&orig_pixels[0][0], 0, &dest_argb[0][0], 0, 256, 1);
+ uint32_t checksum = HashDjb2(&dest_argb[0][0], sizeof(dest_argb), 5381);
+ EXPECT_EQ(3486643515u, checksum);
+}
+
+TEST_F(LibYUVConvertTest, TestUYVYToARGB) {
+ SIMD_ALIGNED(uint8_t orig_pixels[256][2]);
+ SIMD_ALIGNED(uint8_t dest_argb[256][4]);
+
+ for (int i = 0; i < 256; ++i) {
+ for (int j = 0; j < 2; ++j) {
+ orig_pixels[i][j] = i;
+ }
+ }
+ UYVYToARGB(&orig_pixels[0][0], 0, &dest_argb[0][0], 0, 256, 1);
+ uint32_t checksum = HashDjb2(&dest_argb[0][0], sizeof(dest_argb), 5381);
+ EXPECT_EQ(3486643515u, checksum);
+}
#endif // !defined(LEAN_TESTS)
} // namespace libyuv
diff --git a/unit_test/convert_test.cc b/unit_test/convert_test.cc
index f55bace..82163cb 100644
--- a/unit_test/convert_test.cc
+++ b/unit_test/convert_test.cc
@@ -188,6 +188,7 @@
TESTPLANARTOP(H010, uint16_t, 2, 2, 2, H420, uint8_t, 1, 2, 2, 10)
TESTPLANARTOP(H420, uint8_t, 1, 2, 2, H010, uint16_t, 2, 2, 2, 8)
TESTPLANARTOP(H420, uint8_t, 1, 2, 2, H012, uint16_t, 2, 2, 2, 8)
+TESTPLANARTOP(J420, uint8_t, 1, 2, 2, I420, uint8_t, 1, 2, 2, 8)
TESTPLANARTOP(I010, uint16_t, 2, 2, 2, I410, uint16_t, 2, 1, 1, 10)
TESTPLANARTOP(I210, uint16_t, 2, 2, 1, I410, uint16_t, 2, 1, 1, 10)
TESTPLANARTOP(I012, uint16_t, 2, 2, 2, I412, uint16_t, 2, 1, 1, 12)
@@ -326,18 +327,18 @@
#undef TESTAPLANARTOPI
// wrapper to keep API the same
-int I400ToNV21(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* /* src_u */,
- int /* src_stride_u */,
- const uint8_t* /* src_v */,
- int /* src_stride_v */,
- uint8_t* dst_y,
- int dst_stride_y,
- uint8_t* dst_vu,
- int dst_stride_vu,
- int width,
- int height) {
+static int I400ToNV21(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* /* src_u */,
+ int /* src_stride_u */,
+ const uint8_t* /* src_v */,
+ int /* src_stride_v */,
+ uint8_t* dst_y,
+ int dst_stride_y,
+ uint8_t* dst_vu,
+ int dst_stride_vu,
+ int width,
+ int height) {
return I400ToNV21(src_y, src_stride_y, dst_y, dst_stride_y, dst_vu,
dst_stride_vu, width, height);
}
@@ -452,6 +453,7 @@
TESTPLANARTOBP(I444, uint8_t, 1, 1, 1, NV12, uint8_t, 1, 2, 2, 8)
TESTPLANARTOBP(I444, uint8_t, 1, 1, 1, NV21, uint8_t, 1, 2, 2, 8)
TESTPLANARTOBP(I400, uint8_t, 1, 2, 2, NV21, uint8_t, 1, 2, 2, 8)
+TESTPLANARTOBP(I010, uint16_t, 2, 2, 2, NV12, uint8_t, 1, 2, 2, 8)
TESTPLANARTOBP(I010, uint16_t, 2, 2, 2, P010, uint16_t, 2, 2, 2, 10)
TESTPLANARTOBP(I210, uint16_t, 2, 2, 1, P210, uint16_t, 2, 2, 1, 10)
TESTPLANARTOBP(I012, uint16_t, 2, 2, 2, P012, uint16_t, 2, 2, 2, 12)
@@ -592,6 +594,7 @@
TESTBPTOBP(P216, uint16_t, 2, 2, 1, P416, uint16_t, 2, 1, 1, 12, 1, 1)
TESTBPTOBP(MM21, uint8_t, 1, 2, 2, NV12, uint8_t, 1, 2, 2, 8, 16, 32)
TESTBPTOBP(MT2T, uint8_t, 10 / 8, 2, 2, P010, uint16_t, 2, 2, 2, 10, 16, 32)
+TESTBPTOBP(P010, uint16_t, 2, 2, 2, NV12, uint8_t, 1, 2, 2, 8, 1, 1)
#define TESTATOPLANARI(FMT_A, BPP_A, YALIGN, FMT_PLANAR, SUBSAMP_X, SUBSAMP_Y, \
W1280, N, NEG, OFF) \
@@ -663,6 +666,7 @@
TESTATOPLANAR(ARGB, 4, 1, I444, 1, 1)
TESTATOPLANAR(ARGB, 4, 1, J420, 2, 2)
TESTATOPLANAR(ARGB, 4, 1, J422, 2, 1)
+TESTATOPLANAR(ARGB, 4, 1, J444, 1, 1)
TESTATOPLANAR(ABGR, 4, 1, J420, 2, 2)
TESTATOPLANAR(ABGR, 4, 1, J422, 2, 1)
#ifdef LITTLE_ENDIAN_ONLY_TEST
@@ -674,7 +678,9 @@
TESTATOPLANAR(I400, 1, 1, I420, 2, 2)
TESTATOPLANAR(J400, 1, 1, J420, 2, 2)
TESTATOPLANAR(RAW, 3, 1, I420, 2, 2)
+TESTATOPLANAR(RAW, 3, 1, I444, 1, 1)
TESTATOPLANAR(RAW, 3, 1, J420, 2, 2)
+TESTATOPLANAR(RAW, 3, 1, J444, 1, 1)
TESTATOPLANAR(RGB24, 3, 1, I420, 2, 2)
TESTATOPLANAR(RGB24, 3, 1, J420, 2, 2)
TESTATOPLANAR(RGBA, 4, 1, I420, 2, 2)
@@ -2070,7 +2076,7 @@
}
uint32_t checksum = HashDjb2(dest_j420, kSize * 3 / 2 * 2, 5381);
- EXPECT_EQ(2755440272u, checksum);
+ EXPECT_EQ(223551344u, checksum);
free_aligned_buffer_page_end(orig_rgb24);
free_aligned_buffer_page_end(dest_j420);
@@ -2098,13 +2104,35 @@
}
uint32_t checksum = HashDjb2(dest_i420, kSize * 3 / 2 * 2, 5381);
- EXPECT_EQ(1526656597u, checksum);
+ EXPECT_EQ(4197774805u, checksum);
free_aligned_buffer_page_end(orig_rgb24);
free_aligned_buffer_page_end(dest_i420);
}
#endif
+TEST_F(LibYUVConvertTest, TestJ420ToI420) {
+ const uint8_t src_y[12] = {0, 0, 128, 128, 255, 255,
+ 0, 0, 128, 128, 255, 255};
+ const uint8_t src_u[3] = {0, 128, 255};
+ const uint8_t src_v[3] = {0, 128, 255};
+ uint8_t dst_y[12];
+ uint8_t dst_u[3];
+ uint8_t dst_v[3];
+ ASSERT_EQ(J420ToI420(src_y, 6, src_u, 3, src_v, 3, dst_y, 6, dst_u, 3, dst_v,
+ 3, 6, 2),
+ 0);
+ EXPECT_EQ(dst_y[0], 16);
+ EXPECT_EQ(dst_y[2], 126);
+ EXPECT_EQ(dst_y[4], 235);
+ EXPECT_EQ(dst_u[0], 16);
+ EXPECT_EQ(dst_u[1], 128);
+ EXPECT_EQ(dst_u[2], 240);
+ EXPECT_EQ(dst_v[0], 16);
+ EXPECT_EQ(dst_v[1], 128);
+ EXPECT_EQ(dst_v[2], 240);
+}
+
#endif // !defined(LEAN_TESTS)
} // namespace libyuv
diff --git a/unit_test/cpu_test.cc b/unit_test/cpu_test.cc
index 437b663..ebae292 100644
--- a/unit_test/cpu_test.cc
+++ b/unit_test/cpu_test.cc
@@ -8,9 +8,15 @@
* be found in the AUTHORS file in the root of the source tree.
*/
+#include <stdio.h>
#include <stdlib.h>
#include <string.h>
+#ifdef __linux__
+#include <ctype.h>
+#include <sys/utsname.h>
+#endif
+
#include "../unit_test/unit_test.h"
#include "libyuv/basic_types.h"
#include "libyuv/cpu_id.h"
@@ -18,78 +24,201 @@
namespace libyuv {
+#if defined(__i386__) || defined(__x86_64__) || defined(_M_IX86) || \
+ defined(_M_X64)
+TEST_F(LibYUVBaseTest, TestCpuId) {
+ int has_x86 = TestCpuFlag(kCpuHasX86);
+ if (has_x86) {
+ int cpu_info[4];
+ // Vendor ID:
+ // AuthenticAMD AMD processor
+ // CentaurHauls Centaur processor
+ // CyrixInstead Cyrix processor
+ // GenuineIntel Intel processor
+ // GenuineTMx86 Transmeta processor
+ // Geode by NSC National Semiconductor processor
+ // NexGenDriven NexGen processor
+ // RiseRiseRise Rise Technology processor
+ // SiS SiS SiS SiS processor
+ // UMC UMC UMC UMC processor
+ CpuId(0, 0, cpu_info);
+ cpu_info[0] = cpu_info[1]; // Reorder output
+ cpu_info[1] = cpu_info[3];
+ cpu_info[3] = 0;
+ printf("Cpu Vendor: %s 0x%x 0x%x 0x%x\n",
+ reinterpret_cast<char*>(&cpu_info[0]), cpu_info[0], cpu_info[1],
+ cpu_info[2]);
+ EXPECT_EQ(12u, strlen(reinterpret_cast<char*>(&cpu_info[0])));
+
+ // CPU Family and Model
+ // 3:0 - Stepping
+ // 7:4 - Model
+ // 11:8 - Family
+ // 13:12 - Processor Type
+ // 19:16 - Extended Model
+ // 27:20 - Extended Family
+ CpuId(1, 0, cpu_info);
+ int family = ((cpu_info[0] >> 8) & 0x0f) | ((cpu_info[0] >> 16) & 0xff0);
+ int model = ((cpu_info[0] >> 4) & 0x0f) | ((cpu_info[0] >> 12) & 0xf0);
+ printf("Cpu Family %d (0x%x), Model %d (0x%x)\n", family, family, model,
+ model);
+ }
+}
+#endif
+
+#ifdef __linux__
+static void KernelVersion(int* version) {
+ struct utsname buffer;
+ int i = 0;
+
+ version[0] = version[1] = 0;
+ if (uname(&buffer) == 0) {
+ char* v = buffer.release;
+ for (i = 0; *v && i < 2; ++v) {
+ if (isdigit(*v)) {
+ version[i++] = (int)strtol(v, &v, 10);
+ }
+ }
+ }
+}
+#endif
+
TEST_F(LibYUVBaseTest, TestCpuHas) {
- int cpu_flags = TestCpuFlag(-1);
- printf("Cpu Flags 0x%x\n", cpu_flags);
+#if defined(__linux__)
+ {
+ int kernelversion[2];
+ KernelVersion(kernelversion);
+ printf("Kernel Version %d.%d\n", kernelversion[0], kernelversion[1]);
+ }
+#endif // defined(__linux__)
+
#if defined(__arm__) || defined(__aarch64__)
int has_arm = TestCpuFlag(kCpuHasARM);
- printf("Has ARM 0x%x\n", has_arm);
- int has_neon = TestCpuFlag(kCpuHasNEON);
- printf("Has NEON 0x%x\n", has_neon);
-#endif
-#if defined(__riscv) && defined(__linux__)
+ if (has_arm) {
+ int has_neon = TestCpuFlag(kCpuHasNEON);
+ int has_neon_dotprod = TestCpuFlag(kCpuHasNeonDotProd);
+ int has_neon_i8mm = TestCpuFlag(kCpuHasNeonI8MM);
+ int has_sve = TestCpuFlag(kCpuHasSVE);
+ int has_sve2 = TestCpuFlag(kCpuHasSVE2);
+ int has_sme = TestCpuFlag(kCpuHasSME);
+ printf("Has Arm 0x%x\n", has_arm);
+ printf("Has Neon 0x%x\n", has_neon);
+ printf("Has Neon DotProd 0x%x\n", has_neon_dotprod);
+ printf("Has Neon I8MM 0x%x\n", has_neon_i8mm);
+ printf("Has SVE 0x%x\n", has_sve);
+ printf("Has SVE2 0x%x\n", has_sve2);
+ printf("Has SME 0x%x\n", has_sme);
+
+#if defined(__aarch64__)
+ // Read and print the SVE and SME vector lengths.
+ if (has_sve) {
+ int sve_vl;
+ asm(".inst 0x04bf5020 \n" // rdvl x0, #1
+ "mov %w[sve_vl], w0 \n"
+ : [sve_vl] "=r"(sve_vl) // %[sve_vl]
+ :
+ : "x0");
+ printf("SVE vector length: %d bytes\n", sve_vl);
+ }
+ if (has_sme) {
+ int sme_vl;
+ asm(".inst 0x04bf5820 \n" // rdsvl x0, #1
+ "mov %w[sme_vl], w0 \n"
+ : [sme_vl] "=r"(sme_vl) // %[sme_vl]
+ :
+ : "x0");
+ printf("SME vector length: %d bytes\n", sme_vl);
+ }
+#endif // defined(__aarch64__)
+ }
+#endif // if defined(__arm__) || defined(__aarch64__)
+
+#if defined(__riscv)
int has_riscv = TestCpuFlag(kCpuHasRISCV);
- printf("Has RISCV 0x%x\n", has_riscv);
- int has_rvv = TestCpuFlag(kCpuHasRVV);
- printf("Has RVV 0x%x\n", has_rvv);
- int has_rvvzvfh = TestCpuFlag(kCpuHasRVVZVFH);
- printf("Has RVVZVFH 0x%x\n", has_rvvzvfh);
-#endif
+ if (has_riscv) {
+ int has_rvv = TestCpuFlag(kCpuHasRVV);
+ printf("Has RISCV 0x%x\n", has_riscv);
+ printf("Has RVV 0x%x\n", has_rvv);
+
+ // Read and print the RVV vector length.
+ if (has_rvv) {
+ register uint32_t vlenb __asm__("t0");
+ __asm__(".word 0xC22022F3" /* CSRR t0, vlenb */ : "=r"(vlenb));
+ printf("RVV vector length: %d bytes\n", vlenb);
+ }
+ }
+#endif // defined(__riscv)
+
+#if defined(__mips__)
+ int has_mips = TestCpuFlag(kCpuHasMIPS);
+ if (has_mips) {
+ int has_msa = TestCpuFlag(kCpuHasMSA);
+ printf("Has MIPS 0x%x\n", has_mips);
+ printf("Has MSA 0x%x\n", has_msa);
+ }
+#endif // defined(__mips__)
+
+#if defined(__loongarch__)
+ int has_loongarch = TestCpuFlag(kCpuHasLOONGARCH);
+ if (has_loongarch) {
+ int has_lsx = TestCpuFlag(kCpuHasLSX);
+ int has_lasx = TestCpuFlag(kCpuHasLASX);
+ printf("Has LOONGARCH 0x%x\n", has_loongarch);
+ printf("Has LSX 0x%x\n", has_lsx);
+ printf("Has LASX 0x%x\n", has_lasx);
+ }
+#endif // defined(__loongarch__)
+
#if defined(__i386__) || defined(__x86_64__) || defined(_M_IX86) || \
defined(_M_X64)
int has_x86 = TestCpuFlag(kCpuHasX86);
- int has_sse2 = TestCpuFlag(kCpuHasSSE2);
- int has_ssse3 = TestCpuFlag(kCpuHasSSSE3);
- int has_sse41 = TestCpuFlag(kCpuHasSSE41);
- int has_sse42 = TestCpuFlag(kCpuHasSSE42);
- int has_avx = TestCpuFlag(kCpuHasAVX);
- int has_avx2 = TestCpuFlag(kCpuHasAVX2);
- int has_erms = TestCpuFlag(kCpuHasERMS);
- int has_fma3 = TestCpuFlag(kCpuHasFMA3);
- int has_f16c = TestCpuFlag(kCpuHasF16C);
- int has_avx512bw = TestCpuFlag(kCpuHasAVX512BW);
- int has_avx512vl = TestCpuFlag(kCpuHasAVX512VL);
- int has_avx512vnni = TestCpuFlag(kCpuHasAVX512VNNI);
- int has_avx512vbmi = TestCpuFlag(kCpuHasAVX512VBMI);
- int has_avx512vbmi2 = TestCpuFlag(kCpuHasAVX512VBMI2);
- int has_avx512vbitalg = TestCpuFlag(kCpuHasAVX512VBITALG);
- int has_avx10 = TestCpuFlag(kCpuHasAVX10);
- int has_avxvnni = TestCpuFlag(kCpuHasAVXVNNI);
- int has_avxvnniint8 = TestCpuFlag(kCpuHasAVXVNNIINT8);
- printf("Has X86 0x%x\n", has_x86);
- printf("Has SSE2 0x%x\n", has_sse2);
- printf("Has SSSE3 0x%x\n", has_ssse3);
- printf("Has SSE41 0x%x\n", has_sse41);
- printf("Has SSE42 0x%x\n", has_sse42);
- printf("Has AVX 0x%x\n", has_avx);
- printf("Has AVX2 0x%x\n", has_avx2);
- printf("Has ERMS 0x%x\n", has_erms);
- printf("Has FMA3 0x%x\n", has_fma3);
- printf("Has F16C 0x%x\n", has_f16c);
- printf("Has AVX512BW 0x%x\n", has_avx512bw);
- printf("Has AVX512VL 0x%x\n", has_avx512vl);
- printf("Has AVX512VNNI 0x%x\n", has_avx512vnni);
- printf("Has AVX512VBMI 0x%x\n", has_avx512vbmi);
- printf("Has AVX512VBMI2 0x%x\n", has_avx512vbmi2);
- printf("Has AVX512VBITALG 0x%x\n", has_avx512vbitalg);
- printf("Has AVX10 0x%x\n", has_avx10);
- printf("HAS AVXVNNI 0x%x\n", has_avxvnni);
- printf("Has AVXVNNIINT8 0x%x\n", has_avxvnniint8);
-#endif
-#if defined(__mips__)
- int has_mips = TestCpuFlag(kCpuHasMIPS);
- printf("Has MIPS 0x%x\n", has_mips);
- int has_msa = TestCpuFlag(kCpuHasMSA);
- printf("Has MSA 0x%x\n", has_msa);
-#endif
-#if defined(__loongarch__)
- int has_loongarch = TestCpuFlag(kCpuHasLOONGARCH);
- printf("Has LOONGARCH 0x%x\n", has_loongarch);
- int has_lsx = TestCpuFlag(kCpuHasLSX);
- printf("Has LSX 0x%x\n", has_lsx);
- int has_lasx = TestCpuFlag(kCpuHasLASX);
- printf("Has LASX 0x%x\n", has_lasx);
-#endif
+ if (has_x86) {
+ int has_sse2 = TestCpuFlag(kCpuHasSSE2);
+ int has_ssse3 = TestCpuFlag(kCpuHasSSSE3);
+ int has_sse41 = TestCpuFlag(kCpuHasSSE41);
+ int has_sse42 = TestCpuFlag(kCpuHasSSE42);
+ int has_avx = TestCpuFlag(kCpuHasAVX);
+ int has_avx2 = TestCpuFlag(kCpuHasAVX2);
+ int has_erms = TestCpuFlag(kCpuHasERMS);
+ int has_fsmr = TestCpuFlag(kCpuHasFSMR);
+ int has_fma3 = TestCpuFlag(kCpuHasFMA3);
+ int has_f16c = TestCpuFlag(kCpuHasF16C);
+ int has_avx512bw = TestCpuFlag(kCpuHasAVX512BW);
+ int has_avx512vl = TestCpuFlag(kCpuHasAVX512VL);
+ int has_avx512vnni = TestCpuFlag(kCpuHasAVX512VNNI);
+ int has_avx512vbmi = TestCpuFlag(kCpuHasAVX512VBMI);
+ int has_avx512vbmi2 = TestCpuFlag(kCpuHasAVX512VBMI2);
+ int has_avx512vbitalg = TestCpuFlag(kCpuHasAVX512VBITALG);
+ int has_avx10 = TestCpuFlag(kCpuHasAVX10);
+ int has_avx10_2 = TestCpuFlag(kCpuHasAVX10_2);
+ int has_avxvnni = TestCpuFlag(kCpuHasAVXVNNI);
+ int has_avxvnniint8 = TestCpuFlag(kCpuHasAVXVNNIINT8);
+ int has_amxint8 = TestCpuFlag(kCpuHasAMXINT8);
+ printf("Has X86 0x%x\n", has_x86);
+ printf("Has SSE2 0x%x\n", has_sse2);
+ printf("Has SSSE3 0x%x\n", has_ssse3);
+ printf("Has SSE4.1 0x%x\n", has_sse41);
+ printf("Has SSE4.2 0x%x\n", has_sse42);
+ printf("Has AVX 0x%x\n", has_avx);
+ printf("Has AVX2 0x%x\n", has_avx2);
+ printf("Has ERMS 0x%x\n", has_erms);
+ printf("Has FSMR 0x%x\n", has_fsmr);
+ printf("Has FMA3 0x%x\n", has_fma3);
+ printf("Has F16C 0x%x\n", has_f16c);
+ printf("Has AVX512BW 0x%x\n", has_avx512bw);
+ printf("Has AVX512VL 0x%x\n", has_avx512vl);
+ printf("Has AVX512VNNI 0x%x\n", has_avx512vnni);
+ printf("Has AVX512VBMI 0x%x\n", has_avx512vbmi);
+ printf("Has AVX512VBMI2 0x%x\n", has_avx512vbmi2);
+ printf("Has AVX512VBITALG 0x%x\n", has_avx512vbitalg);
+ printf("Has AVX10 0x%x\n", has_avx10);
+ printf("Has AVX10_2 0x%x\n", has_avx10_2);
+ printf("HAS AVXVNNI 0x%x\n", has_avxvnni);
+ printf("Has AVXVNNIINT8 0x%x\n", has_avxvnniint8);
+ printf("Has AMXINT8 0x%x\n", has_amxint8);
+ }
+#endif // defined(__i386__) || defined(__x86_64__) || defined(_M_IX86) ||
+ // defined(_M_X64)
}
TEST_F(LibYUVBaseTest, TestCompilerMacros) {
@@ -142,6 +271,9 @@
#ifdef __riscv_v_intrinsic
printf("__riscv_v_intrinsic %d\n", __riscv_v_intrinsic);
#endif
+#ifdef __riscv_zve64x
+ printf("__riscv_zve64x %d\n", __riscv_zve64x);
+#endif
#ifdef __APPLE__
printf("__APPLE__ %d\n", __APPLE__);
#endif
@@ -199,48 +331,6 @@
#endif
}
-#if defined(__i386__) || defined(__x86_64__) || defined(_M_IX86) || \
- defined(_M_X64)
-TEST_F(LibYUVBaseTest, TestCpuId) {
- int has_x86 = TestCpuFlag(kCpuHasX86);
- if (has_x86) {
- int cpu_info[4];
- // Vendor ID:
- // AuthenticAMD AMD processor
- // CentaurHauls Centaur processor
- // CyrixInstead Cyrix processor
- // GenuineIntel Intel processor
- // GenuineTMx86 Transmeta processor
- // Geode by NSC National Semiconductor processor
- // NexGenDriven NexGen processor
- // RiseRiseRise Rise Technology processor
- // SiS SiS SiS SiS processor
- // UMC UMC UMC UMC processor
- CpuId(0, 0, cpu_info);
- cpu_info[0] = cpu_info[1]; // Reorder output
- cpu_info[1] = cpu_info[3];
- cpu_info[3] = 0;
- printf("Cpu Vendor: %s 0x%x 0x%x 0x%x\n",
- reinterpret_cast<char*>(&cpu_info[0]), cpu_info[0], cpu_info[1],
- cpu_info[2]);
- EXPECT_EQ(12u, strlen(reinterpret_cast<char*>(&cpu_info[0])));
-
- // CPU Family and Model
- // 3:0 - Stepping
- // 7:4 - Model
- // 11:8 - Family
- // 13:12 - Processor Type
- // 19:16 - Extended Model
- // 27:20 - Extended Family
- CpuId(1, 0, cpu_info);
- int family = ((cpu_info[0] >> 8) & 0x0f) | ((cpu_info[0] >> 16) & 0xff0);
- int model = ((cpu_info[0] >> 4) & 0x0f) | ((cpu_info[0] >> 12) & 0xf0);
- printf("Cpu Family %d (0x%x), Model %d (0x%x)\n", family, family, model,
- model);
- }
-}
-#endif
-
static int FileExists(const char* file_name) {
FILE* f = fopen(file_name, "r");
if (!f) {
@@ -250,21 +340,20 @@
return 1;
}
-TEST_F(LibYUVBaseTest, TestLinuxNeon) {
+TEST_F(LibYUVBaseTest, TestLinuxArm) {
if (FileExists("../../unit_test/testdata/arm_v7.txt")) {
printf("Note: testing to load \"../../unit_test/testdata/arm_v7.txt\"\n");
EXPECT_EQ(0, ArmCpuCaps("../../unit_test/testdata/arm_v7.txt"));
EXPECT_EQ(kCpuHasNEON, ArmCpuCaps("../../unit_test/testdata/tegra3.txt"));
- EXPECT_EQ(kCpuHasNEON, ArmCpuCaps("../../unit_test/testdata/juno.txt"));
} else {
printf("WARNING: unable to load \"../../unit_test/testdata/arm_v7.txt\"\n");
}
-#if defined(__linux__) && defined(__ARM_NEON__)
+#if defined(__linux__) && defined(__ARM_NEON__) && !defined(__aarch64__)
if (FileExists("/proc/cpuinfo")) {
if (kCpuHasNEON != ArmCpuCaps("/proc/cpuinfo")) {
- // This can happen on ARM emulator but /proc/cpuinfo is from host.
- printf("WARNING: Neon build enabled but CPU does not have NEON\n");
+ // This can happen on Arm emulator but /proc/cpuinfo is from host.
+ printf("WARNING: Neon build enabled but CPU does not have Neon\n");
}
} else {
printf("WARNING: unable to load \"/proc/cpuinfo\"\n");
@@ -272,6 +361,29 @@
#endif
}
+#if defined(__linux__) && defined(__aarch64__)
+TEST_F(LibYUVBaseTest, TestLinuxAArch64) {
+ // Values taken from a Cortex-A57 machine, only Neon available.
+ EXPECT_EQ(kCpuHasNEON, AArch64CpuCaps(0xffU, 0x0U));
+
+ // Values taken from a Google Pixel 7.
+ int expected = kCpuHasNEON | kCpuHasNeonDotProd;
+ EXPECT_EQ(expected, AArch64CpuCaps(0x119fffU, 0x0U));
+
+ // Values taken from a Google Pixel 8.
+ expected = kCpuHasNEON | kCpuHasNeonDotProd | kCpuHasNeonI8MM | kCpuHasSVE |
+ kCpuHasSVE2;
+ EXPECT_EQ(expected, AArch64CpuCaps(0x3fffffffU, 0x2f33fU));
+
+ // Values taken from a Neoverse N2 machine.
+ EXPECT_EQ(expected, AArch64CpuCaps(0x3fffffffU, 0x2f3ffU));
+
+ // Check for SME feature detection.
+ expected |= kCpuHasSME;
+ EXPECT_EQ(expected, AArch64CpuCaps(0x3fffffffU, 0x82f3ffU));
+}
+#endif
+
TEST_F(LibYUVBaseTest, TestLinuxMipsMsa) {
if (FileExists("../../unit_test/testdata/mips.txt")) {
printf("Note: testing to load \"../../unit_test/testdata/mips.txt\"\n");
diff --git a/unit_test/cpu_thread_test.cc b/unit_test/cpu_thread_test.cc
index 69aab74..b6c0fa0 100644
--- a/unit_test/cpu_thread_test.cc
+++ b/unit_test/cpu_thread_test.cc
@@ -27,7 +27,7 @@
namespace libyuv {
#ifdef LIBYUV_HAVE_PTHREAD
-void* ThreadMain(void* arg) {
+static void* ThreadMain(void* arg) {
int* flags = static_cast<int*>(arg);
*flags = TestCpuFlag(kCpuInitialized);
diff --git a/unit_test/planar_test.cc b/unit_test/planar_test.cc
index ec1d72e..576696b 100644
--- a/unit_test/planar_test.cc
+++ b/unit_test/planar_test.cc
@@ -1551,98 +1551,78 @@
#endif
}
-TEST_F(LibYUVPlanarTest, TestCopyPlane) {
- int err = 0;
- int yw = benchmark_width_;
- int yh = benchmark_height_;
- int b = 12;
- int i, j;
-
- int y_plane_size = (yw + b * 2) * (yh + b * 2);
- align_buffer_page_end(orig_y, y_plane_size);
+static int TestCopyPlane(int benchmark_width,
+ int benchmark_height,
+ int benchmark_iterations,
+ int disable_cpu_flags,
+ int benchmark_cpu_info,
+ int invert,
+ int off) {
+ const int y_plane_size = benchmark_width * benchmark_height;
+ align_buffer_page_end(orig_y, y_plane_size + off);
align_buffer_page_end(dst_c, y_plane_size);
align_buffer_page_end(dst_opt, y_plane_size);
- memset(orig_y, 0, y_plane_size);
- memset(dst_c, 0, y_plane_size);
- memset(dst_opt, 0, y_plane_size);
-
- // Fill image buffers with random data.
- for (i = b; i < (yh + b); ++i) {
- for (j = b; j < (yw + b); ++j) {
- orig_y[i * (yw + b * 2) + j] = fastrand() & 0xff;
- }
- }
-
- // Fill destination buffers with random data.
- for (i = 0; i < y_plane_size; ++i) {
- uint8_t random_number = fastrand() & 0x7f;
- dst_c[i] = random_number;
- dst_opt[i] = dst_c[i];
- }
-
- int y_off = b * (yw + b * 2) + b;
-
- int y_st = yw + b * 2;
- int stride = 8;
-
- // Disable all optimizations.
- MaskCpuFlags(disable_cpu_flags_);
- for (j = 0; j < benchmark_iterations_; j++) {
- CopyPlane(orig_y + y_off, y_st, dst_c + y_off, stride, yw, yh);
- }
-
- // Enable optimizations.
- MaskCpuFlags(benchmark_cpu_info_);
- for (j = 0; j < benchmark_iterations_; j++) {
- CopyPlane(orig_y + y_off, y_st, dst_opt + y_off, stride, yw, yh);
- }
-
- for (i = 0; i < y_plane_size; ++i) {
- if (dst_c[i] != dst_opt[i]) {
- ++err;
- }
- }
-
- free_aligned_buffer_page_end(orig_y);
- free_aligned_buffer_page_end(dst_c);
- free_aligned_buffer_page_end(dst_opt);
-
- EXPECT_EQ(0, err);
-}
-
-TEST_F(LibYUVPlanarTest, CopyPlane_Opt) {
- int i;
- int y_plane_size = benchmark_width_ * benchmark_height_;
- align_buffer_page_end(orig_y, y_plane_size);
- align_buffer_page_end(dst_c, y_plane_size);
- align_buffer_page_end(dst_opt, y_plane_size);
-
- MemRandomize(orig_y, y_plane_size);
+ MemRandomize(orig_y + off, y_plane_size);
memset(dst_c, 1, y_plane_size);
memset(dst_opt, 2, y_plane_size);
// Disable all optimizations.
- MaskCpuFlags(disable_cpu_flags_);
- for (i = 0; i < benchmark_iterations_; i++) {
- CopyPlane(orig_y, benchmark_width_, dst_c, benchmark_width_,
- benchmark_width_, benchmark_height_);
+ MaskCpuFlags(disable_cpu_flags);
+ for (int i = 0; i < benchmark_iterations; i++) {
+ CopyPlane(orig_y + off, benchmark_width, dst_c, benchmark_width,
+ benchmark_width, benchmark_height * invert);
}
// Enable optimizations.
- MaskCpuFlags(benchmark_cpu_info_);
- for (i = 0; i < benchmark_iterations_; i++) {
- CopyPlane(orig_y, benchmark_width_, dst_opt, benchmark_width_,
- benchmark_width_, benchmark_height_);
+ MaskCpuFlags(benchmark_cpu_info);
+ for (int i = 0; i < benchmark_iterations; i++) {
+ CopyPlane(orig_y + off, benchmark_width, dst_opt, benchmark_width,
+ benchmark_width, benchmark_height * invert);
}
- for (i = 0; i < y_plane_size; ++i) {
- EXPECT_EQ(dst_c[i], dst_opt[i]);
+ int max_diff = 0;
+ for (int i = 0; i < y_plane_size; ++i) {
+ int abs_diff =
+ abs(static_cast<int>(dst_c[i]) - static_cast<int>(dst_opt[i]));
+ if (abs_diff > max_diff) {
+ max_diff = abs_diff;
+ }
}
free_aligned_buffer_page_end(orig_y);
free_aligned_buffer_page_end(dst_c);
free_aligned_buffer_page_end(dst_opt);
+
+ return max_diff;
+}
+
+TEST_F(LibYUVPlanarTest, CopyPlane_Any) {
+ int max_diff = TestCopyPlane(benchmark_width_ + 1, benchmark_height_,
+ benchmark_iterations_, disable_cpu_flags_,
+ benchmark_cpu_info_, +1, 0);
+ EXPECT_LE(max_diff, 0);
+}
+
+TEST_F(LibYUVPlanarTest, CopyPlane_Unaligned) {
+ int max_diff =
+ TestCopyPlane(benchmark_width_, benchmark_height_, benchmark_iterations_,
+ disable_cpu_flags_, benchmark_cpu_info_, +1, 1);
+ EXPECT_LE(max_diff, 0);
+}
+
+TEST_F(LibYUVPlanarTest, CopyPlane_Invert) {
+ int max_diff =
+ TestCopyPlane(benchmark_width_, benchmark_height_, benchmark_iterations_,
+ disable_cpu_flags_, benchmark_cpu_info_, -1, 0);
+ EXPECT_LE(max_diff, 0);
+}
+
+TEST_F(LibYUVPlanarTest, CopyPlane_Opt) {
+ int max_diff =
+ TestCopyPlane(benchmark_width_, benchmark_height_, benchmark_iterations_,
+ disable_cpu_flags_, benchmark_cpu_info_, +1, 0);
+ EXPECT_LE(max_diff, 0);
}
TEST_F(LibYUVPlanarTest, TestCopyPlaneZero) {
@@ -1906,28 +1886,28 @@
int max_diff = TestMultiply(benchmark_width_ + 1, benchmark_height_,
benchmark_iterations_, disable_cpu_flags_,
benchmark_cpu_info_, +1, 0);
- EXPECT_LE(max_diff, 1);
+ EXPECT_LE(max_diff, 0);
}
TEST_F(LibYUVPlanarTest, ARGBMultiply_Unaligned) {
int max_diff =
TestMultiply(benchmark_width_, benchmark_height_, benchmark_iterations_,
disable_cpu_flags_, benchmark_cpu_info_, +1, 1);
- EXPECT_LE(max_diff, 1);
+ EXPECT_LE(max_diff, 0);
}
TEST_F(LibYUVPlanarTest, ARGBMultiply_Invert) {
int max_diff =
TestMultiply(benchmark_width_, benchmark_height_, benchmark_iterations_,
disable_cpu_flags_, benchmark_cpu_info_, -1, 0);
- EXPECT_LE(max_diff, 1);
+ EXPECT_LE(max_diff, 0);
}
TEST_F(LibYUVPlanarTest, ARGBMultiply_Opt) {
int max_diff =
TestMultiply(benchmark_width_, benchmark_height_, benchmark_iterations_,
disable_cpu_flags_, benchmark_cpu_info_, +1, 0);
- EXPECT_LE(max_diff, 1);
+ EXPECT_LE(max_diff, 0);
}
static int TestAdd(int width,
@@ -2496,42 +2476,45 @@
}
}
-int TestHalfFloatPlane(int benchmark_width,
- int benchmark_height,
- int benchmark_iterations,
- int disable_cpu_flags,
- int benchmark_cpu_info,
- float scale,
- int mask) {
+static int TestHalfFloatPlane(int benchmark_width,
+ int benchmark_height,
+ int benchmark_iterations,
+ int disable_cpu_flags,
+ int benchmark_cpu_info,
+ float scale,
+ int mask,
+ int invert,
+ int off) {
int i, j;
const int y_plane_size = benchmark_width * benchmark_height * 2;
+ align_buffer_page_end(orig_y, y_plane_size + off);
+ align_buffer_page_end(dst_c, y_plane_size);
+ align_buffer_page_end(dst_opt, y_plane_size);
- align_buffer_page_end(orig_y, y_plane_size * 3);
- uint8_t* dst_opt = orig_y + y_plane_size;
- uint8_t* dst_c = orig_y + y_plane_size * 2;
-
- MemRandomize(orig_y, y_plane_size);
- memset(dst_c, 0, y_plane_size);
- memset(dst_opt, 1, y_plane_size);
+ MemRandomize(orig_y + off, y_plane_size);
+ memset(dst_c, 1, y_plane_size);
+ memset(dst_opt, 2, y_plane_size);
for (i = 0; i < y_plane_size / 2; ++i) {
- reinterpret_cast<uint16_t*>(orig_y)[i] &= mask;
+ reinterpret_cast<uint16_t*>(orig_y + off)[i] &= mask;
}
// Disable all optimizations.
MaskCpuFlags(disable_cpu_flags);
for (j = 0; j < benchmark_iterations; j++) {
- HalfFloatPlane(reinterpret_cast<uint16_t*>(orig_y), benchmark_width * 2,
- reinterpret_cast<uint16_t*>(dst_c), benchmark_width * 2,
- scale, benchmark_width, benchmark_height);
+ HalfFloatPlane(reinterpret_cast<uint16_t*>(orig_y + off),
+ benchmark_width * 2, reinterpret_cast<uint16_t*>(dst_c),
+ benchmark_width * 2, scale, benchmark_width,
+ benchmark_height * invert);
}
// Enable optimizations.
MaskCpuFlags(benchmark_cpu_info);
for (j = 0; j < benchmark_iterations; j++) {
- HalfFloatPlane(reinterpret_cast<uint16_t*>(orig_y), benchmark_width * 2,
- reinterpret_cast<uint16_t*>(dst_opt), benchmark_width * 2,
- scale, benchmark_width, benchmark_height);
+ HalfFloatPlane(reinterpret_cast<uint16_t*>(orig_y + off),
+ benchmark_width * 2, reinterpret_cast<uint16_t*>(dst_opt),
+ benchmark_width * 2, scale, benchmark_width,
+ benchmark_height * invert);
}
int max_diff = 0;
@@ -2545,97 +2528,130 @@
}
free_aligned_buffer_page_end(orig_y);
+ free_aligned_buffer_page_end(dst_c);
+ free_aligned_buffer_page_end(dst_opt);
return max_diff;
}
-#if defined(__arm__)
-static void EnableFlushDenormalToZero(void) {
- uint32_t cw;
- __asm__ __volatile__(
- "vmrs %0, fpscr \n"
- "orr %0, %0, #0x1000000 \n"
- "vmsr fpscr, %0 \n"
- : "=r"(cw)::"memory");
-}
-#endif
-
-// 5 bit exponent with bias of 15 will underflow to a denormal if scale causes
-// exponent to be less than 0. 15 - log2(65536) = -1/ This shouldnt normally
-// happen since scale is 1/(1<<bits) where bits is 9, 10 or 12.
-
-TEST_F(LibYUVPlanarTest, TestHalfFloatPlane_16bit_denormal) {
-// 32 bit arm rounding on denormal case is off by 1 compared to C.
-#if defined(__arm__)
- EnableFlushDenormalToZero();
-#endif
- int diff = TestHalfFloatPlane(benchmark_width_, benchmark_height_,
- benchmark_iterations_, disable_cpu_flags_,
- benchmark_cpu_info_, 1.0f / 65536.0f, 65535);
- EXPECT_EQ(0, diff);
-}
-
TEST_F(LibYUVPlanarTest, TestHalfFloatPlane_16bit_One) {
int diff = TestHalfFloatPlane(benchmark_width_, benchmark_height_,
benchmark_iterations_, disable_cpu_flags_,
- benchmark_cpu_info_, 1.0f, 65535);
+ benchmark_cpu_info_, 1.0f, 65535, +1, 0);
EXPECT_LE(diff, 1);
}
TEST_F(LibYUVPlanarTest, TestHalfFloatPlane_16bit_Opt) {
- int diff = TestHalfFloatPlane(benchmark_width_, benchmark_height_,
- benchmark_iterations_, disable_cpu_flags_,
- benchmark_cpu_info_, 1.0f / 4096.0f, 65535);
+ int diff = TestHalfFloatPlane(
+ benchmark_width_, benchmark_height_, benchmark_iterations_,
+ disable_cpu_flags_, benchmark_cpu_info_, 1.0f / 65535.0f, 65535, +1, 0);
+ EXPECT_EQ(0, diff);
+}
+
+TEST_F(LibYUVPlanarTest, TestHalfFloatPlane_12bit_Opt) {
+ int diff = TestHalfFloatPlane(
+ benchmark_width_, benchmark_height_, benchmark_iterations_,
+ disable_cpu_flags_, benchmark_cpu_info_, 1.0f / 4095.0f, 4095, +1, 0);
EXPECT_EQ(0, diff);
}
TEST_F(LibYUVPlanarTest, TestHalfFloatPlane_10bit_Opt) {
- int diff = TestHalfFloatPlane(benchmark_width_, benchmark_height_,
- benchmark_iterations_, disable_cpu_flags_,
- benchmark_cpu_info_, 1.0f / 1024.0f, 1023);
+ int diff = TestHalfFloatPlane(
+ benchmark_width_, benchmark_height_, benchmark_iterations_,
+ disable_cpu_flags_, benchmark_cpu_info_, 1.0f / 1023.0f, 1023, +1, 0);
EXPECT_EQ(0, diff);
}
TEST_F(LibYUVPlanarTest, TestHalfFloatPlane_9bit_Opt) {
int diff = TestHalfFloatPlane(benchmark_width_, benchmark_height_,
benchmark_iterations_, disable_cpu_flags_,
- benchmark_cpu_info_, 1.0f / 512.0f, 511);
+ benchmark_cpu_info_, 1.0f / 511.0f, 511, +1, 0);
+ EXPECT_EQ(0, diff);
+}
+
+TEST_F(LibYUVPlanarTest, TestHalfFloatPlane_Any) {
+ int diff = TestHalfFloatPlane(
+ benchmark_width_ + 1, benchmark_height_, benchmark_iterations_,
+ disable_cpu_flags_, benchmark_cpu_info_, 1.0f / 4096.0f, 4095, +1, 0);
+ EXPECT_EQ(0, diff);
+}
+
+TEST_F(LibYUVPlanarTest, TestHalfFloatPlane_Unaligned) {
+ int diff = TestHalfFloatPlane(
+ benchmark_width_, benchmark_height_, benchmark_iterations_,
+ disable_cpu_flags_, benchmark_cpu_info_, 1.0f / 4096.0f, 4095, +1, 2);
+ EXPECT_EQ(0, diff);
+}
+
+TEST_F(LibYUVPlanarTest, TestHalfFloatPlane_Invert) {
+ int diff = TestHalfFloatPlane(
+ benchmark_width_, benchmark_height_, benchmark_iterations_,
+ disable_cpu_flags_, benchmark_cpu_info_, 1.0f / 4096.0f, 4095, -1, 0);
EXPECT_EQ(0, diff);
}
TEST_F(LibYUVPlanarTest, TestHalfFloatPlane_Opt) {
- int diff = TestHalfFloatPlane(benchmark_width_, benchmark_height_,
- benchmark_iterations_, disable_cpu_flags_,
- benchmark_cpu_info_, 1.0f / 4096.0f, 4095);
+ int diff = TestHalfFloatPlane(
+ benchmark_width_, benchmark_height_, benchmark_iterations_,
+ disable_cpu_flags_, benchmark_cpu_info_, 1.0f / 4096.0f, 4095, +1, 0);
EXPECT_EQ(0, diff);
}
-TEST_F(LibYUVPlanarTest, TestHalfFloatPlane_Offby1) {
- int diff = TestHalfFloatPlane(benchmark_width_, benchmark_height_,
- benchmark_iterations_, disable_cpu_flags_,
- benchmark_cpu_info_, 1.0f / 4095.0f, 4095);
+TEST_F(LibYUVPlanarTest, TestHalfFloatPlane_16bit_denormal) {
+ int diff = TestHalfFloatPlane(
+ benchmark_width_, benchmark_height_, benchmark_iterations_,
+ disable_cpu_flags_, benchmark_cpu_info_, 1.0f / 65535.0f, 65535, +1, 0);
EXPECT_EQ(0, diff);
}
-TEST_F(LibYUVPlanarTest, TestHalfFloatPlane_One) {
- int diff = TestHalfFloatPlane(benchmark_width_, benchmark_height_,
- benchmark_iterations_, disable_cpu_flags_,
- benchmark_cpu_info_, 1.0f, 2047);
+#if defined(__arm__)
+static void EnableFlushDenormalToZero(void) {
+ uint32_t cw;
+ asm volatile(
+ "vmrs %0, fpscr \n"
+ "orr %0, %0, #0x1000000 \n"
+ "vmsr fpscr, %0 \n"
+ : "=r"(cw)::"memory", "cc"); // Clobber List
+}
+
+static void DisableFlushDenormalToZero(void) {
+ uint32_t cw;
+ asm volatile(
+ "vmrs %0, fpscr \n"
+ "bic %0, %0, #0x1000000 \n"
+ "vmsr fpscr, %0 \n"
+ : "=r"(cw)::"memory", "cc"); // Clobber List
+}
+
+// 5 bit exponent with bias of 15 will underflow to a denormal if scale causes
+// exponent to be less than 0. 15 - log2(65536) = -1/ This shouldnt normally
+// happen since scale is 1/(1<<bits) where bits is 9, 10 or 12.
+
+TEST_F(LibYUVPlanarTest, TestHalfFloatPlane_16bit_flush_denormal) {
+ // 32 bit arm rounding on denormal case is off by 1 compared to C.
+ EnableFlushDenormalToZero();
+ int diff = TestHalfFloatPlane(
+ benchmark_width_, benchmark_height_, benchmark_iterations_,
+ disable_cpu_flags_, benchmark_cpu_info_, 1.0f / 65535.0f, 65535, +1, 0);
+ DisableFlushDenormalToZero();
EXPECT_EQ(0, diff);
}
-TEST_F(LibYUVPlanarTest, TestHalfFloatPlane_12bit_One) {
- int diff = TestHalfFloatPlane(benchmark_width_, benchmark_height_,
- benchmark_iterations_, disable_cpu_flags_,
- benchmark_cpu_info_, 1.0f, 4095);
- EXPECT_LE(diff, 1);
+TEST_F(LibYUVPlanarTest, TestHalfFloatPlane_10bit_flush_denormal) {
+ EnableFlushDenormalToZero();
+ int diff = TestHalfFloatPlane(
+ benchmark_width_, benchmark_height_, benchmark_iterations_,
+ disable_cpu_flags_, benchmark_cpu_info_, 1.0f / 1023.0f, 1023, +1, 0);
+ DisableFlushDenormalToZero();
+ EXPECT_EQ(0, diff);
}
+#endif // defined(__arm__)
-float TestByteToFloat(int benchmark_width,
- int benchmark_height,
- int benchmark_iterations,
- int disable_cpu_flags,
- int benchmark_cpu_info,
- float scale) {
+static float TestByteToFloat(int benchmark_width,
+ int benchmark_height,
+ int benchmark_iterations,
+ int disable_cpu_flags,
+ int benchmark_cpu_info,
+ float scale) {
int i, j;
const int y_plane_size = benchmark_width * benchmark_height;
@@ -3146,35 +3162,44 @@
// Round count up to multiple of 16
const int kPixels = benchmark_width_ * benchmark_height_;
align_buffer_page_end(src_pixels, kPixels * 3);
- align_buffer_page_end(tmp_pixels_r, kPixels);
- align_buffer_page_end(tmp_pixels_g, kPixels);
- align_buffer_page_end(tmp_pixels_b, kPixels);
- align_buffer_page_end(dst_pixels_opt, kPixels * 3);
+ align_buffer_page_end(tmp_pixels_c_r, kPixels);
+ align_buffer_page_end(tmp_pixels_opt_r, kPixels);
+ align_buffer_page_end(tmp_pixels_c_g, kPixels);
+ align_buffer_page_end(tmp_pixels_opt_g, kPixels);
+ align_buffer_page_end(tmp_pixels_c_b, kPixels);
+ align_buffer_page_end(tmp_pixels_opt_b, kPixels);
align_buffer_page_end(dst_pixels_c, kPixels * 3);
+ align_buffer_page_end(dst_pixels_opt, kPixels * 3);
MemRandomize(src_pixels, kPixels * 3);
- MemRandomize(tmp_pixels_r, kPixels);
- MemRandomize(tmp_pixels_g, kPixels);
- MemRandomize(tmp_pixels_b, kPixels);
- MemRandomize(dst_pixels_opt, kPixels * 3);
+ MemRandomize(tmp_pixels_c_r, kPixels);
+ MemRandomize(tmp_pixels_opt_r, kPixels);
+ MemRandomize(tmp_pixels_c_g, kPixels);
+ MemRandomize(tmp_pixels_opt_g, kPixels);
+ MemRandomize(tmp_pixels_c_b, kPixels);
+ MemRandomize(tmp_pixels_opt_b, kPixels);
MemRandomize(dst_pixels_c, kPixels * 3);
+ MemRandomize(dst_pixels_opt, kPixels * 3);
MaskCpuFlags(disable_cpu_flags_);
- SplitRGBPlane(src_pixels, benchmark_width_ * 3, tmp_pixels_r,
- benchmark_width_, tmp_pixels_g, benchmark_width_, tmp_pixels_b,
- benchmark_width_, benchmark_width_, benchmark_height_);
- MergeRGBPlane(tmp_pixels_r, benchmark_width_, tmp_pixels_g, benchmark_width_,
- tmp_pixels_b, benchmark_width_, dst_pixels_c,
- benchmark_width_ * 3, benchmark_width_, benchmark_height_);
+ SplitRGBPlane(src_pixels, benchmark_width_ * 3, tmp_pixels_c_r,
+ benchmark_width_, tmp_pixels_c_g, benchmark_width_,
+ tmp_pixels_c_b, benchmark_width_, benchmark_width_,
+ benchmark_height_);
+ MergeRGBPlane(tmp_pixels_c_r, benchmark_width_, tmp_pixels_c_g,
+ benchmark_width_, tmp_pixels_c_b, benchmark_width_,
+ dst_pixels_c, benchmark_width_ * 3, benchmark_width_,
+ benchmark_height_);
MaskCpuFlags(benchmark_cpu_info_);
- SplitRGBPlane(src_pixels, benchmark_width_ * 3, tmp_pixels_r,
- benchmark_width_, tmp_pixels_g, benchmark_width_, tmp_pixels_b,
- benchmark_width_, benchmark_width_, benchmark_height_);
+ SplitRGBPlane(src_pixels, benchmark_width_ * 3, tmp_pixels_opt_r,
+ benchmark_width_, tmp_pixels_opt_g, benchmark_width_,
+ tmp_pixels_opt_b, benchmark_width_, benchmark_width_,
+ benchmark_height_);
for (int i = 0; i < benchmark_iterations_; ++i) {
- MergeRGBPlane(tmp_pixels_r, benchmark_width_, tmp_pixels_g,
- benchmark_width_, tmp_pixels_b, benchmark_width_,
+ MergeRGBPlane(tmp_pixels_opt_r, benchmark_width_, tmp_pixels_opt_g,
+ benchmark_width_, tmp_pixels_opt_b, benchmark_width_,
dst_pixels_opt, benchmark_width_ * 3, benchmark_width_,
benchmark_height_);
}
@@ -3184,99 +3209,122 @@
}
free_aligned_buffer_page_end(src_pixels);
- free_aligned_buffer_page_end(tmp_pixels_r);
- free_aligned_buffer_page_end(tmp_pixels_g);
- free_aligned_buffer_page_end(tmp_pixels_b);
- free_aligned_buffer_page_end(dst_pixels_opt);
+ free_aligned_buffer_page_end(tmp_pixels_c_r);
+ free_aligned_buffer_page_end(tmp_pixels_opt_r);
+ free_aligned_buffer_page_end(tmp_pixels_c_g);
+ free_aligned_buffer_page_end(tmp_pixels_opt_g);
+ free_aligned_buffer_page_end(tmp_pixels_c_b);
+ free_aligned_buffer_page_end(tmp_pixels_opt_b);
free_aligned_buffer_page_end(dst_pixels_c);
+ free_aligned_buffer_page_end(dst_pixels_opt);
}
TEST_F(LibYUVPlanarTest, SplitRGBPlane_Opt) {
// Round count up to multiple of 16
const int kPixels = benchmark_width_ * benchmark_height_;
align_buffer_page_end(src_pixels, kPixels * 3);
- align_buffer_page_end(tmp_pixels_r, kPixels);
- align_buffer_page_end(tmp_pixels_g, kPixels);
- align_buffer_page_end(tmp_pixels_b, kPixels);
- align_buffer_page_end(dst_pixels_opt, kPixels * 3);
+ align_buffer_page_end(tmp_pixels_c_r, kPixels);
+ align_buffer_page_end(tmp_pixels_opt_r, kPixels);
+ align_buffer_page_end(tmp_pixels_c_g, kPixels);
+ align_buffer_page_end(tmp_pixels_opt_g, kPixels);
+ align_buffer_page_end(tmp_pixels_c_b, kPixels);
+ align_buffer_page_end(tmp_pixels_opt_b, kPixels);
align_buffer_page_end(dst_pixels_c, kPixels * 3);
+ align_buffer_page_end(dst_pixels_opt, kPixels * 3);
MemRandomize(src_pixels, kPixels * 3);
- MemRandomize(tmp_pixels_r, kPixels);
- MemRandomize(tmp_pixels_g, kPixels);
- MemRandomize(tmp_pixels_b, kPixels);
- MemRandomize(dst_pixels_opt, kPixels * 3);
+ MemRandomize(tmp_pixels_c_r, kPixels);
+ MemRandomize(tmp_pixels_opt_r, kPixels);
+ MemRandomize(tmp_pixels_c_g, kPixels);
+ MemRandomize(tmp_pixels_opt_g, kPixels);
+ MemRandomize(tmp_pixels_c_b, kPixels);
+ MemRandomize(tmp_pixels_opt_b, kPixels);
MemRandomize(dst_pixels_c, kPixels * 3);
+ MemRandomize(dst_pixels_opt, kPixels * 3);
MaskCpuFlags(disable_cpu_flags_);
- SplitRGBPlane(src_pixels, benchmark_width_ * 3, tmp_pixels_r,
- benchmark_width_, tmp_pixels_g, benchmark_width_, tmp_pixels_b,
- benchmark_width_, benchmark_width_, benchmark_height_);
- MergeRGBPlane(tmp_pixels_r, benchmark_width_, tmp_pixels_g, benchmark_width_,
- tmp_pixels_b, benchmark_width_, dst_pixels_c,
- benchmark_width_ * 3, benchmark_width_, benchmark_height_);
+ SplitRGBPlane(src_pixels, benchmark_width_ * 3, tmp_pixels_c_r,
+ benchmark_width_, tmp_pixels_c_g, benchmark_width_,
+ tmp_pixels_c_b, benchmark_width_, benchmark_width_,
+ benchmark_height_);
+ MergeRGBPlane(tmp_pixels_c_r, benchmark_width_, tmp_pixels_c_g,
+ benchmark_width_, tmp_pixels_c_b, benchmark_width_,
+ dst_pixels_c, benchmark_width_ * 3, benchmark_width_,
+ benchmark_height_);
MaskCpuFlags(benchmark_cpu_info_);
for (int i = 0; i < benchmark_iterations_; ++i) {
- SplitRGBPlane(src_pixels, benchmark_width_ * 3, tmp_pixels_r,
- benchmark_width_, tmp_pixels_g, benchmark_width_,
- tmp_pixels_b, benchmark_width_, benchmark_width_,
+ SplitRGBPlane(src_pixels, benchmark_width_ * 3, tmp_pixels_opt_r,
+ benchmark_width_, tmp_pixels_opt_g, benchmark_width_,
+ tmp_pixels_opt_b, benchmark_width_, benchmark_width_,
benchmark_height_);
}
- MergeRGBPlane(tmp_pixels_r, benchmark_width_, tmp_pixels_g, benchmark_width_,
- tmp_pixels_b, benchmark_width_, dst_pixels_opt,
- benchmark_width_ * 3, benchmark_width_, benchmark_height_);
+ MergeRGBPlane(tmp_pixels_opt_r, benchmark_width_, tmp_pixels_opt_g,
+ benchmark_width_, tmp_pixels_opt_b, benchmark_width_,
+ dst_pixels_opt, benchmark_width_ * 3, benchmark_width_,
+ benchmark_height_);
for (int i = 0; i < kPixels * 3; ++i) {
EXPECT_EQ(dst_pixels_c[i], dst_pixels_opt[i]);
}
free_aligned_buffer_page_end(src_pixels);
- free_aligned_buffer_page_end(tmp_pixels_r);
- free_aligned_buffer_page_end(tmp_pixels_g);
- free_aligned_buffer_page_end(tmp_pixels_b);
- free_aligned_buffer_page_end(dst_pixels_opt);
+ free_aligned_buffer_page_end(tmp_pixels_c_r);
+ free_aligned_buffer_page_end(tmp_pixels_opt_r);
+ free_aligned_buffer_page_end(tmp_pixels_c_g);
+ free_aligned_buffer_page_end(tmp_pixels_opt_g);
+ free_aligned_buffer_page_end(tmp_pixels_c_b);
+ free_aligned_buffer_page_end(tmp_pixels_opt_b);
free_aligned_buffer_page_end(dst_pixels_c);
+ free_aligned_buffer_page_end(dst_pixels_opt);
}
TEST_F(LibYUVPlanarTest, MergeARGBPlane_Opt) {
const int kPixels = benchmark_width_ * benchmark_height_;
align_buffer_page_end(src_pixels, kPixels * 4);
- align_buffer_page_end(tmp_pixels_r, kPixels);
- align_buffer_page_end(tmp_pixels_g, kPixels);
- align_buffer_page_end(tmp_pixels_b, kPixels);
- align_buffer_page_end(tmp_pixels_a, kPixels);
- align_buffer_page_end(dst_pixels_opt, kPixels * 4);
+ align_buffer_page_end(tmp_pixels_c_r, kPixels);
+ align_buffer_page_end(tmp_pixels_opt_r, kPixels);
+ align_buffer_page_end(tmp_pixels_c_g, kPixels);
+ align_buffer_page_end(tmp_pixels_opt_g, kPixels);
+ align_buffer_page_end(tmp_pixels_c_b, kPixels);
+ align_buffer_page_end(tmp_pixels_opt_b, kPixels);
+ align_buffer_page_end(tmp_pixels_c_a, kPixels);
+ align_buffer_page_end(tmp_pixels_opt_a, kPixels);
align_buffer_page_end(dst_pixels_c, kPixels * 4);
+ align_buffer_page_end(dst_pixels_opt, kPixels * 4);
MemRandomize(src_pixels, kPixels * 4);
- MemRandomize(tmp_pixels_r, kPixels);
- MemRandomize(tmp_pixels_g, kPixels);
- MemRandomize(tmp_pixels_b, kPixels);
- MemRandomize(tmp_pixels_a, kPixels);
+ MemRandomize(tmp_pixels_c_r, kPixels);
+ MemRandomize(tmp_pixels_opt_r, kPixels);
+ MemRandomize(tmp_pixels_c_g, kPixels);
+ MemRandomize(tmp_pixels_opt_g, kPixels);
+ MemRandomize(tmp_pixels_c_b, kPixels);
+ MemRandomize(tmp_pixels_opt_b, kPixels);
+ MemRandomize(tmp_pixels_c_a, kPixels);
+ MemRandomize(tmp_pixels_opt_a, kPixels);
MemRandomize(dst_pixels_opt, kPixels * 4);
MemRandomize(dst_pixels_c, kPixels * 4);
MaskCpuFlags(disable_cpu_flags_);
- SplitARGBPlane(src_pixels, benchmark_width_ * 4, tmp_pixels_r,
- benchmark_width_, tmp_pixels_g, benchmark_width_, tmp_pixels_b,
- benchmark_width_, tmp_pixels_a, benchmark_width_,
- benchmark_width_, benchmark_height_);
- MergeARGBPlane(tmp_pixels_r, benchmark_width_, tmp_pixels_g, benchmark_width_,
- tmp_pixels_b, benchmark_width_, tmp_pixels_a, benchmark_width_,
- dst_pixels_c, benchmark_width_ * 4, benchmark_width_,
- benchmark_height_);
+ SplitARGBPlane(src_pixels, benchmark_width_ * 4, tmp_pixels_c_r,
+ benchmark_width_, tmp_pixels_c_g, benchmark_width_,
+ tmp_pixels_c_b, benchmark_width_, tmp_pixels_c_a,
+ benchmark_width_, benchmark_width_, benchmark_height_);
+ MergeARGBPlane(tmp_pixels_c_r, benchmark_width_, tmp_pixels_c_g,
+ benchmark_width_, tmp_pixels_c_b, benchmark_width_,
+ tmp_pixels_c_a, benchmark_width_, dst_pixels_c,
+ benchmark_width_ * 4, benchmark_width_, benchmark_height_);
MaskCpuFlags(benchmark_cpu_info_);
- SplitARGBPlane(src_pixels, benchmark_width_ * 4, tmp_pixels_r,
- benchmark_width_, tmp_pixels_g, benchmark_width_, tmp_pixels_b,
- benchmark_width_, tmp_pixels_a, benchmark_width_,
- benchmark_width_, benchmark_height_);
+ SplitARGBPlane(src_pixels, benchmark_width_ * 4, tmp_pixels_opt_r,
+ benchmark_width_, tmp_pixels_opt_g, benchmark_width_,
+ tmp_pixels_opt_b, benchmark_width_, tmp_pixels_opt_a,
+ benchmark_width_, benchmark_width_, benchmark_height_);
for (int i = 0; i < benchmark_iterations_; ++i) {
- MergeARGBPlane(tmp_pixels_r, benchmark_width_, tmp_pixels_g,
- benchmark_width_, tmp_pixels_b, benchmark_width_,
- tmp_pixels_a, benchmark_width_, dst_pixels_opt,
+ MergeARGBPlane(tmp_pixels_opt_r, benchmark_width_, tmp_pixels_opt_g,
+ benchmark_width_, tmp_pixels_opt_b, benchmark_width_,
+ tmp_pixels_opt_a, benchmark_width_, dst_pixels_opt,
benchmark_width_ * 4, benchmark_width_, benchmark_height_);
}
@@ -3285,10 +3333,14 @@
}
free_aligned_buffer_page_end(src_pixels);
- free_aligned_buffer_page_end(tmp_pixels_r);
- free_aligned_buffer_page_end(tmp_pixels_g);
- free_aligned_buffer_page_end(tmp_pixels_b);
- free_aligned_buffer_page_end(tmp_pixels_a);
+ free_aligned_buffer_page_end(tmp_pixels_c_r);
+ free_aligned_buffer_page_end(tmp_pixels_opt_r);
+ free_aligned_buffer_page_end(tmp_pixels_c_g);
+ free_aligned_buffer_page_end(tmp_pixels_opt_g);
+ free_aligned_buffer_page_end(tmp_pixels_c_b);
+ free_aligned_buffer_page_end(tmp_pixels_opt_b);
+ free_aligned_buffer_page_end(tmp_pixels_c_a);
+ free_aligned_buffer_page_end(tmp_pixels_opt_a);
free_aligned_buffer_page_end(dst_pixels_opt);
free_aligned_buffer_page_end(dst_pixels_c);
}
@@ -3296,41 +3348,171 @@
TEST_F(LibYUVPlanarTest, SplitARGBPlane_Opt) {
const int kPixels = benchmark_width_ * benchmark_height_;
align_buffer_page_end(src_pixels, kPixels * 4);
- align_buffer_page_end(tmp_pixels_r, kPixels);
- align_buffer_page_end(tmp_pixels_g, kPixels);
- align_buffer_page_end(tmp_pixels_b, kPixels);
- align_buffer_page_end(tmp_pixels_a, kPixels);
+ align_buffer_page_end(tmp_pixels_c_r, kPixels);
+ align_buffer_page_end(tmp_pixels_opt_r, kPixels);
+ align_buffer_page_end(tmp_pixels_c_g, kPixels);
+ align_buffer_page_end(tmp_pixels_opt_g, kPixels);
+ align_buffer_page_end(tmp_pixels_c_b, kPixels);
+ align_buffer_page_end(tmp_pixels_opt_b, kPixels);
+ align_buffer_page_end(tmp_pixels_c_a, kPixels);
+ align_buffer_page_end(tmp_pixels_opt_a, kPixels);
align_buffer_page_end(dst_pixels_opt, kPixels * 4);
align_buffer_page_end(dst_pixels_c, kPixels * 4);
MemRandomize(src_pixels, kPixels * 4);
- MemRandomize(tmp_pixels_r, kPixels);
- MemRandomize(tmp_pixels_g, kPixels);
- MemRandomize(tmp_pixels_b, kPixels);
- MemRandomize(tmp_pixels_a, kPixels);
+ MemRandomize(tmp_pixels_c_r, kPixels);
+ MemRandomize(tmp_pixels_opt_r, kPixels);
+ MemRandomize(tmp_pixels_c_g, kPixels);
+ MemRandomize(tmp_pixels_opt_g, kPixels);
+ MemRandomize(tmp_pixels_c_b, kPixels);
+ MemRandomize(tmp_pixels_opt_b, kPixels);
+ MemRandomize(tmp_pixels_c_a, kPixels);
+ MemRandomize(tmp_pixels_opt_a, kPixels);
MemRandomize(dst_pixels_opt, kPixels * 4);
MemRandomize(dst_pixels_c, kPixels * 4);
MaskCpuFlags(disable_cpu_flags_);
- SplitARGBPlane(src_pixels, benchmark_width_ * 4, tmp_pixels_r,
- benchmark_width_, tmp_pixels_g, benchmark_width_, tmp_pixels_b,
- benchmark_width_, tmp_pixels_a, benchmark_width_,
- benchmark_width_, benchmark_height_);
- MergeARGBPlane(tmp_pixels_r, benchmark_width_, tmp_pixels_g, benchmark_width_,
- tmp_pixels_b, benchmark_width_, tmp_pixels_a, benchmark_width_,
+ SplitARGBPlane(src_pixels, benchmark_width_ * 4, tmp_pixels_c_r,
+ benchmark_width_, tmp_pixels_c_g, benchmark_width_,
+ tmp_pixels_c_b, benchmark_width_, tmp_pixels_c_a,
+ benchmark_width_, benchmark_width_, benchmark_height_);
+ MergeARGBPlane(tmp_pixels_c_r, benchmark_width_, tmp_pixels_c_g,
+ benchmark_width_, tmp_pixels_c_b, benchmark_width_,
+ tmp_pixels_c_a, benchmark_width_, dst_pixels_c,
+ benchmark_width_ * 4, benchmark_width_, benchmark_height_);
+
+ MaskCpuFlags(benchmark_cpu_info_);
+ for (int i = 0; i < benchmark_iterations_; ++i) {
+ SplitARGBPlane(src_pixels, benchmark_width_ * 4, tmp_pixels_opt_r,
+ benchmark_width_, tmp_pixels_opt_g, benchmark_width_,
+ tmp_pixels_opt_b, benchmark_width_, tmp_pixels_opt_a,
+ benchmark_width_, benchmark_width_, benchmark_height_);
+ }
+
+ MergeARGBPlane(tmp_pixels_opt_r, benchmark_width_, tmp_pixels_opt_g,
+ benchmark_width_, tmp_pixels_opt_b, benchmark_width_,
+ tmp_pixels_opt_a, benchmark_width_, dst_pixels_opt,
+ benchmark_width_ * 4, benchmark_width_, benchmark_height_);
+
+ for (int i = 0; i < kPixels * 4; ++i) {
+ EXPECT_EQ(dst_pixels_c[i], dst_pixels_opt[i]);
+ }
+
+ free_aligned_buffer_page_end(src_pixels);
+ free_aligned_buffer_page_end(tmp_pixels_c_r);
+ free_aligned_buffer_page_end(tmp_pixels_opt_r);
+ free_aligned_buffer_page_end(tmp_pixels_c_g);
+ free_aligned_buffer_page_end(tmp_pixels_opt_g);
+ free_aligned_buffer_page_end(tmp_pixels_c_b);
+ free_aligned_buffer_page_end(tmp_pixels_opt_b);
+ free_aligned_buffer_page_end(tmp_pixels_c_a);
+ free_aligned_buffer_page_end(tmp_pixels_opt_a);
+ free_aligned_buffer_page_end(dst_pixels_c);
+ free_aligned_buffer_page_end(dst_pixels_opt);
+}
+
+TEST_F(LibYUVPlanarTest, MergeXRGBPlane_Opt) {
+ const int kPixels = benchmark_width_ * benchmark_height_;
+ align_buffer_page_end(src_pixels, kPixels * 4);
+ align_buffer_page_end(tmp_pixels_c_r, kPixels);
+ align_buffer_page_end(tmp_pixels_opt_r, kPixels);
+ align_buffer_page_end(tmp_pixels_c_g, kPixels);
+ align_buffer_page_end(tmp_pixels_opt_g, kPixels);
+ align_buffer_page_end(tmp_pixels_c_b, kPixels);
+ align_buffer_page_end(tmp_pixels_opt_b, kPixels);
+ align_buffer_page_end(dst_pixels_c, kPixels * 4);
+ align_buffer_page_end(dst_pixels_opt, kPixels * 4);
+
+ MemRandomize(src_pixels, kPixels * 4);
+ MemRandomize(tmp_pixels_c_r, kPixels);
+ MemRandomize(tmp_pixels_opt_r, kPixels);
+ MemRandomize(tmp_pixels_c_g, kPixels);
+ MemRandomize(tmp_pixels_opt_g, kPixels);
+ MemRandomize(tmp_pixels_c_b, kPixels);
+ MemRandomize(tmp_pixels_opt_b, kPixels);
+ MemRandomize(dst_pixels_c, kPixels * 4);
+ MemRandomize(dst_pixels_opt, kPixels * 4);
+
+ MaskCpuFlags(disable_cpu_flags_);
+ SplitARGBPlane(src_pixels, benchmark_width_ * 4, tmp_pixels_c_r,
+ benchmark_width_, tmp_pixels_c_g, benchmark_width_,
+ tmp_pixels_c_b, benchmark_width_, NULL, 0, benchmark_width_,
+ benchmark_height_);
+ MergeARGBPlane(tmp_pixels_c_r, benchmark_width_, tmp_pixels_c_g,
+ benchmark_width_, tmp_pixels_c_b, benchmark_width_, NULL, 0,
+ dst_pixels_c, benchmark_width_ * 4, benchmark_width_,
+ benchmark_height_);
+
+ MaskCpuFlags(benchmark_cpu_info_);
+ SplitARGBPlane(src_pixels, benchmark_width_ * 4, tmp_pixels_opt_r,
+ benchmark_width_, tmp_pixels_opt_g, benchmark_width_,
+ tmp_pixels_opt_b, benchmark_width_, NULL, 0, benchmark_width_,
+ benchmark_height_);
+
+ for (int i = 0; i < benchmark_iterations_; ++i) {
+ MergeARGBPlane(tmp_pixels_opt_r, benchmark_width_, tmp_pixels_opt_g,
+ benchmark_width_, tmp_pixels_opt_b, benchmark_width_, NULL,
+ 0, dst_pixels_opt, benchmark_width_ * 4, benchmark_width_,
+ benchmark_height_);
+ }
+
+ for (int i = 0; i < kPixels * 4; ++i) {
+ EXPECT_EQ(dst_pixels_c[i], dst_pixels_opt[i]);
+ }
+
+ free_aligned_buffer_page_end(src_pixels);
+ free_aligned_buffer_page_end(tmp_pixels_c_r);
+ free_aligned_buffer_page_end(tmp_pixels_opt_r);
+ free_aligned_buffer_page_end(tmp_pixels_c_g);
+ free_aligned_buffer_page_end(tmp_pixels_opt_g);
+ free_aligned_buffer_page_end(tmp_pixels_c_b);
+ free_aligned_buffer_page_end(tmp_pixels_opt_b);
+ free_aligned_buffer_page_end(dst_pixels_c);
+ free_aligned_buffer_page_end(dst_pixels_opt);
+}
+
+TEST_F(LibYUVPlanarTest, SplitXRGBPlane_Opt) {
+ const int kPixels = benchmark_width_ * benchmark_height_;
+ align_buffer_page_end(src_pixels, kPixels * 4);
+ align_buffer_page_end(tmp_pixels_c_r, kPixels);
+ align_buffer_page_end(tmp_pixels_opt_r, kPixels);
+ align_buffer_page_end(tmp_pixels_c_g, kPixels);
+ align_buffer_page_end(tmp_pixels_opt_g, kPixels);
+ align_buffer_page_end(tmp_pixels_c_b, kPixels);
+ align_buffer_page_end(tmp_pixels_opt_b, kPixels);
+ align_buffer_page_end(dst_pixels_c, kPixels * 4);
+ align_buffer_page_end(dst_pixels_opt, kPixels * 4);
+
+ MemRandomize(src_pixels, kPixels * 4);
+ MemRandomize(tmp_pixels_c_r, kPixels);
+ MemRandomize(tmp_pixels_opt_r, kPixels);
+ MemRandomize(tmp_pixels_c_g, kPixels);
+ MemRandomize(tmp_pixels_opt_g, kPixels);
+ MemRandomize(tmp_pixels_c_b, kPixels);
+ MemRandomize(tmp_pixels_opt_b, kPixels);
+ MemRandomize(dst_pixels_c, kPixels * 4);
+ MemRandomize(dst_pixels_opt, kPixels * 4);
+
+ MaskCpuFlags(disable_cpu_flags_);
+ SplitARGBPlane(src_pixels, benchmark_width_ * 4, tmp_pixels_c_r,
+ benchmark_width_, tmp_pixels_c_g, benchmark_width_,
+ tmp_pixels_c_b, benchmark_width_, NULL, 0, benchmark_width_,
+ benchmark_height_);
+ MergeARGBPlane(tmp_pixels_c_r, benchmark_width_, tmp_pixels_c_g,
+ benchmark_width_, tmp_pixels_c_b, benchmark_width_, NULL, 0,
dst_pixels_c, benchmark_width_ * 4, benchmark_width_,
benchmark_height_);
MaskCpuFlags(benchmark_cpu_info_);
for (int i = 0; i < benchmark_iterations_; ++i) {
- SplitARGBPlane(src_pixels, benchmark_width_ * 4, tmp_pixels_r,
- benchmark_width_, tmp_pixels_g, benchmark_width_,
- tmp_pixels_b, benchmark_width_, tmp_pixels_a,
- benchmark_width_, benchmark_width_, benchmark_height_);
+ SplitARGBPlane(src_pixels, benchmark_width_ * 4, tmp_pixels_opt_r,
+ benchmark_width_, tmp_pixels_opt_g, benchmark_width_,
+ tmp_pixels_opt_b, benchmark_width_, NULL, 0,
+ benchmark_width_, benchmark_height_);
}
- MergeARGBPlane(tmp_pixels_r, benchmark_width_, tmp_pixels_g, benchmark_width_,
- tmp_pixels_b, benchmark_width_, tmp_pixels_a, benchmark_width_,
+ MergeARGBPlane(tmp_pixels_opt_r, benchmark_width_, tmp_pixels_opt_g,
+ benchmark_width_, tmp_pixels_opt_b, benchmark_width_, NULL, 0,
dst_pixels_opt, benchmark_width_ * 4, benchmark_width_,
benchmark_height_);
@@ -3339,111 +3521,14 @@
}
free_aligned_buffer_page_end(src_pixels);
- free_aligned_buffer_page_end(tmp_pixels_r);
- free_aligned_buffer_page_end(tmp_pixels_g);
- free_aligned_buffer_page_end(tmp_pixels_b);
- free_aligned_buffer_page_end(tmp_pixels_a);
- free_aligned_buffer_page_end(dst_pixels_opt);
+ free_aligned_buffer_page_end(tmp_pixels_c_r);
+ free_aligned_buffer_page_end(tmp_pixels_opt_r);
+ free_aligned_buffer_page_end(tmp_pixels_c_g);
+ free_aligned_buffer_page_end(tmp_pixels_opt_g);
+ free_aligned_buffer_page_end(tmp_pixels_c_b);
+ free_aligned_buffer_page_end(tmp_pixels_opt_b);
free_aligned_buffer_page_end(dst_pixels_c);
-}
-
-TEST_F(LibYUVPlanarTest, MergeXRGBPlane_Opt) {
- const int kPixels = benchmark_width_ * benchmark_height_;
- align_buffer_page_end(src_pixels, kPixels * 4);
- align_buffer_page_end(tmp_pixels_r, kPixels);
- align_buffer_page_end(tmp_pixels_g, kPixels);
- align_buffer_page_end(tmp_pixels_b, kPixels);
- align_buffer_page_end(dst_pixels_opt, kPixels * 4);
- align_buffer_page_end(dst_pixels_c, kPixels * 4);
-
- MemRandomize(src_pixels, kPixels * 4);
- MemRandomize(tmp_pixels_r, kPixels);
- MemRandomize(tmp_pixels_g, kPixels);
- MemRandomize(tmp_pixels_b, kPixels);
- MemRandomize(dst_pixels_opt, kPixels * 4);
- MemRandomize(dst_pixels_c, kPixels * 4);
-
- MaskCpuFlags(disable_cpu_flags_);
- SplitARGBPlane(src_pixels, benchmark_width_ * 4, tmp_pixels_r,
- benchmark_width_, tmp_pixels_g, benchmark_width_, tmp_pixels_b,
- benchmark_width_, NULL, 0, benchmark_width_,
- benchmark_height_);
- MergeARGBPlane(tmp_pixels_r, benchmark_width_, tmp_pixels_g, benchmark_width_,
- tmp_pixels_b, benchmark_width_, NULL, 0, dst_pixels_c,
- benchmark_width_ * 4, benchmark_width_, benchmark_height_);
-
- MaskCpuFlags(benchmark_cpu_info_);
- SplitARGBPlane(src_pixels, benchmark_width_ * 4, tmp_pixels_r,
- benchmark_width_, tmp_pixels_g, benchmark_width_, tmp_pixels_b,
- benchmark_width_, NULL, 0, benchmark_width_,
- benchmark_height_);
-
- for (int i = 0; i < benchmark_iterations_; ++i) {
- MergeARGBPlane(tmp_pixels_r, benchmark_width_, tmp_pixels_g,
- benchmark_width_, tmp_pixels_b, benchmark_width_, NULL, 0,
- dst_pixels_opt, benchmark_width_ * 4, benchmark_width_,
- benchmark_height_);
- }
-
- for (int i = 0; i < kPixels * 4; ++i) {
- EXPECT_EQ(dst_pixels_c[i], dst_pixels_opt[i]);
- }
-
- free_aligned_buffer_page_end(src_pixels);
- free_aligned_buffer_page_end(tmp_pixels_r);
- free_aligned_buffer_page_end(tmp_pixels_g);
- free_aligned_buffer_page_end(tmp_pixels_b);
free_aligned_buffer_page_end(dst_pixels_opt);
- free_aligned_buffer_page_end(dst_pixels_c);
-}
-
-TEST_F(LibYUVPlanarTest, SplitXRGBPlane_Opt) {
- const int kPixels = benchmark_width_ * benchmark_height_;
- align_buffer_page_end(src_pixels, kPixels * 4);
- align_buffer_page_end(tmp_pixels_r, kPixels);
- align_buffer_page_end(tmp_pixels_g, kPixels);
- align_buffer_page_end(tmp_pixels_b, kPixels);
- align_buffer_page_end(dst_pixels_opt, kPixels * 4);
- align_buffer_page_end(dst_pixels_c, kPixels * 4);
-
- MemRandomize(src_pixels, kPixels * 4);
- MemRandomize(tmp_pixels_r, kPixels);
- MemRandomize(tmp_pixels_g, kPixels);
- MemRandomize(tmp_pixels_b, kPixels);
- MemRandomize(dst_pixels_opt, kPixels * 4);
- MemRandomize(dst_pixels_c, kPixels * 4);
-
- MaskCpuFlags(disable_cpu_flags_);
- SplitARGBPlane(src_pixels, benchmark_width_ * 4, tmp_pixels_r,
- benchmark_width_, tmp_pixels_g, benchmark_width_, tmp_pixels_b,
- benchmark_width_, NULL, 0, benchmark_width_,
- benchmark_height_);
- MergeARGBPlane(tmp_pixels_r, benchmark_width_, tmp_pixels_g, benchmark_width_,
- tmp_pixels_b, benchmark_width_, NULL, 0, dst_pixels_c,
- benchmark_width_ * 4, benchmark_width_, benchmark_height_);
-
- MaskCpuFlags(benchmark_cpu_info_);
- for (int i = 0; i < benchmark_iterations_; ++i) {
- SplitARGBPlane(src_pixels, benchmark_width_ * 4, tmp_pixels_r,
- benchmark_width_, tmp_pixels_g, benchmark_width_,
- tmp_pixels_b, benchmark_width_, NULL, 0, benchmark_width_,
- benchmark_height_);
- }
-
- MergeARGBPlane(tmp_pixels_r, benchmark_width_, tmp_pixels_g, benchmark_width_,
- tmp_pixels_b, benchmark_width_, NULL, 0, dst_pixels_opt,
- benchmark_width_ * 4, benchmark_width_, benchmark_height_);
-
- for (int i = 0; i < kPixels * 4; ++i) {
- EXPECT_EQ(dst_pixels_c[i], dst_pixels_opt[i]);
- }
-
- free_aligned_buffer_page_end(src_pixels);
- free_aligned_buffer_page_end(tmp_pixels_r);
- free_aligned_buffer_page_end(tmp_pixels_g);
- free_aligned_buffer_page_end(tmp_pixels_b);
- free_aligned_buffer_page_end(dst_pixels_opt);
- free_aligned_buffer_page_end(dst_pixels_c);
}
// Merge 4 channels
@@ -3717,6 +3802,37 @@
free_aligned_buffer_page_end(dst_pixels_y_c);
}
+TEST_F(LibYUVPlanarTest, Convert8To8Plane) {
+ const int kPixels = benchmark_width_ * benchmark_height_;
+ align_buffer_page_end(src_pixels_y, kPixels);
+ align_buffer_page_end(dst_pixels_y_opt, kPixels);
+ align_buffer_page_end(dst_pixels_y_c, kPixels);
+
+ MemRandomize(src_pixels_y, kPixels);
+ memset(dst_pixels_y_opt, 0, kPixels);
+ memset(dst_pixels_y_c, 1, kPixels);
+
+ MaskCpuFlags(disable_cpu_flags_);
+ Convert8To8Plane(src_pixels_y, benchmark_width_, dst_pixels_y_c,
+ benchmark_width_, 220, 16, benchmark_width_,
+ benchmark_height_);
+ MaskCpuFlags(benchmark_cpu_info_);
+
+ for (int i = 0; i < benchmark_iterations_; ++i) {
+ Convert8To8Plane(src_pixels_y, benchmark_width_, dst_pixels_y_opt,
+ benchmark_width_, 220, 16, benchmark_width_,
+ benchmark_height_);
+ }
+
+ for (int i = 0; i < kPixels; ++i) {
+ EXPECT_EQ(dst_pixels_y_opt[i], dst_pixels_y_c[i]);
+ }
+
+ free_aligned_buffer_page_end(src_pixels_y);
+ free_aligned_buffer_page_end(dst_pixels_y_opt);
+ free_aligned_buffer_page_end(dst_pixels_y_c);
+}
+
TEST_F(LibYUVPlanarTest, YUY2ToY) {
const int kPixels = benchmark_width_ * benchmark_height_;
align_buffer_page_end(src_pixels_y, kPixels * 2);
@@ -4437,15 +4553,11 @@
int dst_height = (benchmark_height_ + 1) / 2;
align_buffer_page_end(src_pixels_u, benchmark_width_ * benchmark_height_);
align_buffer_page_end(src_pixels_v, benchmark_width_ * benchmark_height_);
- align_buffer_page_end(tmp_pixels_u, dst_width * dst_height);
- align_buffer_page_end(tmp_pixels_v, dst_width * dst_height);
align_buffer_page_end(dst_pixels_uv_opt, dst_width * 2 * dst_height);
align_buffer_page_end(dst_pixels_uv_c, dst_width * 2 * dst_height);
MemRandomize(src_pixels_u, benchmark_width_ * benchmark_height_);
MemRandomize(src_pixels_v, benchmark_width_ * benchmark_height_);
- MemRandomize(tmp_pixels_u, dst_width * dst_height);
- MemRandomize(tmp_pixels_v, dst_width * dst_height);
MemRandomize(dst_pixels_uv_opt, dst_width * 2 * dst_height);
MemRandomize(dst_pixels_uv_c, dst_width * 2 * dst_height);
@@ -4467,8 +4579,6 @@
free_aligned_buffer_page_end(src_pixels_u);
free_aligned_buffer_page_end(src_pixels_v);
- free_aligned_buffer_page_end(tmp_pixels_u);
- free_aligned_buffer_page_end(tmp_pixels_v);
free_aligned_buffer_page_end(dst_pixels_uv_opt);
free_aligned_buffer_page_end(dst_pixels_uv_c);
}
diff --git a/unit_test/rotate_argb_test.cc b/unit_test/rotate_argb_test.cc
index 74952c4..4c7b0b2 100644
--- a/unit_test/rotate_argb_test.cc
+++ b/unit_test/rotate_argb_test.cc
@@ -16,15 +16,15 @@
namespace libyuv {
-void TestRotateBpp(int src_width,
- int src_height,
- int dst_width,
- int dst_height,
- libyuv::RotationMode mode,
- int benchmark_iterations,
- int disable_cpu_flags,
- int benchmark_cpu_info,
- const int kBpp) {
+static void TestRotateBpp(int src_width,
+ int src_height,
+ int dst_width,
+ int dst_height,
+ libyuv::RotationMode mode,
+ int benchmark_iterations,
+ int disable_cpu_flags,
+ int benchmark_cpu_info,
+ const int kBpp) {
if (src_width < 1) {
src_width = 1;
}
diff --git a/unit_test/scale_argb_test.cc b/unit_test/scale_argb_test.cc
index f54a68f..66fd4cf 100644
--- a/unit_test/scale_argb_test.cc
+++ b/unit_test/scale_argb_test.cc
@@ -320,16 +320,18 @@
#ifndef DISABLE_SLOW_TESTS
// Test scale to a specified size with all 4 filters.
-#define TEST_SCALETO(name, width, height) \
- TEST_SCALETO1(, name, width, height, None, 0) \
- TEST_SCALETO1(, name, width, height, Linear, 3) \
- TEST_SCALETO1(, name, width, height, Bilinear, 3)
+#define TEST_SCALETO(name, width, height) \
+ TEST_SCALETO1(, name, width, height, None, 0) \
+ TEST_SCALETO1(, name, width, height, Linear, 3) \
+ TEST_SCALETO1(, name, width, height, Bilinear, 3) \
+ TEST_SCALETO1(, name, width, height, Box, 3)
#else
#if defined(ENABLE_FULL_TESTS)
-#define TEST_SCALETO(name, width, height) \
- TEST_SCALETO1(DISABLED_, name, width, height, None, 0) \
- TEST_SCALETO1(DISABLED_, name, width, height, Linear, 3) \
- TEST_SCALETO1(DISABLED_, name, width, height, Bilinear, 3)
+#define TEST_SCALETO(name, width, height) \
+ TEST_SCALETO1(DISABLED_, name, width, height, None, 0) \
+ TEST_SCALETO1(DISABLED_, name, width, height, Linear, 3) \
+ TEST_SCALETO1(DISABLED_, name, width, height, Bilinear, 3) \
+ TEST_SCALETO1(DISABLED_, name, width, height, Box, 3)
#else
#define TEST_SCALETO(name, width, height) \
TEST_SCALETO1(DISABLED_, name, width, height, Bilinear, 3)
@@ -340,6 +342,7 @@
TEST_SCALETO(ARGBScale, 569, 480)
TEST_SCALETO(ARGBScale, 640, 360)
#ifndef DISABLE_SLOW_TESTS
+TEST_SCALETO(ARGBScale, 50, 1)
TEST_SCALETO(ARGBScale, 256, 144) /* 128x72 * 2 */
TEST_SCALETO(ARGBScale, 320, 240)
TEST_SCALETO(ARGBScale, 1280, 720)
@@ -369,26 +372,25 @@
// Scale with YUV conversion to ARGB and clipping.
// TODO(fbarchard): Add fourcc support. All 4 ARGB formats is easy to support.
-LIBYUV_API
-int YUVToARGBScaleReference2(const uint8_t* src_y,
- int src_stride_y,
- const uint8_t* src_u,
- int src_stride_u,
- const uint8_t* src_v,
- int src_stride_v,
- uint32_t /* src_fourcc */,
- int src_width,
- int src_height,
- uint8_t* dst_argb,
- int dst_stride_argb,
- uint32_t /* dst_fourcc */,
- int dst_width,
- int dst_height,
- int clip_x,
- int clip_y,
- int clip_width,
- int clip_height,
- enum FilterMode filtering) {
+static int YUVToARGBScaleReference2(const uint8_t* src_y,
+ int src_stride_y,
+ const uint8_t* src_u,
+ int src_stride_u,
+ const uint8_t* src_v,
+ int src_stride_v,
+ uint32_t /* src_fourcc */,
+ int src_width,
+ int src_height,
+ uint8_t* dst_argb,
+ int dst_stride_argb,
+ uint32_t /* dst_fourcc */,
+ int dst_width,
+ int dst_height,
+ int clip_x,
+ int clip_y,
+ int clip_width,
+ int clip_height,
+ enum FilterMode filtering) {
uint8_t* argb_buffer =
static_cast<uint8_t*>(malloc(src_width * src_height * 4));
int r;
diff --git a/unit_test/scale_plane_test.cc b/unit_test/scale_plane_test.cc
index 9ce47a0..3df71a5 100644
--- a/unit_test/scale_plane_test.cc
+++ b/unit_test/scale_plane_test.cc
@@ -128,11 +128,6 @@
}
#endif // HAS_SCALEROWDOWN2_SSSE3
-extern "C" void ScaleRowDown2Box_16_NEON(const uint16_t* src_ptr,
- ptrdiff_t src_stride,
- uint16_t* dst,
- int dst_width);
-
TEST_F(LibYUVScaleTest, TestScaleRowDown2Box_16) {
SIMD_ALIGNED(uint16_t orig_pixels[2560 * 2]);
SIMD_ALIGNED(uint16_t dst_pixels_c[1280]);
diff --git a/unit_test/scale_test.cc b/unit_test/scale_test.cc
index 6e3b927..fd8fff8 100644
--- a/unit_test/scale_test.cc
+++ b/unit_test/scale_test.cc
@@ -1053,10 +1053,12 @@
TEST_SCALETO(Scale, 640, 360)
#ifndef DISABLE_SLOW_TESTS
TEST_SCALETO(Scale, 256, 144) /* 128x72 * 2 */
+TEST_SCALETO(Scale, 264, 216)
TEST_SCALETO(Scale, 320, 240)
TEST_SCALETO(Scale, 1280, 720)
TEST_SCALETO(Scale, 1920, 1080)
-#endif // DISABLE_SLOW_TESTS
+TEST_SCALETO(Scale, 1080, 1920) // for rotated phones
+#endif // DISABLE_SLOW_TESTS
#undef TEST_SCALETO1
#undef TEST_SCALETO
diff --git a/unit_test/testdata/juno.txt b/unit_test/testdata/juno.txt
deleted file mode 100644
index dd46527..0000000
--- a/unit_test/testdata/juno.txt
+++ /dev/null
@@ -1,15 +0,0 @@
-Processor : AArch64 Processor rev 0 (aarch64)
-processor : 0
-processor : 1
-processor : 2
-processor : 3
-processor : 4
-processor : 5
-Features : fp asimd evtstrm aes pmull sha1 sha2 crc32
-CPU implementer : 0x41
-CPU architecture: AArch64
-CPU variant : 0x0
-CPU part : 0xd07
-CPU revision : 0
-
-Hardware : Juno
diff --git a/unit_test/unit_test.cc b/unit_test/unit_test.cc
index 239d5b9..d917f53 100644
--- a/unit_test/unit_test.cc
+++ b/unit_test/unit_test.cc
@@ -67,12 +67,29 @@
}
#endif
-int TestCpuEnv(int cpu_info) {
+static int TestCpuEnv(int cpu_info) {
#if defined(__arm__) || defined(__aarch64__)
if (TestEnv("LIBYUV_DISABLE_NEON")) {
cpu_info &= ~libyuv::kCpuHasNEON;
}
#endif
+#if defined(__aarch64__)
+ if (TestEnv("LIBYUV_DISABLE_NEON_DOTPROD")) {
+ cpu_info &= ~libyuv::kCpuHasNeonDotProd;
+ }
+ if (TestEnv("LIBYUV_DISABLE_NEON_I8MM")) {
+ cpu_info &= ~libyuv::kCpuHasNeonI8MM;
+ }
+ if (TestEnv("LIBYUV_DISABLE_SVE")) {
+ cpu_info &= ~libyuv::kCpuHasSVE;
+ }
+ if (TestEnv("LIBYUV_DISABLE_SVE2")) {
+ cpu_info &= ~libyuv::kCpuHasSVE2;
+ }
+ if (TestEnv("LIBYUV_DISABLE_SME")) {
+ cpu_info &= ~libyuv::kCpuHasSME;
+ }
+#endif
#if defined(__mips__) && defined(__linux__)
if (TestEnv("LIBYUV_DISABLE_MSA")) {
cpu_info &= ~libyuv::kCpuHasMSA;
@@ -147,12 +164,18 @@
if (TestEnv("LIBYUV_DISABLE_AVX10")) {
cpu_info &= ~libyuv::kCpuHasAVX10;
}
+ if (TestEnv("LIBYUV_DISABLE_AVX10_2")) {
+ cpu_info &= ~libyuv::kCpuHasAVX10_2;
+ }
if (TestEnv("LIBYUV_DISABLE_AVXVNNI")) {
cpu_info &= ~libyuv::kCpuHasAVXVNNI;
}
if (TestEnv("LIBYUV_DISABLE_AVXVNNIINT8")) {
cpu_info &= ~libyuv::kCpuHasAVXVNNIINT8;
}
+ if (TestEnv("LIBYUV_DISABLE_AMXINT8")) {
+ cpu_info &= ~libyuv::kCpuHasAMXINT8;
+ }
#endif
if (TestEnv("LIBYUV_DISABLE_ASM")) {
cpu_info = libyuv::kCpuInitialized;
@@ -204,7 +227,7 @@
}
const char* cpu_info = getenv("LIBYUV_CPU_INFO");
if (cpu_info) {
- benchmark_cpu_info_ = atoi(cpu_flags); // NOLINT
+ benchmark_cpu_info_ = atoi(cpu_info); // NOLINT
}
if (LIBYUV_GET_FLAG(FLAGS_libyuv_cpu_info)) {
benchmark_cpu_info_ = LIBYUV_GET_FLAG(FLAGS_libyuv_cpu_info);
@@ -260,7 +283,7 @@
}
const char* cpu_info = getenv("LIBYUV_CPU_INFO");
if (cpu_info) {
- benchmark_cpu_info_ = atoi(cpu_flags); // NOLINT
+ benchmark_cpu_info_ = atoi(cpu_info); // NOLINT
}
if (LIBYUV_GET_FLAG(FLAGS_libyuv_cpu_info)) {
benchmark_cpu_info_ = LIBYUV_GET_FLAG(FLAGS_libyuv_cpu_info);
@@ -316,7 +339,7 @@
}
const char* cpu_info = getenv("LIBYUV_CPU_INFO");
if (cpu_info) {
- benchmark_cpu_info_ = atoi(cpu_flags); // NOLINT
+ benchmark_cpu_info_ = atoi(cpu_info); // NOLINT
}
if (LIBYUV_GET_FLAG(FLAGS_libyuv_cpu_info)) {
benchmark_cpu_info_ = LIBYUV_GET_FLAG(FLAGS_libyuv_cpu_info);
@@ -372,7 +395,7 @@
}
const char* cpu_info = getenv("LIBYUV_CPU_INFO");
if (cpu_info) {
- benchmark_cpu_info_ = atoi(cpu_flags); // NOLINT
+ benchmark_cpu_info_ = atoi(cpu_info); // NOLINT
}
if (LIBYUV_GET_FLAG(FLAGS_libyuv_cpu_info)) {
benchmark_cpu_info_ = LIBYUV_GET_FLAG(FLAGS_libyuv_cpu_info);
@@ -428,7 +451,7 @@
}
const char* cpu_info = getenv("LIBYUV_CPU_INFO");
if (cpu_info) {
- benchmark_cpu_info_ = atoi(cpu_flags); // NOLINT
+ benchmark_cpu_info_ = atoi(cpu_info); // NOLINT
}
if (LIBYUV_GET_FLAG(FLAGS_libyuv_cpu_info)) {
benchmark_cpu_info_ = LIBYUV_GET_FLAG(FLAGS_libyuv_cpu_info);
@@ -484,7 +507,7 @@
}
const char* cpu_info = getenv("LIBYUV_CPU_INFO");
if (cpu_info) {
- benchmark_cpu_info_ = atoi(cpu_flags); // NOLINT
+ benchmark_cpu_info_ = atoi(cpu_info); // NOLINT
}
if (LIBYUV_GET_FLAG(FLAGS_libyuv_cpu_info)) {
benchmark_cpu_info_ = LIBYUV_GET_FLAG(FLAGS_libyuv_cpu_info);
@@ -540,7 +563,7 @@
}
const char* cpu_info = getenv("LIBYUV_CPU_INFO");
if (cpu_info) {
- benchmark_cpu_info_ = atoi(cpu_flags); // NOLINT
+ benchmark_cpu_info_ = atoi(cpu_info); // NOLINT
}
if (LIBYUV_GET_FLAG(FLAGS_libyuv_cpu_info)) {
benchmark_cpu_info_ = LIBYUV_GET_FLAG(FLAGS_libyuv_cpu_info);
diff --git a/unit_test/unit_test.h b/unit_test/unit_test.h
index 99cc8d1..7440cc7 100644
--- a/unit_test/unit_test.h
+++ b/unit_test/unit_test.h
@@ -70,9 +70,11 @@
}
#define align_buffer_page_end(var, size) \
+ uint8_t* var = NULL; \
uint8_t* var##_mem = \
reinterpret_cast<uint8_t*>(malloc(((size) + 4095 + 63) & ~4095)); \
- uint8_t* var = reinterpret_cast<uint8_t*>( \
+ if (var##_mem) \
+ var = reinterpret_cast<uint8_t*>( \
(intptr_t)(var##_mem + (((size) + 4095 + 63) & ~4095) - (size)) & ~63)
#define free_aligned_buffer_page_end(var) \
@@ -80,9 +82,11 @@
var = NULL
#define align_buffer_page_end_16(var, size) \
+ uint16_t* var = NULL; \
uint8_t* var##_mem = \
reinterpret_cast<uint8_t*>(malloc(((size)*2 + 4095 + 63) & ~4095)); \
- uint16_t* var = reinterpret_cast<uint16_t*>( \
+ if (var##_mem) \
+ var = reinterpret_cast<uint16_t*>( \
(intptr_t)(var##_mem + (((size)*2 + 4095 + 63) & ~4095) - (size)*2) & \
~63)
diff --git a/util/cpuid.c b/util/cpuid.c
index c07e6e9..0f54b42 100644
--- a/util/cpuid.c
+++ b/util/cpuid.c
@@ -12,24 +12,126 @@
#include <stdlib.h>
#include <string.h>
+#ifdef __linux__
+#include <ctype.h>
+#include <sys/utsname.h>
+#endif
+
#include "libyuv/cpu_id.h"
#ifdef __cplusplus
using namespace libyuv;
#endif
+#ifdef __linux__
+static void KernelVersion(int *version) {
+ struct utsname buffer;
+ int i = 0;
+
+ version[0] = version[1] = 0;
+ if (uname(&buffer) == 0) {
+ char *v = buffer.release;
+ for (i = 0; *v && i < 2; ++v) {
+ if (isdigit(*v)) {
+ version[i++] = (int) strtol(v, &v, 10);
+ }
+ }
+ }
+}
+#endif
+
int main(int argc, const char* argv[]) {
- int cpu_flags = TestCpuFlag(-1);
- int has_arm = TestCpuFlag(kCpuHasARM);
- int has_riscv = TestCpuFlag(kCpuHasRISCV);
- int has_x86 = TestCpuFlag(kCpuHasX86);
- int has_mips = TestCpuFlag(kCpuHasMIPS);
- int has_loongarch = TestCpuFlag(kCpuHasLOONGARCH);
(void)argc;
(void)argv;
+#if defined(__linux__)
+ {
+ int kernelversion[2];
+ KernelVersion(kernelversion);
+ printf("Kernel Version %d.%d\n", kernelversion[0], kernelversion[1]);
+ }
+#endif // defined(__linux__)
+
+#if defined(__arm__) || defined(__aarch64__)
+ int has_arm = TestCpuFlag(kCpuHasARM);
+ if (has_arm) {
+ int has_neon = TestCpuFlag(kCpuHasNEON);
+ int has_neon_dotprod = TestCpuFlag(kCpuHasNeonDotProd);
+ int has_neon_i8mm = TestCpuFlag(kCpuHasNeonI8MM);
+ int has_sve = TestCpuFlag(kCpuHasSVE);
+ int has_sve2 = TestCpuFlag(kCpuHasSVE2);
+ int has_sme = TestCpuFlag(kCpuHasSME);
+ printf("Has Arm 0x%x\n", has_arm);
+ printf("Has Neon 0x%x\n", has_neon);
+ printf("Has Neon DotProd 0x%x\n", has_neon_dotprod);
+ printf("Has Neon I8MM 0x%x\n", has_neon_i8mm);
+ printf("Has SVE 0x%x\n", has_sve);
+ printf("Has SVE2 0x%x\n", has_sve2);
+ printf("Has SME 0x%x\n", has_sme);
+
+#if __aarch64__
+ // Read and print the SVE and SME vector lengths.
+ if (has_sve) {
+ int sve_vl;
+ __asm__(".inst 0x04bf5020 \n" // rdvl x0, #1
+ "mov %w[sve_vl], w0 \n"
+ : [sve_vl] "=r"(sve_vl) // %[sve_vl]
+ :
+ : "x0");
+ printf("SVE vector length: %d bytes\n", sve_vl);
+ }
+ if (has_sme) {
+ int sme_vl;
+ __asm__(".inst 0x04bf5820 \n" // rdsvl x0, #1
+ "mov %w[sme_vl], w0 \n"
+ : [sme_vl] "=r"(sme_vl) // %[sme_vl]
+ :
+ : "x0");
+ printf("SME vector length: %d bytes\n", sme_vl);
+ }
+#endif // defined(__aarch64__)
+ }
+#endif // if defined(__arm__) || defined(__aarch64__)
+
+#if defined(__riscv)
+ int has_riscv = TestCpuFlag(kCpuHasRISCV);
+ if (has_riscv) {
+ int has_rvv = TestCpuFlag(kCpuHasRVV);
+ printf("Has RISCV 0x%x\n", has_riscv);
+ printf("Has RVV 0x%x\n", has_rvv);
+
+ // Read and print the RVV vector length.
+ if (has_rvv) {
+ register uint32_t vlenb __asm__ ("t0");
+ __asm__(".word 0xC22022F3" /* CSRR t0, vlenb */ : "=r" (vlenb));
+ printf("RVV vector length: %d bytes\n", vlenb);
+ }
+ }
+#endif // defined(__riscv)
+
+#if defined(__mips__)
+ int has_mips = TestCpuFlag(kCpuHasMIPS);
+ if (has_mips) {
+ int has_msa = TestCpuFlag(kCpuHasMSA);
+ printf("Has MIPS 0x%x\n", has_mips);
+ printf("Has MSA 0x%x\n", has_msa);
+ }
+#endif // defined(__mips__)
+
+#if defined(__loongarch__)
+ int has_loongarch = TestCpuFlag(kCpuHasLOONGARCH);
+ if (has_loongarch) {
+ int has_lsx = TestCpuFlag(kCpuHasLSX);
+ int has_lasx = TestCpuFlag(kCpuHasLASX);
+ printf("Has LOONGARCH 0x%x\n", has_loongarch);
+ printf("Has LSX 0x%x\n", has_lsx);
+ printf("Has LASX 0x%x\n", has_lasx);
+ }
+#endif // defined(__loongarch__)
+
#if defined(__i386__) || defined(__x86_64__) || \
defined(_M_IX86) || defined(_M_X64)
+ int has_x86 = TestCpuFlag(kCpuHasX86);
if (has_x86) {
int family, model, cpu_info[4];
// Vendor ID:
@@ -61,32 +163,7 @@
model = ((cpu_info[0] >> 4) & 0x0f) | ((cpu_info[0] >> 12) & 0xf0);
printf("Cpu Family %d (0x%x), Model %d (0x%x)\n", family, family,
model, model);
- }
-#endif
- printf("Cpu Flags 0x%x\n", cpu_flags);
- if (has_arm) {
- int has_neon = TestCpuFlag(kCpuHasNEON);
- printf("Has ARM 0x%x\n", has_arm);
- printf("Has NEON 0x%x\n", has_neon);
- }
- if (has_riscv) {
- int has_rvv = TestCpuFlag(kCpuHasRVV);
- printf("Has RISCV 0x%x\n", has_riscv);
- printf("Has RVV 0x%x\n", has_rvv);
- }
- if (has_mips) {
- int has_msa = TestCpuFlag(kCpuHasMSA);
- printf("Has MIPS 0x%x\n", has_mips);
- printf("Has MSA 0x%x\n", has_msa);
- }
- if (has_loongarch) {
- int has_lsx = TestCpuFlag(kCpuHasLSX);
- int has_lasx = TestCpuFlag(kCpuHasLASX);
- printf("Has LOONGARCH 0x%x\n", has_loongarch);
- printf("Has LSX 0x%x\n", has_lsx);
- printf("Has LASX 0x%x\n", has_lasx);
- }
- if (has_x86) {
+
int has_sse2 = TestCpuFlag(kCpuHasSSE2);
int has_ssse3 = TestCpuFlag(kCpuHasSSSE3);
int has_sse41 = TestCpuFlag(kCpuHasSSE41);
@@ -94,6 +171,7 @@
int has_avx = TestCpuFlag(kCpuHasAVX);
int has_avx2 = TestCpuFlag(kCpuHasAVX2);
int has_erms = TestCpuFlag(kCpuHasERMS);
+ int has_fsmr = TestCpuFlag(kCpuHasFSMR);
int has_fma3 = TestCpuFlag(kCpuHasFMA3);
int has_f16c = TestCpuFlag(kCpuHasF16C);
int has_avx512bw = TestCpuFlag(kCpuHasAVX512BW);
@@ -103,8 +181,10 @@
int has_avx512vbmi2 = TestCpuFlag(kCpuHasAVX512VBMI2);
int has_avx512vbitalg = TestCpuFlag(kCpuHasAVX512VBITALG);
int has_avx10 = TestCpuFlag(kCpuHasAVX10);
+ int has_avx10_2 = TestCpuFlag(kCpuHasAVX10_2);
int has_avxvnni = TestCpuFlag(kCpuHasAVXVNNI);
int has_avxvnniint8 = TestCpuFlag(kCpuHasAVXVNNIINT8);
+ int has_amxint8 = TestCpuFlag(kCpuHasAMXINT8);
printf("Has X86 0x%x\n", has_x86);
printf("Has SSE2 0x%x\n", has_sse2);
printf("Has SSSE3 0x%x\n", has_ssse3);
@@ -113,6 +193,7 @@
printf("Has AVX 0x%x\n", has_avx);
printf("Has AVX2 0x%x\n", has_avx2);
printf("Has ERMS 0x%x\n", has_erms);
+ printf("Has FSMR 0x%x\n", has_fsmr);
printf("Has FMA3 0x%x\n", has_fma3);
printf("Has F16C 0x%x\n", has_f16c);
printf("Has AVX512BW 0x%x\n", has_avx512bw);
@@ -122,9 +203,12 @@
printf("Has AVX512VBMI2 0x%x\n", has_avx512vbmi2);
printf("Has AVX512VBITALG 0x%x\n", has_avx512vbitalg);
printf("Has AVX10 0x%x\n", has_avx10);
+ printf("Has AVX10_2 0x%x\n", has_avx10_2);
printf("HAS AVXVNNI 0x%x\n", has_avxvnni);
printf("Has AVXVNNIINT8 0x%x\n", has_avxvnniint8);
+ printf("Has AMXINT8 0x%x\n", has_amxint8);
}
+#endif // defined(__i386__) || defined(__x86_64__) || defined(_M_IX86) || defined(_M_X64)
return 0;
}
diff --git a/util/psnr_main.cc b/util/psnr_main.cc
index 8b9fd97..a11cd3f 100644
--- a/util/psnr_main.cc
+++ b/util/psnr_main.cc
@@ -65,9 +65,9 @@
#endif
// Parse PYUV format. ie name.1920x800_24Hz_P420.yuv
-bool ExtractResolutionFromFilename(const char* name,
- int* width_ptr,
- int* height_ptr) {
+static bool ExtractResolutionFromFilename(const char* name,
+ int* width_ptr,
+ int* height_ptr) {
// Isolate the .width_height. section of the filename by searching for a
// dot or underscore followed by a digit.
for (int i = 0; name[i]; ++i) {
@@ -105,25 +105,12 @@
return false;
}
-// Scale Y channel from 16..240 to 0..255.
-// This can be useful when comparing codecs that are inconsistant about Y
-uint8_t ScaleY(uint8_t y) {
- int ny = (y - 16) * 256 / 224;
- if (ny < 0) {
- ny = 0;
- }
- if (ny > 255) {
- ny = 255;
- }
- return static_cast<uint8_t>(ny);
-}
-
// MSE = Mean Square Error
-double GetMSE(double sse, double size) {
+static double GetMSE(double sse, double size) {
return sse / size;
}
-void PrintHelp(const char* program) {
+static void PrintHelp(const char* program) {
printf("%s [-options] org_seq rec_seq [rec_seq2.. etc]\n", program);
#ifdef HAVE_JPEG
printf("jpeg or raw YUV 420 supported.\n");
@@ -151,7 +138,7 @@
exit(0);
}
-void ParseOptions(int argc, const char* argv[]) {
+static void ParseOptions(int argc, const char* argv[]) {
if (argc <= 1) {
PrintHelp(argv[0]);
}
@@ -240,15 +227,15 @@
}
}
-bool UpdateMetrics(uint8_t* ch_org,
- uint8_t* ch_rec,
- const int y_size,
- const int uv_size,
- const size_t total_size,
- int number_of_frames,
- metric* cur_distortion_psnr,
- metric* distorted_frame,
- bool compute_psnr) {
+static bool UpdateMetrics(uint8_t* ch_org,
+ uint8_t* ch_rec,
+ const int y_size,
+ const int uv_size,
+ const size_t total_size,
+ int number_of_frames,
+ metric* cur_distortion_psnr,
+ metric* distorted_frame,
+ bool compute_psnr) {
const int uv_offset = (do_swap_uv ? uv_size : 0);
const uint8_t* const u_org = ch_org + y_size + uv_offset;
const uint8_t* const u_rec = ch_rec + y_size;
