build/android-arm64v8a/src/core/CL/cl_kernels/depthwise_convolution.clembed - platform/external/ComputeLibrary - Git at Google

 R"(

 /*
  * Copyright (c) 2017-2020 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to
  * deal in the Software without restriction, including without limitation the
  * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
  * The above copyright notice and this permission notice shall be included in all
  * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  * SOFTWARE.
  */
 /*
  * Copyright (c) 2016-2020 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to
  * deal in the Software without restriction, including without limitation the
  * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
  * The above copyright notice and this permission notice shall be included in all
  * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  * SOFTWARE.
  */
 #ifndef ARM_COMPUTE_HELPER_H
 #define ARM_COMPUTE_HELPER_H

 /*
  * Copyright (c) 2020 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to
  * deal in the Software without restriction, including without limitation the
  * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
  * The above copyright notice and this permission notice shall be included in all
  * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  * SOFTWARE.
  */

 /** Store the 0 to (n-1)th rows of the given variables
  * @name STORE_ROW_n
  *
  * @param[in] N0        The width of the passed in vector. Supported: 1, 2, 3, 4, 8, 16
  * @param[in] DATA_TYPE The data type of the vectors
  * @param[in] BASENAME  The basename of the variables
  * @param[in] PTR       The base pointer
  * @param[in] STRIDE_Y  The stride value in y-axis direction
  * @param[in] Z         The offset in z-axis direction
  * @{
  */
 #define STORE_ROW_1(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     VSTORE(N0)                                                 \
     (BASENAME##0, 0, (__global DATA_TYPE *)(PTR + 0 * STRIDE_Y + Z##0));

 #define STORE_ROW_2(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_1(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE(N0)                                                 \
     (BASENAME##1, 0, (__global DATA_TYPE *)(PTR + 1 * STRIDE_Y + Z##1));

 #define STORE_ROW_3(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_2(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE(N0)                                                 \
     (BASENAME##2, 0, (__global DATA_TYPE *)(PTR + 2 * STRIDE_Y + Z##2));

 #define STORE_ROW_4(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_3(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE(N0)                                                 \
     (BASENAME##3, 0, (__global DATA_TYPE *)(PTR + 3 * STRIDE_Y + Z##3));

 #define STORE_ROW_5(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_4(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE(N0)                                                 \
     (BASENAME##4, 0, (__global DATA_TYPE *)(PTR + 4 * STRIDE_Y + Z##4));

 #define STORE_ROW_6(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_5(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE(N0)                                                 \
     (BASENAME##5, 0, (__global DATA_TYPE *)(PTR + 5 * STRIDE_Y + Z##5));

 #define STORE_ROW_7(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_6(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE(N0)                                                 \
     (BASENAME##6, 0, (__global DATA_TYPE *)(PTR + 6 * STRIDE_Y + Z##6));

 #define STORE_ROW_8(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_7(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE(N0)                                                 \
     (BASENAME##7, 0, (__global DATA_TYPE *)(PTR + 7 * STRIDE_Y + Z##7));

 #define STORE_ROW_9(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_8(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE(N0)                                                 \
     (BASENAME##8, 0, (__global DATA_TYPE *)(PTR + 8 * STRIDE_Y + Z##8));

 #define STORE_ROW_10(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_9(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)      \
     VSTORE(N0)                                                  \
     (BASENAME##9, 0, (__global DATA_TYPE *)(PTR + 9 * STRIDE_Y + Z##9));

 #define STORE_ROW_11(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_10(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE(N0)                                                  \
     (BASENAME##A, 0, (__global DATA_TYPE *)(PTR + 10 * STRIDE_Y + Z##A));

 #define STORE_ROW_12(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_11(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE(N0)                                                  \
     (BASENAME##B, 0, (__global DATA_TYPE *)(PTR + 11 * STRIDE_Y + Z##B));

 #define STORE_ROW_13(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_12(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE(N0)                                                  \
     (BASENAME##C, 0, (__global DATA_TYPE *)(PTR + 12 * STRIDE_Y + Z##C));

 #define STORE_ROW_14(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_13(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE(N0)                                                  \
     (BASENAME##D, 0, (__global DATA_TYPE *)(PTR + 13 * STRIDE_Y + Z##D));

 #define STORE_ROW_15(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_14(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE(N0)                                                  \
     (BASENAME##E, 0, (__global DATA_TYPE *)(PTR + 14 * STRIDE_Y + Z##E));

 #define STORE_ROW_16(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_15(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE(N0)                                                  \
     (BASENAME##F, 0, (__global DATA_TYPE *)(PTR + 15 * STRIDE_Y + Z##F));
 /** @} */ // end of groupd STORE_ROW_n

 /** Convert and store the 0th to (n-1)th rows of the given variables
  * @name CONVERT_STORE_ROW_n
  *
  * @param[in] N0        The size of the vectors
  * @param[in] DATA_TYPE The data type of the vectors
  * @param[in] BASENAME  The basename of the variables
  * @param[in] PTR       The base pointer
  * @param[in] STRIDE_Y  The stride value in y-axis direction
  * @param[in] Z         The offset in z-axis direction
  * @{
  */
 #define CONVERT_STORE_ROW_1(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     VSTORE(N0)                                                         \
     (CONVERT_SAT((BASENAME##0), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 0 * STRIDE_Y + Z##0));

 #define CONVERT_STORE_ROW_2(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     CONVERT_STORE_ROW_1(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE(N0)                                                         \
     (CONVERT_SAT((BASENAME##1), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 1 * STRIDE_Y + Z##1));

 #define CONVERT_STORE_ROW_3(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     CONVERT_STORE_ROW_2(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE(N0)                                                         \
     (CONVERT_SAT((BASENAME##2), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 2 * STRIDE_Y + Z##2));

 #define CONVERT_STORE_ROW_4(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     CONVERT_STORE_ROW_3(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE(N0)                                                         \
     (CONVERT_SAT((BASENAME##3), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 3 * STRIDE_Y + Z##3));

 #define CONVERT_STORE_ROW_5(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     CONVERT_STORE_ROW_4(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE(N0)                                                         \
     (CONVERT_SAT((BASENAME##4), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 4 * STRIDE_Y + Z##4));

 #define CONVERT_STORE_ROW_6(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     CONVERT_STORE_ROW_5(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE(N0)                                                         \
     (CONVERT_SAT((BASENAME##5), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 5 * STRIDE_Y + Z##5));

 #define CONVERT_STORE_ROW_7(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     CONVERT_STORE_ROW_6(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE(N0)                                                         \
     (CONVERT_SAT((BASENAME##6), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 6 * STRIDE_Y + Z##6));

 #define CONVERT_STORE_ROW_8(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     CONVERT_STORE_ROW_7(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE(N0)                                                         \
     (CONVERT_SAT((BASENAME##7), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 7 * STRIDE_Y + Z##7));

 #define CONVERT_STORE_ROW_9(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     CONVERT_STORE_ROW_8(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE(N0)                                                         \
     (CONVERT_SAT((BASENAME##8), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 8 * STRIDE_Y + Z##8));

 #define CONVERT_STORE_ROW_10(N0, DATA, BASENAME, PTR, STRIDE_Y, Z) \
     CONVERT_STORE_ROW_9(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     VSTORE(N0)                                                     \
     (CONVERT_SAT((BASENAME##9), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 9 * STRIDE_Y + Z##9));

 #define CONVERT_STORE_ROW_11(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     CONVERT_STORE_ROW_10(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE(N0)                                                          \
     (CONVERT_SAT((BASENAME##A), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 10 * STRIDE_Y + Z##A));

 #define CONVERT_STORE_ROW_12(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     CONVERT_STORE_ROW_11(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE(N0)                                                          \
     (CONVERT_SAT((BASENAME##B), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 11 * STRIDE_Y + Z##B));

 #define CONVERT_STORE_ROW_13(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     CONVERT_STORE_ROW_12(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE(N0)                                                          \
     (CONVERT_SAT((BASENAME##C), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 12 * STRIDE_Y + Z##C));

 #define CONVERT_STORE_ROW_14(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     CONVERT_STORE_ROW_13(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE(N0)                                                          \
     (CONVERT_SAT((BASENAME##D), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 13 * STRIDE_Y + Z##D));

 #define CONVERT_STORE_ROW_15(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     CONVERT_STORE_ROW_14(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE(N0)                                                          \
     (CONVERT_SAT((BASENAME##E), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 14 * STRIDE_Y + Z##E));

 #define CONVERT_STORE_ROW_16(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     CONVERT_STORE_ROW_15(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE(N0)                                                          \
     (CONVERT_SAT((BASENAME##F), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 15 * STRIDE_Y + Z##F));

 /** @} */ // end of groupd CONVERT_STORE_ROW_n

 /** Store a block of the given size M0xN0
  * @name STORE_BLOCK
  *
  * Supported cases are M0=1,2,3,...,16 and N0=2,3,4,8,16.
  * The data to store is expected to have consecutive names for each row.
  * E.g., for M0=3 and basename=c, the expected names are c0, c1 and c2.
  * The Z offset is expected to have consecutive names.
  * E.g., for M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2.
  *
  * @param[in] M0        The number of rows to store
  * @param[in] N0        The size of each vector
  * @param[in] DATA_TYPE The data type of the vectors
  * @param[in] BASENAME  The basename of the variables
  * @param[in] PTR       The base pointer
  * @param[in] STRIDE_Y  The stride value in y-axis direction
  * @param[in] Z         The offset in z-axis direction
  * @{
  */
 #define STORE_BLOCK_STR(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) STORE_ROW_##M0(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)
 #define STORE_BLOCK(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) STORE_BLOCK_STR(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)
 /** @} */ // end of group STORE_BLOCK

 /** Convert and store a block of the given size M0xN0
  * @name CONVERT_STORE_BLOCK
  *
  * Supported cases are M0=1,2,3,...,16 and N0=2,3,4,8,16.
  * The data to store is expected to have consecutive names for each row.
  * E.g., for M0=3 and basename=c, the expected names are c0, c1 and c2.
  * The Z offset is expected to have consecutive names.
  * E.g., for M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2.
  *
  * @param[in] M0        The number of rows to store
  * @param[in] N0        The size of each vector
  * @param[in] DATA_TYPE The data type of the vectors
  * @param[in] BASENAME  The basename of the variables
  * @param[in] PTR       The base pointer
  * @param[in] STRIDE_Y  The stride value in y-axis direction
  * @param[in] Z         The offset in z-axis direction
  * @{
  */
 #define CONVERT_STORE_BLOCK_STR(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) CONVERT_STORE_ROW_##M0(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)
 #define CONVERT_STORE_BLOCK(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) CONVERT_STORE_BLOCK_STR(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)
 /** @} */ // end of group CONVERT_STORE_BLOCK

 /** Partially store the 0 to (n-1)th rows of the given variables
  * @name STORE_ROW_PARTIAL_n
  * Within each row, store the lower @p STORE_N0 elements of vectors of width @p N0
  *
  * @note in case @p STORE_N0 != 1, 2, 3, 4, 8, 16, extra vstore(s) will be invoked, thus incurring small performance penalty.
  *
  * @param[in] N0        The width of the passed in vector. Supported: 1, 2, 3, 4, 8, 16
  * @param[in] STORE_N0  The **lower** size of the vectors to store. Supported: [1-16 and <= @p N0
  * @param[in] DATA_TYPE The data type of the vectors
  * @param[in] BASENAME  The basename of the variables
  * @param[in] PTR       The base pointer
  * @param[in] STRIDE_Y  The stride value in y-axis direction
  * @param[in] Z         The offset in z-axis direction
  * @{
  */
 #define STORE_ROW_PARTIAL_1(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     VSTORE_PARTIAL(N0, STORE_N0)                                                 \
     (BASENAME##0, 0, (__global DATA_TYPE *)(PTR + 0 * STRIDE_Y + Z##0));

 #define STORE_ROW_PARTIAL_2(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_PARTIAL_1(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE_PARTIAL(N0, STORE_N0)                                                 \
     (BASENAME##1, 0, (__global DATA_TYPE *)(PTR + 1 * STRIDE_Y + Z##1));

 #define STORE_ROW_PARTIAL_3(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_PARTIAL_2(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE_PARTIAL(N0, STORE_N0)                                                 \
     (BASENAME##2, 0, (__global DATA_TYPE *)(PTR + 2 * STRIDE_Y + Z##2));

 #define STORE_ROW_PARTIAL_4(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_PARTIAL_3(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE_PARTIAL(N0, STORE_N0)                                                 \
     (BASENAME##3, 0, (__global DATA_TYPE *)(PTR + 3 * STRIDE_Y + Z##3));

 #define STORE_ROW_PARTIAL_5(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_PARTIAL_4(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE_PARTIAL(N0, STORE_N0)                                                 \
     (BASENAME##4, 0, (__global DATA_TYPE *)(PTR + 4 * STRIDE_Y + Z##4));

 #define STORE_ROW_PARTIAL_6(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_PARTIAL_5(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE_PARTIAL(N0, STORE_N0)                                                 \
     (BASENAME##5, 0, (__global DATA_TYPE *)(PTR + 5 * STRIDE_Y + Z##5));

 #define STORE_ROW_PARTIAL_7(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_PARTIAL_6(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE_PARTIAL(N0, STORE_N0)                                                 \
     (BASENAME##6, 0, (__global DATA_TYPE *)(PTR + 6 * STRIDE_Y + Z##6));

 #define STORE_ROW_PARTIAL_8(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_PARTIAL_7(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE_PARTIAL(N0, STORE_N0)                                                 \
     (BASENAME##7, 0, (__global DATA_TYPE *)(PTR + 7 * STRIDE_Y + Z##7));

 #define STORE_ROW_PARTIAL_9(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_PARTIAL_8(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE_PARTIAL(N0, STORE_N0)                                                 \
     (BASENAME##8, 0, (__global DATA_TYPE *)(PTR + 8 * STRIDE_Y + Z##8));

 #define STORE_ROW_PARTIAL_10(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_PARTIAL_9(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)      \
     VSTORE_PARTIAL(N0, STORE_N0)                                                  \
     (BASENAME##9, 0, (__global DATA_TYPE *)(PTR + 9 * STRIDE_Y + Z##9));

 #define STORE_ROW_PARTIAL_11(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_PARTIAL_10(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE_PARTIAL(N0, STORE_N0)                                                  \
     (BASENAME##A, 0, (__global DATA_TYPE *)(PTR + 10 * STRIDE_Y + Z##A));

 #define STORE_ROW_PARTIAL_12(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_PARTIAL_11(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE_PARTIAL(N0, STORE_N0)                                                  \
     (BASENAME##B, 0, (__global DATA_TYPE *)(PTR + 11 * STRIDE_Y + Z##B));

 #define STORE_ROW_PARTIAL_13(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_PARTIAL_12(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE_PARTIAL(N0, STORE_N0)                                                  \
     (BASENAME##C, 0, (__global DATA_TYPE *)(PTR + 12 * STRIDE_Y + Z##C));

 #define STORE_ROW_PARTIAL_14(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_PARTIAL_13(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE_PARTIAL(N0, STORE_N0)                                                  \
     (BASENAME##D, 0, (__global DATA_TYPE *)(PTR + 13 * STRIDE_Y + Z##D));

 #define STORE_ROW_PARTIAL_15(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_PARTIAL_14(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE_PARTIAL(N0, STORE_N0)                                                  \
     (BASENAME##E, 0, (__global DATA_TYPE *)(PTR + 14 * STRIDE_Y + Z##E));

 #define STORE_ROW_PARTIAL_16(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_PARTIAL_15(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE_PARTIAL(N0, STORE_N0)                                                  \
     (BASENAME##F, 0, (__global DATA_TYPE *)(PTR + 15 * STRIDE_Y + Z##F));
 /** @} */ // end of groupd STORE_ROW_PARTIAL_n

 /** Partially store a block of the given size STORE_M0xSTORE_N0
  * @name STORE_BLOCK_PARTIAL
  *
  * @note The vector width @p N0 is also required for correct partial storing behaviour.
  * @note in case @p STORE_N0 != 1, 2, 3, 4, 8, 16, extra vstore(s) will be invoked, thus incurring small performance penalty.
  *
  * The data to store is expected to have consecutive names for each row.
  * E.g., for STORE_M0=3 and basename=c, the expected names are c0, c1 and c2.
  * The Z offset is expected to have consecutive names.
  * E.g., for STORE_M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2.
  *
  * @param[in] STORE_M0  The number of rows to store. Supported: 1-16
  * @param[in] STORE_N0  The lower number of elements of vectors to store. Supported: 1-16 and <= @p N0
  * @param[in] N0        The size of each vector. Supported: 1, 2, 3, 4, 8, 16
  * @param[in] DATA_TYPE The data type of the vectors
  * @param[in] BASENAME  The basename of the variables
  * @param[in] PTR       The base pointer
  * @param[in] STRIDE_Y  The stride value in y-axis direction
  * @param[in] Z         The offset in z-axis direction
  * @{
  */
 #define STORE_BLOCK_PARTIAL_STR(STORE_M0, STORE_N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) STORE_ROW_PARTIAL_##STORE_M0(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)
 #define STORE_BLOCK_PARTIAL(STORE_M0, STORE_N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) STORE_BLOCK_PARTIAL_STR(STORE_M0, STORE_N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)
 /** Store a block that can be partial in both x and y dimensions
  *
  * @note in cases @p PARTIAL_STORE_N0 != 1, 2, 3, 4, 8, 16, extra vstore(s) will be invoked, thus incurring small performance penalty.
  *
  * The data to store is expected to have consecutive names for each row.
  * E.g., for M0=3 and basename=c, the expected names are c0, c1 and c2.
  * The Z offset is expected to have consecutive names.
  * E.g., for M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2.
  *
  * @param[in] M0               The number of rows to store, for non-partial blocks. Supported: 1-16
  * @param[in] N0               The size of each vector, for non-partial blocks. Supported: 1, 2, 3, 4, 8, 16
  * @param[in] DATA_TYPE        The data type of the vectors
  * @param[in] BASENAME         The basename of the variables
  * @param[in] PTR              The base pointer
  * @param[in] STRIDE_Y         The stride value in y-axis direction
  * @param[in] Z                The offset in z-axis direction
  * @param[in] PARTIAL_STORE_M0 The partial size in y, for partial blocks. Supported range: [1, @p M0)
  * @param[in] PARTIAL_STORE_N0 The partial size in x, for partial blocks. Supported range: [1, @p N0)
  * @param[in] PARTIAL_COND_Y   Condition on the y axis to perform the partial store Y. True to use PARTIAL_STORE_M0 rather than M0.
  * @param[in] PARTIAL_COND_X   Condition on the x axis to perform the partial store X. True to use PARTIAL_STORE_N0 rather than N0.
  */
 #define STORE_BLOCK_PARTIAL_IN_X_AND_Y(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_STORE_N0, PARTIAL_COND_Y, PARTIAL_COND_X) \
     if(!(PARTIAL_COND_X) && !(PARTIAL_COND_Y))                                                                                                            \
     {                                                                                                                                                     \
         STORE_BLOCK_PARTIAL(M0, N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z);                                                                           \
     }                                                                                                                                                     \
     else if((PARTIAL_COND_Y) && !(PARTIAL_COND_X))                                                                                                        \
     {                                                                                                                                                     \
         STORE_BLOCK_PARTIAL(PARTIAL_STORE_M0, N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z);                                                             \
     }                                                                                                                                                     \
     else if(!(PARTIAL_COND_Y) && (PARTIAL_COND_X))                                                                                                        \
     {                                                                                                                                                     \
         STORE_BLOCK_PARTIAL(M0, PARTIAL_STORE_N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z);                                                             \
     }                                                                                                                                                     \
     else                                                                                                                                                  \
     {                                                                                                                                                     \
         STORE_BLOCK_PARTIAL(PARTIAL_STORE_M0, PARTIAL_STORE_N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z);                                               \
     }
 /** Store a block that can only be partial in x but not y.
  *
  * @note in case @p N0 or @p PARTIAL_STORE_N0 != 1, 2, 3, 4, 8, 16, extra vstore(s) will be invoked, thus incurring small performance penalty.
  *
  * The data to store is expected to have consecutive names for each row.
  * E.g., for M0=3 and basename=c, the expected names are c0, c1 and c2.
  * The Z offset is expected to have consecutive names.
  * E.g., for M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2.
  *
  * @param[in] M0               The number of rows to store, for non-partial blocks. Supported: 1-16
  * @param[in] N0               The size of each vector, for non-partial blocks. Supported: 1, 2, 3, 4, 8, 16
  * @param[in] DATA_TYPE        The data type of the vectors
  * @param[in] BASENAME         The basename of the variables
  * @param[in] PTR              The base pointer
  * @param[in] STRIDE_Y         The stride value in y-axis direction
  * @param[in] Z                The offset in z-axis direction
  * @param[in] PARTIAL_STORE_N0 The partial size in x, for partial blocks. Supported range: [1, @p N0)
  * @param[in] PARTIAL_COND_X   Condition on the x axis to perform the partial store X. True to use PARTIAL_STORE_N0 rather than N0.
  */
 #define STORE_BLOCK_PARTIAL_IN_X(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_N0, PARTIAL_COND_X) \
     if(!(PARTIAL_COND_X))                                                                                         \
     {                                                                                                             \
         STORE_BLOCK_PARTIAL(M0, N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z);                                   \
     }                                                                                                             \
     else                                                                                                          \
     {                                                                                                             \
         STORE_BLOCK_PARTIAL(M0, PARTIAL_STORE_N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z);                     \
     }
 /** Store a block that can only be partial in y but not x.
  *
  * @note in case @p N0 or @p PARTIAL_STORE_N0 != 1, 2, 3, 4, 8, 16, extra vstore(s) will be invoked, thus incurring small performance penalty.
  *
  * The data to store is expected to have consecutive names for each row.
  * E.g., for M0=3 and basename=c, the expected names are c0, c1 and c2.
  * The Z offset is expected to have consecutive names.
  * E.g., for M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2.
  *
  * @param[in] M0               The number of rows to store, for non-partial blocks. Supported: 1-16
  * @param[in] N0               The size of each vector, for non-partial blocks. Supported: 1, 2, 3, 4, 8, 16
  * @param[in] DATA_TYPE        The data type of the vectors
  * @param[in] BASENAME         The basename of the variables
  * @param[in] PTR              The base pointer
  * @param[in] STRIDE_Y         The stride value in y-axis direction
  * @param[in] Z                The offset in z-axis direction
  * @param[in] PARTIAL_STORE_M0 The partial size in y, for partial blocks. Supported range: [1, @p M0)
  * @param[in] PARTIAL_COND_Y   Condition on the y axis to perform the partial store Y. True to use PARTIAL_STORE_M0 rather than M0.
  */
 #define STORE_BLOCK_PARTIAL_IN_Y(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_COND_Y) \
     if(!(PARTIAL_COND_Y))                                                                                         \
     {                                                                                                             \
         STORE_BLOCK_PARTIAL(M0, N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z);                                   \
     }                                                                                                             \
     else                                                                                                          \
     {                                                                                                             \
         STORE_BLOCK_PARTIAL(PARTIAL_STORE_M0, N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z);                     \
     }
 /** @} */ // end of group STORE_BLOCK_PARTIAL

 #if defined(PARTIAL_STORE_M0) && defined(PARTIAL_STORE_N0)

 /** Boundary-aware GEMM block store
  * @name STORE_BLOCK_BOUNDARY_AWARE
  * This macro assumes the following schemes to achieve boundary-awareness:
  *  - Overlapping load in Y axis from lhs tensor. This implies lhs has no padding along y dim.
  *  - Non-Overlapping(normal) load from rhs tensor. This imples rhs can have paddings.
  *  - Overlapping load in Y axis from bias tensor. This implies rhs has no padding along y dim.
  * The macro then ensures that the dst tensor can be stored without any paddings in both x and y dim.
  *
  * In the y dimension, we place the partial blocks **at the beginning** while in the x dimension, we place the partial
  * blocks **at the end**.
  * Say, the dst tensor is of shape MxN and we have M0 and N0 as the block size, this is how we define "partial blocks"/
  * "boundary block" (we use the 2 terms "partial blocks" and "boundary blocks" interchangeably) and its various parameters:
  *
  *  *--x-->                         x == 0                        x == 1
  *  |                  |<------------------------------N-------------------------->|
  *  y                  |<--------------N0------------->|<----PARTIAL_STORE_N0----->|
  *  |     -------------#############################################################
  *  *     |          | |...............................|...........................|
  * y == 0 | PAR_..._M0 |......Boundary block in y......|.Boundary block in x and y.|
  *        |          | |...............................|...........................|
  *        M          --#############################################################
  *        |          | |                               |...........................|
  * y == 1 |         M0 |      Non-boundary block       |....Boundary block in x....|
  *        |          | |                               |...........................|
  *        |------------#############################################################
  *
  * Then @p PARTIAL_STORE_M0 = M % M0      and @p PARTIAL_STORE_N0 = N % N0
  *
  * @note in cases @p PARTIAL_STORE_N0 != 1, 2, 3, 4, 8, 16, extra vstore(s) will be invoked, thus incurring small performance penalty.
  *
  * It automatically detects if a giving M,N,M0,N0 combination can yield partial blocks in either X and Y dimension,
  * and select corresponding store methods such that the boundary detection logic is only added when needed.
  *
  * The data to store is expected to have consecutive names for each row.
  * E.g., for M0=3 and basename=c, the expected names are c0, c1 and c2.
  * The Z offset is expected to have consecutive names.
  * E.g., for M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2.
  *
  * @param[in] M0               The number of rows to store, for non-partial blocks. Supported: 1-16
  * @param[in] N0               The size of each vector, for non-partial blocks. Supported: 1, 2, 3, 4, 8, 16
  * @param[in] DATA_TYPE        The data type of the vectors
  * @param[in] BASENAME         The basename of the variables
  * @param[in] PTR              The base pointer
  * @param[in] STRIDE_Y         The stride value in y-axis direction
  * @param[in] Z                The offset in z-axis direction
  * @param[in] PARTIAL_STORE_M0 The partial size in y, for partial blocks. Supported: [0, @p M0)
  * @param[in] PARTIAL_STORE_N0 The partial size in x, for partial blocks. Supported: [0, @p N0)
  * @param[in] PARTIAL_COND_Y   Condition on the y axis to perform the partial store Y. True to use PARTIAL_STORE_M0 rather than M0.
  * @param[in] PARTIAL_COND_X   Condition on the x axis to perform the partial store X. True to use PARTIAL_STORE_N0 rather than N0.
  * @{
  */
 #if PARTIAL_STORE_M0 == 0 && PARTIAL_STORE_N0 == 0
 // Case1: No partial blocks in either x or y
 #define STORE_BLOCK_BOUNDARY_AWARE(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_STORE_N0, PARTIAL_COND_Y, PARTIAL_COND_X) \
     STORE_BLOCK(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)

 #elif PARTIAL_STORE_M0 > 0 && PARTIAL_STORE_N0 == 0
 // Case2: Partial blocks in y
 #define STORE_BLOCK_BOUNDARY_AWARE(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_STORE_N0, PARTIAL_COND_Y, PARTIAL_COND_X) \
     STORE_BLOCK_PARTIAL_IN_Y(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_COND_Y)

 #elif PARTIAL_STORE_M0 == 0 && PARTIAL_STORE_N0 > 0
 // Case3: Partial blocks in x
 #define STORE_BLOCK_BOUNDARY_AWARE(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_STORE_N0, PARTIAL_COND_Y, PARTIAL_COND_X) \
     STORE_BLOCK_PARTIAL_IN_X(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_N0, PARTIAL_COND_X)

 #else // PARTIAL_STORE_M0 == 0 && PARTIAL_STORE_N0 == 0
 // Case4: Partial blocks in both x and y
 #define STORE_BLOCK_BOUNDARY_AWARE(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_STORE_N0, PARTIAL_COND_Y, PARTIAL_COND_X) \
     STORE_BLOCK_PARTIAL_IN_X_AND_Y(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_STORE_N0, PARTIAL_COND_Y, PARTIAL_COND_X)

 #endif // PARTIAL_STORE_M0 == 0 && PARTIAL_STORE_N0 == 0

 #endif    // defined(PARTIAL_STORE_M0) && defined(PARTIAL_STORE_N0)
 /** @} */ // end of group STORE_BLOCK_BOUNDARY_AWARE

 #if defined(PARTIAL_STORE_M0)
 /** Compute the start m0 row (LHS, BIAS and DST) in a boundary-aware way so as to avoid padding
  * @name COMPUTE_M0_START_ROW
  * If there're any partial blocks in y dimension, they are placed at the beginning of the rows.
  * This shift amount is added to all rows such that the partial block (at the beginning) overlaps with the subsequent
  * blocks in the y dimension to avoid any padding.
  * EG: M0=4, PARTIAL_STORE_M0=1:
  *                  | Non-overlapping | +M0_ROW_SHIFT (Overlapping)
  * block 0 (partial)| start row = 0   | start row = 0
  * block 1 (full)   | start row = 4   | start row = 1
  * block 2 (full)   | start row = 8   | start row = 5
  *
  * @param[in] y                Global id of current block in y.
  * @param[in] M0               The number of rows to store, for non-partial blocks. Supported: 1-16
  * @param[in] PARTIAL_STORE_M0 The partial size in y, for partial blocks. Supported: [0, @p M0)
  * @{
  */
 #define COMPUTE_M0_START_ROW(y, M0, PARTIAL_STORE_M0) \
     ((uint)(max(0, (int)(y * M0) - (int)((M0 - PARTIAL_STORE_M0) % M0))))
 #else // defined(PARTIAL_STORE_M0)
 #define COMPUTE_M0_START_ROW(y, M0, PARTIAL_STORE_M0) \
     ((uint)(y * M0))
 #endif    // defined(PARTIAL_STORE_M0)
 /** @} */ // end of group COMPUTE_M0_START_ROW

 /** Store a vector that can only be partial in x.
  *
  * @note in case @p vec_size or @p leftover != 1, 2, 3, 4, 8, 16, extra vstore(s) will be invoked, thus incurring small performance penalty.
  *
  * The data to store is expected to end in a 0.
  * E.g., for basename=c, the expected name is c0.
  *
  * @param[in] basename  The name of the variable without trailing 0
  * @param[in] data_type The data type of the vector
  * @param[in] ptr       The base pointer
  * @param[in] vec_size  The vector size if cond = false. Supported: 1, 2, 3, 4, 8, 16
  * @param[in] leftover  The vector size if cond = true. Supported range: [1, @p vec_size0)
  * @param[in] cond      Condition to select either vec_size0 or vec_size1
  * @{
  */
 #define STORE_VECTOR_SELECT(basename, data_type, ptr, vec_size, leftover, cond) \
     STORE_BLOCK_PARTIAL_IN_X(1, vec_size, data_type, basename, ptr, 0, 0, leftover, cond)
 /** @} */ // end of group STORE_VECTOR_SELECT

 #if defined(ARM_COMPUTE_OPENCL_FP16_ENABLED) && defined(cl_khr_fp16)
 #pragma OPENCL EXTENSION cl_khr_fp16 : enable
 #endif // defined(ARM_COMPUTE_OPENCL_FP16_ENABLED) && defined(cl_khr_fp16)

 #if defined(ARM_COMPUTE_OPENCL_DOT8_ENABLED) && defined(cl_arm_integer_dot_product_int8)
 #pragma OPENCL EXTENSION cl_arm_integer_dot_product_int8 : enable
 #endif // defined(ARM_COMPUTE_OPENCL_DOT8_ENABLED) && defined(cl_arm_integer_dot_product_int8)

 #if defined(ARM_COMPUTE_OPENCL_DOT8_ACC_ENABLED) && defined(cl_arm_integer_dot_product_accumulate_int8)
 #pragma OPENCL EXTENSION cl_arm_integer_dot_product_accumulate_int8 : enable
 #endif // defined(ARM_COMPUTE_OPENCL_DOT8_ACC_ENABLED) && defined(cl_arm_integer_dot_product_accumulate_int8)

 #if defined(ARM_COMPUTE_DEBUG_ENABLED) && defined(cl_arm_printf)
 #pragma OPENCL EXTENSION cl_arm_printf : enable
 #endif // defined(ARM_COMPUTE_DEBUG_ENABLED) && defined(cl_arm_printf)

 #define GPU_ARCH_MIDGARD 0x100
 #define GPU_ARCH_BIFROST 0x200

 /** Concatenate two inputs.
  *
  * @param[in] a The first input to be concatenated
  * @param[in] b The second input to be concatenated
  *
  * @return The concatenated output
  */
 #define CONCAT(a, b) a##b

 /** Expand the given vector
  *
  * @param[in] x The vector to be expanded
  *
  * @return The expanded output
  */
 #define EXPAND(x) x

 /** Clamp the given value between an upper and lower bound.
  *
  * @param[in] x       The value to be clamped
  * @param[in] min_val The lower bound
  * @param[in] max_val The upper bound
  *
  * @return The clamped value.
  */
 #define CLAMP(x, min_val, max_val) min(max(x, min_val), max_val)

 /** REVn reverses the given vector whose size is n.
  * @name REVn
  *
  * @param[in] x The vector to be reversed
  *
  * @return The reversed vector
  * @{
  */
 #define REV1(x) ((x))
 #define REV2(x) ((x).s10)
 #define REV3(x) ((x).s210)
 #define REV4(x) ((x).s3210)
 #define REV8(x) ((x).s76543210)
 #define REV16(x) ((x).sFEDCBA9876543210)
 /** @} */ // end of group REVn

 /** Reverse the given vector.
  * @name REVERSE
  *
  * @param[in] x The vector to be reversed
  * @param[in] s The size of the vector
  *
  * @return The reversed vector
  * @{
  */
 #define REVERSE_STR(x, s) REV##s((x))
 #define REVERSE(x, s) REVERSE_STR(x, s)
 /** @} */ // end of group REVERSE

 /** Circular-right-shift (rotate-right) the vector of size s by the amount of n.
  * @name ROTs_n
  *
  * @param[in] x The vector to be shifted
  *
  * @return The shifted vector
  * @{
  */
 #define ROT1_0(x) ((x))

 #define ROT2_0(x) ((x))
 #define ROT2_1(x) ((x).s10)

 #define ROT3_0(x) ((x))
 #define ROT3_1(x) ((x).s201)
 #define ROT3_2(x) ((x).s120)

 #define ROT4_0(x) ((x))
 #define ROT4_1(x) ((x).s3012)
 #define ROT4_2(x) ((x).s2301)
 #define ROT4_3(x) ((x).s1230)

 #define ROT8_0(x) ((x))
 #define ROT8_1(x) ((x).s70123456)
 #define ROT8_2(x) ((x).s67012345)
 #define ROT8_3(x) ((x).s56701234)
 #define ROT8_4(x) ((x).s45670123)
 #define ROT8_5(x) ((x).s34567012)
 #define ROT8_6(x) ((x).s23456701)
 #define ROT8_7(x) ((x).s12345670)

 #define ROT16_0(x) ((x))
 #define ROT16_1(x) ((x).sF0123456789ABCDE)
 #define ROT16_2(x) ((x).sEF0123456789ABCD)
 #define ROT16_3(x) ((x).sDEF0123456789ABC)
 #define ROT16_4(x) ((x).sCDEF0123456789AB)
 #define ROT16_5(x) ((x).sBCDEF0123456789A)
 #define ROT16_6(x) ((x).sABCDEF0123456789)
 #define ROT16_7(x) ((x).s9ABCDEF012345678)
 #define ROT16_8(x) ((x).s89ABCDEF01234567)
 #define ROT16_9(x) ((x).s789ABCDEF0123456)
 #define ROT16_10(x) ((x).s6789ABCDEF012345)
 #define ROT16_11(x) ((x).s56789ABCDEF01234)
 #define ROT16_12(x) ((x).s456789ABCDEF0123)
 #define ROT16_13(x) ((x).s3456789ABCDEF012)
 #define ROT16_14(x) ((x).s23456789ABCDEF01)
 #define ROT16_15(x) ((x).s123456789ABCDEF0)
 /** @} */ // end of group ROTs_n

 /** Circular-right-shift (rotate-right) the given vector by the given amount.
  * @name ROTATE
  *
  * @param[in] x The vector to be shifted
  * @param[in] s The size of the vector
  * @param[in] n The amount to be shifted
  *
  * @return The shifted vector
  * @{
  */
 #define ROTATE_STR(x, s, n) ROT##s##_##n(x)
 #define ROTATE(x, s, n) ROTATE_STR(x, s, n)
 /** @} */ // end of group ROTATE

 /** Creates a vector of size n filled with offset values corresponding to the location of each element.
  * @name V_OFFSn
  *
  * @param[in] dt The data type of the output vector
  *
  * @return The vector filled with offset values
  * @{
  */
 #define V_OFFS1(dt) (dt##1)(0)
 #define V_OFFS2(dt) (dt##2)(0, 1)
 #define V_OFFS3(dt) (dt##3)(0, 1, 2)
 #define V_OFFS4(dt) (dt##4)(0, 1, 2, 3)
 #define V_OFFS8(dt) (dt##8)(0, 1, 2, 3, 4, 5, 6, 7)
 #define V_OFFS16(dt) (dt##16)(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)
 /** @} */ // end of group V_OFFSn

 /** Create a vector filled with offset values corresponding to the location of each element.
  * @name VEC_OFFS
  *
  * @param[in] dt The data type of the output vector
  * @param[in] s  The size of the output vector
  *
  * @return The vector filled with offset values
  * @{
  */
 #define VEC_OFFS_STR(dt, s) V_OFFS##s(dt)
 #define VEC_OFFS(dt, s) VEC_OFFS_STR(dt, s)
 /** @} */ // end of group VEC_OFFS

 #define VLOAD_STR(size) vload##size
 #define VLOAD(size) VLOAD_STR(size)

 #define PIXEL_UNIT4 1
 #define PIXEL_UNIT8 2
 #define PIXEL_UNIT16 4

 /** Utility macro to convert a vector size in pixel unit.
  *
  * @name CONVERT_VECTOR_SIZE_TO_PIXEL_UNIT
  *
  * @param[in] vec_size Vector size. Only 4,8 and 16 is supported
  *
  * @return The pixel unit (number of pixels)
  * @{
  */
 #define CONVERT_VECTOR_SIZE_TO_PIXEL_UNIT_STR(vec_size) PIXEL_UNIT##vec_size
 #define CONVERT_VECTOR_SIZE_TO_PIXEL_UNIT(vec_size) CONVERT_VECTOR_SIZE_TO_PIXEL_UNIT_STR(vec_size)
 /** @} */ // end of group CONVERT_VECTOR_SIZE_TO_PIXEL_UNIT

 #define read_image2d_floatx1(img, x_coord, y_coord) (float4)(read_imagef(img, (int2)(x_coord, y_coord)));
 #define read_image2d_floatx2(img, x_coord, y_coord) (float8)(read_imagef(img, (int2)(x_coord, y_coord)), read_imagef(img, (int2)(x_coord + 1, y_coord)));
 #define read_image2d_floatx4(img, x_coord, y_coord) (float16)(read_imagef(img, (int2)(x_coord, y_coord)), read_imagef(img, (int2)(x_coord + 1, y_coord)), read_imagef(img, (int2)(x_coord + 2, y_coord)), read_imagef(img, (int2)(x_coord + 3, y_coord)));

 #if defined(ARM_COMPUTE_OPENCL_FP16_ENABLED) && defined(cl_khr_fp16)
 #define read_image2d_halfx1(img, x_coord, y_coord) (half4)(read_imageh(img, (int2)(x_coord, y_coord)));
 #define read_image2d_halfx2(img, x_coord, y_coord) (half8)(read_imageh(img, (int2)(x_coord, y_coord)), read_imageh(img, (int2)(x_coord + 1, y_coord)));
 #define read_image2d_halfx4(img, x_coord, y_coord) (half16)(read_imageh(img, (int2)(x_coord, y_coord)), read_imageh(img, (int2)(x_coord + 1, y_coord)), read_imageh(img, (int2)(x_coord + 2, y_coord)), read_imageh(img, (int2)(x_coord + 3, y_coord)));
 #endif // defined(ARM_COMPUTE_OPENCL_FP16_ENABLED) && defined(cl_khr_fp16)

 /** Utility macro to read a 2D OpenCL image object.
  *
  * @note Coordinates are not normalized
  *
  * @param[in] data_type Data type
  * @param[in] n0        Number of pixel to read. Only 1,2 and 4 is supported
  * @param[in] img       OpenCL image object
  * @param[in] x_coord   The x coordinate for the top-left pixel
  * @param[in] y_coord   The y coordinate for the top-left pixel
  *
  * @return Pixels from the 2D OpenCL image object
  * @{
  */
 #define READ_IMAGE2D_STR(data_type, n0, img, x_coord, y_coord) read_image2d_##data_type##x##n0(img, x_coord, y_coord)
 #define READ_IMAGE2D(data_type, n0, img, x_coord, y_coord) READ_IMAGE2D_STR(data_type, n0, img, x_coord, y_coord)

 #define VSTORE_STR(size) vstore##size
 #define VSTORE(size) VSTORE_STR(size)

 #define float1 float
 #define half1 half
 #define char1 char
 #define uchar1 uchar
 #define short1 short
 #define ushort1 ushort
 #define int1 int
 #define uint1 uint
 #define long1 long
 #define ulong1 ulong
 #define double1 double

 #define vload1(OFFSET, PTR) *(OFFSET + PTR)
 #define vstore1(DATA, OFFSET, PTR) *(OFFSET + PTR) = DATA

 /** Extended partial vstore that correctly handles scalar values as well.
  * Store the **lower** 0 to (n-1)th elements of the given vector while minimising the amount of vstore ops
  * @name VSTORE_PARTIAL
  *
  * @note With this macro, the passed data can be both a vector and a scalar
  * @note @p store_size needs to be <= @p size
  * eg 1: Valid
  * VSTORE_PARTIAL(16, 15) ...;
  * eg 2: Invalid
  * VSTORE_PARTIAL(4, 7) ...;
  *
  * @param[in] size       The width of @p DATA. Supported values: 1(scalar), 2, 3, 4, 8, 16
  * @param[in] store_size The number of lower elements to store. Supported values: 1-16, but has to be <= @p size
  * @{
  */
 #define VSTORE_PARTIAL_STR(size, store_size) vstore_partial_##size##_##store_size
 #define VSTORE_PARTIAL(size, store_size) VSTORE_PARTIAL_STR(size, store_size)

 #define NO_STORE(data, offs, ptr) \
     {                             \
     }

 // Size == 1 (scalar)
 #define vstore_partial_1_0 NO_STORE
 #define vstore_partial_1_1 vstore1
 #define vstore_partial_1_2 NO_STORE
 #define vstore_partial_1_3 NO_STORE
 #define vstore_partial_1_4 NO_STORE
 #define vstore_partial_1_5 NO_STORE
 #define vstore_partial_1_6 NO_STORE
 #define vstore_partial_1_7 NO_STORE
 #define vstore_partial_1_8 NO_STORE
 #define vstore_partial_1_9 NO_STORE
 #define vstore_partial_1_10 NO_STORE
 #define vstore_partial_1_11 NO_STORE
 #define vstore_partial_1_12 NO_STORE
 #define vstore_partial_1_13 NO_STORE
 #define vstore_partial_1_14 NO_STORE
 #define vstore_partial_1_15 NO_STORE
 #define vstore_partial_1_16 NO_STORE
 // Size == 2
 #define vstore_partial_2_0 NO_STORE
 #define vstore_partial_2_1 vstore_partial_1
 #define vstore_partial_2_2 vstore_partial_2
 #define vstore_partial_2_3 NO_STORE
 #define vstore_partial_2_4 NO_STORE
 #define vstore_partial_2_5 NO_STORE
 #define vstore_partial_2_6 NO_STORE
 #define vstore_partial_2_7 NO_STORE
 #define vstore_partial_2_8 NO_STORE
 #define vstore_partial_2_9 NO_STORE
 #define vstore_partial_2_10 NO_STORE
 #define vstore_partial_2_11 NO_STORE
 #define vstore_partial_2_12 NO_STORE
 #define vstore_partial_2_13 NO_STORE
 #define vstore_partial_2_14 NO_STORE
 #define vstore_partial_2_15 NO_STORE
 #define vstore_partial_2_16 NO_STORE
 // Size == 3
 #define vstore_partial_3_0 NO_STORE
 #define vstore_partial_3_1 vstore_partial_1
 #define vstore_partial_3_2 vstore_partial_2
 #define vstore_partial_3_3 vstore_partial_3
 #define vstore_partial_3_4 NO_STORE
 #define vstore_partial_3_5 NO_STORE
 #define vstore_partial_3_6 NO_STORE
 #define vstore_partial_3_7 NO_STORE
 #define vstore_partial_3_8 NO_STORE
 #define vstore_partial_3_9 NO_STORE
 #define vstore_partial_3_10 NO_STORE
 #define vstore_partial_3_11 NO_STORE
 #define vstore_partial_3_12 NO_STORE
 #define vstore_partial_3_13 NO_STORE
 #define vstore_partial_3_14 NO_STORE
 #define vstore_partial_3_15 NO_STORE
 #define vstore_partial_3_16 NO_STORE
 // Size == 4
 #define vstore_partial_4_0 NO_STORE
 #define vstore_partial_4_1 vstore_partial_1
 #define vstore_partial_4_2 vstore_partial_2
 #define vstore_partial_4_3 vstore_partial_3
 #define vstore_partial_4_4 vstore_partial_4
 #define vstore_partial_4_5 NO_STORE
 #define vstore_partial_4_6 NO_STORE
 #define vstore_partial_4_7 NO_STORE
 #define vstore_partial_4_8 NO_STORE
 #define vstore_partial_4_9 NO_STORE
 #define vstore_partial_4_10 NO_STORE
 #define vstore_partial_4_11 NO_STORE
 #define vstore_partial_4_12 NO_STORE
 #define vstore_partial_4_13 NO_STORE
 #define vstore_partial_4_14 NO_STORE
 #define vstore_partial_4_15 NO_STORE
 #define vstore_partial_4_16 NO_STORE
 // Size == 8
 #define vstore_partial_8_0 NO_STORE
 #define vstore_partial_8_1 vstore_partial_1
 #define vstore_partial_8_2 vstore_partial_2
 #define vstore_partial_8_3 vstore_partial_3
 #define vstore_partial_8_4 vstore_partial_4
 #define vstore_partial_8_5 vstore_partial_5
 #define vstore_partial_8_6 vstore_partial_6
 #define vstore_partial_8_7 vstore_partial_7
 #define vstore_partial_8_8 vstore_partial_8
 #define vstore_partial_8_9 NO_STORE
 #define vstore_partial_8_10 NO_STORE
 #define vstore_partial_8_11 NO_STORE
 #define vstore_partial_8_12 NO_STORE
 #define vstore_partial_8_13 NO_STORE
 #define vstore_partial_8_14 NO_STORE
 #define vstore_partial_8_15 NO_STORE
 #define vstore_partial_8_16 NO_STORE
 // Size == 16
 #define vstore_partial_16_0 NO_STORE
 #define vstore_partial_16_1 vstore_partial_1
 #define vstore_partial_16_2 vstore_partial_2
 #define vstore_partial_16_3 vstore_partial_3
 #define vstore_partial_16_4 vstore_partial_4
 #define vstore_partial_16_5 vstore_partial_5
 #define vstore_partial_16_6 vstore_partial_6
 #define vstore_partial_16_7 vstore_partial_7
 #define vstore_partial_16_8 vstore_partial_8
 #define vstore_partial_16_9 vstore_partial_9
 #define vstore_partial_16_10 vstore_partial_10
 #define vstore_partial_16_11 vstore_partial_11
 #define vstore_partial_16_12 vstore_partial_12
 #define vstore_partial_16_13 vstore_partial_13
 #define vstore_partial_16_14 vstore_partial_14
 #define vstore_partial_16_15 vstore_partial_15
 #define vstore_partial_16_16 vstore_partial_16

 /** Partial vstore. Store the **lower** 0 to (n-1)th elements of the given vector while minimising the amount of vstore ops
  * @name vstore_partial_n
  *
  * @note @p DATA needs to be a vector not a scalar
  * @note n needs to be <= the vector width of the input variable @p DATA
  * eg 1: Valid
  * vstore_partial_15(var:float16, 0, 0xabcd);
  * eg 2: Invalid
  * vstore_partial_7(var:float4, 0, 0xabcd);
  *
  * @note in cases n == 1, 2, 3, 4, 8, 16, no extra vstore is invoked, thus there's no performance penalty.
  *
  * @param[in] DATA   The name of the variable
  * @param[in] OFFSET Offset in n
  * @param[in] PTR    The base pointer
  * @{
  */
 #define vstore_partial_1(DATA, OFFSET, PTR) \
     vstore1(DATA.s0, OFFSET, PTR);

 #define vstore_partial_2(DATA, OFFSET, PTR) \
     vstore2(DATA.s01, OFFSET, PTR);

 #define vstore_partial_3(DATA, OFFSET, PTR) \
     vstore3(DATA.s012, OFFSET, PTR);

 #define vstore_partial_4(DATA, OFFSET, PTR) \
     vstore4(DATA.s0123, OFFSET, PTR);

 #define vstore_partial_5(DATA, OFFSET, PTR)    \
     vstore_partial_4(DATA.s0123, OFFSET, PTR); \
     vstore1(DATA.s4, OFFSET, PTR + 4);

 #define vstore_partial_6(DATA, OFFSET, PTR)    \
     vstore_partial_4(DATA.s0123, OFFSET, PTR); \
     vstore_partial_2(DATA.s45, OFFSET, PTR + 4);

 #define vstore_partial_7(DATA, OFFSET, PTR)    \
     vstore_partial_4(DATA.s0123, OFFSET, PTR); \
     vstore_partial_3(DATA.s456, OFFSET, PTR + 4);

 #define vstore_partial_8(DATA, OFFSET, PTR) \
     vstore8(DATA.s01234567, OFFSET, PTR);

 #define vstore_partial_9(DATA, OFFSET, PTR)        \
     vstore_partial_8(DATA.s01234567, OFFSET, PTR); \
     vstore1(DATA.s8, OFFSET, PTR + 8);

 #define vstore_partial_10(DATA, OFFSET, PTR)       \
     vstore_partial_8(DATA.s01234567, OFFSET, PTR); \
     vstore_partial_2(DATA.s89, OFFSET, PTR + 8);

 #define vstore_partial_11(DATA, OFFSET, PTR)       \
     vstore_partial_8(DATA.s01234567, OFFSET, PTR); \
     vstore_partial_3(DATA.s89a, OFFSET, PTR + 8);

 #define vstore_partial_12(DATA, OFFSET, PTR)       \
     vstore_partial_8(DATA.s01234567, OFFSET, PTR); \
     vstore_partial_4(DATA.s89ab, OFFSET, PTR + 8);

 #define vstore_partial_13(DATA, OFFSET, PTR)       \
     vstore_partial_8(DATA.s01234567, OFFSET, PTR); \
     vstore_partial_5(DATA.s89abcdef, OFFSET, PTR + 8);

 #define vstore_partial_14(DATA, OFFSET, PTR)       \
     vstore_partial_8(DATA.s01234567, OFFSET, PTR); \
     vstore_partial_6(DATA.s89abcdef, OFFSET, PTR + 8);

 #define vstore_partial_15(DATA, OFFSET, PTR)       \
     vstore_partial_8(DATA.s01234567, OFFSET, PTR); \
     vstore_partial_7(DATA.s89abcdef, OFFSET, PTR + 8);

 #define vstore_partial_16(DATA, OFFSET, PTR) \
     vstore16(DATA, OFFSET, PTR);
 /** @} */ // end of groupd vstore_partial_n
 /** @} */ // end of groupd VSTORE_PARTIAL

 // Convert built-in functions with _sat modifier are not supported in floating point so we create defines
 // without _sat to overcome this issue
 #define convert_float_sat convert_float
 #define convert_float1_sat convert_float
 #define convert_float2_sat convert_float2
 #define convert_float3_sat convert_float3
 #define convert_float4_sat convert_float4
 #define convert_float8_sat convert_float8
 #define convert_float16_sat convert_float16
 #define convert_half_sat convert_float
 #define convert_half1_sat convert_half
 #define convert_half2_sat convert_half2
 #define convert_half3_sat convert_half3
 #define convert_half4_sat convert_half4
 #define convert_half8_sat convert_half8
 #define convert_half16_sat convert_half16

 #define convert_float1 convert_float
 #define convert_half1 convert_half
 #define convert_char1 convert_char
 #define convert_uchar1 convert_uchar
 #define convert_short1 convert_short
 #define convert_ushort1 convert_ushort
 #define convert_int1 convert_int
 #define convert_uint1 convert_uint
 #define convert_long1 convert_long
 #define convert_ulong1 convert_ulong
 #define convert_double1 convert_double

 #define convert_char1_sat convert_char_sat
 #define convert_uchar1_sat convert_uchar_sat
 #define convert_short1_sat convert_short_sat
 #define convert_ushort1_sat convert_ushort_sat
 #define convert_int1_sat convert_int_sat
 #define convert_uint1_sat convert_uint_sat
 #define convert_long1_sat convert_long_sat
 #define convert_ulong1_sat convert_ulong_sat
 #define convert_double1_sat convert_double_sat

 #define VEC_DATA_TYPE_STR(type, size) type##size
 #define VEC_DATA_TYPE(type, size) VEC_DATA_TYPE_STR(type, size)

 #define CONVERT_STR(x, type) (convert_##type((x)))
 #define CONVERT(x, type) CONVERT_STR(x, type)

 #define CONVERT_SAT_STR(x, type) (convert_##type##_sat((x)))
 #define CONVERT_SAT(x, type) CONVERT_SAT_STR(x, type)

 #define CONVERT_SAT_ROUND_STR(x, type, round) (convert_##type##_sat_##round((x)))
 #define CONVERT_SAT_ROUND(x, type, round) CONVERT_SAT_ROUND_STR(x, type, round)

 #define select_vec_dt_uchar(size) uchar##size
 #define select_vec_dt_char(size) char##size
 #define select_vec_dt_ushort(size) ushort##size
 #define select_vec_dt_short(size) short##size
 #define select_vec_dt_half(size) short##size
 #define select_vec_dt_uint(size) uint##size
 #define select_vec_dt_int(size) int##size
 #define select_vec_dt_float(size) int##size
 #define select_vec_dt_ulong(size) ulong##size
 #define select_vec_dt_long(size) long##size

 #define SELECT_VEC_DATA_TYPE_STR(type, size) select_vec_dt_##type(size)
 #define SELECT_VEC_DATA_TYPE(type, size) SELECT_VEC_DATA_TYPE_STR(type, size)
 #define SELECT_DATA_TYPE(type) SELECT_VEC_DATA_TYPE_STR(type, 1)

 #define sum_reduce_1(x) (x)
 #define sum_reduce_2(x) ((x).s0) + ((x).s1)
 #define sum_reduce_3(x) sum_reduce_2((x).s01) + ((x).s2)
 #define sum_reduce_4(x) sum_reduce_2((x).s01) + sum_reduce_2((x).s23)
 #define sum_reduce_8(x) sum_reduce_4((x).s0123) + sum_reduce_4((x).s4567)
 #define sum_reduce_16(x) sum_reduce_8((x).s01234567) + sum_reduce_8((x).s89ABCDEF)

 #define SUM_REDUCE_STR(x, size) sum_reduce_##size(x)
 #define SUM_REDUCE(x, size) SUM_REDUCE_STR(x, size)

 #define max_reduce_1(x) (x)
 #define max_reduce_2(x) max(((x).s0), ((x).s1))
 #define max_reduce_3(x) max(max_reduce_2((x).s01), ((x).s2))
 #define max_reduce_4(x) max(max_reduce_2((x).s01), max_reduce_2((x).s23))
 #define max_reduce_8(x) max(max_reduce_4((x).s0123), max_reduce_4((x).s4567))
 #define max_reduce_16(x) max(max_reduce_8((x).s01234567), max_reduce_8((x).s89ABCDEF))

 #define MAX_REDUCE_STR(x, size) max_reduce_##size(x)
 #define MAX_REDUCE(x, size) MAX_REDUCE_STR(x, size)

 #define VECTOR_DECLARATION(name)     \
     __global uchar *name##_ptr,      \
     uint        name##_stride_x, \
     uint        name##_step_x,   \
     uint        name##_offset_first_element_in_bytes

 #define IMAGE_DECLARATION(name)      \
     __global uchar *name##_ptr,      \
     uint        name##_stride_x, \
     uint        name##_step_x,   \
     uint        name##_stride_y, \
     uint        name##_step_y,   \
     uint        name##_offset_first_element_in_bytes

 #define TENSOR3D_DECLARATION(name)   \
     __global uchar *name##_ptr,      \
     uint        name##_stride_x, \
     uint        name##_step_x,   \
     uint        name##_stride_y, \
     uint        name##_step_y,   \
     uint        name##_stride_z, \
     uint        name##_step_z,   \
     uint        name##_offset_first_element_in_bytes

 #define TENSOR4D_DECLARATION(name)   \
     __global uchar *name##_ptr,      \
     uint        name##_stride_x, \
     uint        name##_step_x,   \
     uint        name##_stride_y, \
     uint        name##_step_y,   \
     uint        name##_stride_z, \
     uint        name##_step_z,   \
     uint        name##_stride_w, \
     uint        name##_step_w,   \
     uint        name##_offset_first_element_in_bytes

 #define CONVERT_TO_VECTOR_STRUCT(name) \
     update_vector_workitem_ptr(name##_ptr, name##_offset_first_element_in_bytes, name##_stride_x, name##_step_x)

 #define CONVERT_TO_VECTOR_STRUCT_NO_STEP(name) \
     update_vector_workitem_ptr(name##_ptr, name##_offset_first_element_in_bytes, name##_stride_x, 0)

 #define CONVERT_TO_IMAGE_STRUCT(name) \
     update_image_workitem_ptr(name##_ptr, name##_offset_first_element_in_bytes, name##_stride_x, name##_step_x, name##_stride_y, name##_step_y)

 #define CONVERT_TO_IMAGE_STRUCT_NO_STEP(name) \
     update_image_workitem_ptr(name##_ptr, name##_offset_first_element_in_bytes, name##_stride_x, 0, name##_stride_y, 0)

 #define CONVERT_TENSOR3D_TO_IMAGE_STRUCT(name) \
     update_image_from_tensor3D_workitem_ptr(name##_ptr, name##_offset_first_element_in_bytes, name##_stride_x, name##_step_x, name##_stride_y, name##_step_y, name##_stride_z, name##_step_z)

 #define CONVERT_TENSOR3D_TO_IMAGE_STRUCT_NO_STEP(name) \
     update_image_from_tensor3D_workitem_ptr(name##_ptr, name##_offset_first_element_in_bytes, name##_stride_x, 0, name##_stride_y, 0, name##_stride_z, name##_step_z)

 #define CONVERT_TENSOR3D_TO_IMAGE_STRUCT(name) \
     update_image_from_tensor3D_workitem_ptr(name##_ptr, name##_offset_first_element_in_bytes, name##_stride_x, name##_step_x, name##_stride_y, name##_step_y, name##_stride_z, name##_step_z)

 #define CONVERT_TO_TENSOR3D_STRUCT(name)                                                                                                           \
     update_tensor3D_workitem_ptr(name##_ptr, name##_offset_first_element_in_bytes, name##_stride_x, name##_step_x, name##_stride_y, name##_step_y, \
                                  name##_stride_z, name##_step_z)

 #define CONVERT_TO_TENSOR3D_STRUCT_NO_STEP(name) \
     update_tensor3D_workitem_ptr(name##_ptr, name##_offset_first_element_in_bytes, name##_stride_x, 0, name##_stride_y, 0, name##_stride_z, 0)

 #define CONVERT_TO_TENSOR4D_STRUCT(name, mod_size)                                                                                                 \
     update_tensor4D_workitem_ptr(name##_ptr, name##_offset_first_element_in_bytes, name##_stride_x, name##_step_x, name##_stride_y, name##_step_y, \
                                  name##_stride_z, name##_step_z, name##_stride_w, name##_step_w, mod_size)

 #define CONVERT_TO_TENSOR4D_STRUCT_NO_STEP(name, mod_size) \
     update_tensor4D_workitem_ptr(name##_ptr, name##_offset_first_element_in_bytes, name##_stride_x, 0, name##_stride_y, 0, name##_stride_z, 0, name##_stride_w, 0, mod_size)

 #define CONVERT_TO_TENSOR3D_STRUCT_NO_UPDATE_PTR(name)                                                                                       \
     tensor3D_ptr_no_update(name##_ptr, name##_offset_first_element_in_bytes, name##_stride_x, name##_step_x, name##_stride_y, name##_step_y, \
                            name##_stride_z, name##_step_z)

 /** Structure to hold Vector information */
 typedef struct Vector
 {
     __global uchar *ptr;                           /**< Pointer to the starting postion of the buffer */
     int             offset_first_element_in_bytes; /**< The offset of the first element in the source image */
     int             stride_x;                      /**< Stride of the image in X dimension (in bytes) */
 } Vector;

 /** Structure to hold Image information */
 typedef struct Image
 {
     __global uchar *ptr;                           /**< Pointer to the starting postion of the buffer */
     int             offset_first_element_in_bytes; /**< The offset of the first element in the source image */
     int             stride_x;                      /**< Stride of the image in X dimension (in bytes) */
     int             stride_y;                      /**< Stride of the image in Y dimension (in bytes) */
 } Image;

 /** Structure to hold 3D tensor information */
 typedef struct Tensor3D
 {
     __global uchar *ptr;                           /**< Pointer to the starting postion of the buffer */
     int             offset_first_element_in_bytes; /**< The offset of the first element in the source image */
     int             stride_x;                      /**< Stride of the image in X dimension (in bytes) */
     int             stride_y;                      /**< Stride of the image in Y dimension (in bytes) */
     int             stride_z;                      /**< Stride of the image in Z dimension (in bytes) */
 } Tensor3D;

 /** Structure to hold 4D tensor information */
 typedef struct Tensor4D
 {
     __global uchar *ptr;                           /**< Pointer to the starting postion of the buffer */
     int             offset_first_element_in_bytes; /**< The offset of the first element in the source image */
     int             stride_x;                      /**< Stride of the image in X dimension (in bytes) */
     int             stride_y;                      /**< Stride of the image in Y dimension (in bytes) */
     int             stride_z;                      /**< Stride of the image in Z dimension (in bytes) */
     int             stride_w;                      /**< Stride of the image in W dimension (in bytes) */
 } Tensor4D;

 /** Wrap vector information into an Vector structure, and make the pointer point at this workitem's data.
  *
  * @param[in] ptr                           Pointer to the starting postion of the buffer
  * @param[in] offset_first_element_in_bytes The offset of the first element in the source vector
  * @param[in] stride_x                      Stride of the vector in X dimension (in bytes)
  * @param[in] step_x                        stride_x * number of elements along X processed per workitem(in bytes)
  *
  * @return An image object
  */
 inline Vector update_vector_workitem_ptr(__global uchar *ptr, uint offset_first_element_in_bytes, uint stride_x, uint step_x)
 {
     Vector vector =
     {
         .ptr                           = ptr,
         .offset_first_element_in_bytes = offset_first_element_in_bytes,
         .stride_x                      = stride_x,
     };
     vector.ptr += vector.offset_first_element_in_bytes + get_global_id(0) * step_x;
     return vector;
 }

 /** Wrap image information into an Image structure, and make the pointer point at this workitem's data.
  *
  * @param[in] ptr                           Pointer to the starting postion of the buffer
  * @param[in] offset_first_element_in_bytes The offset of the first element in the source image
  * @param[in] stride_x                      Stride of the image in X dimension (in bytes)
  * @param[in] step_x                        stride_x * number of elements along X processed per workitem(in bytes)
  * @param[in] stride_y                      Stride of the image in Y dimension (in bytes)
  * @param[in] step_y                        stride_y * number of elements along Y processed per workitem(in bytes)
  *
  * @return An image object
  */
 inline Image update_image_workitem_ptr(__global uchar *ptr, uint offset_first_element_in_bytes, uint stride_x, uint step_x, uint stride_y, uint step_y)
 {
     Image img =
     {
         .ptr                           = ptr,
         .offset_first_element_in_bytes = offset_first_element_in_bytes,
         .stride_x                      = stride_x,
         .stride_y                      = stride_y
     };
     img.ptr += img.offset_first_element_in_bytes + get_global_id(0) * step_x + get_global_id(1) * step_y;
     return img;
 }

 /** Wrap 3D tensor information into an image structure, and make the pointer point at this workitem's data.
  *
  * @param[in] ptr                           Pointer to the starting postion of the buffer
  * @param[in] offset_first_element_in_bytes The offset of the first element in the source image
  * @param[in] stride_x                      Stride of the image in X dimension (in bytes)
  * @param[in] step_x                        stride_x * number of elements along X processed per workitem(in bytes)
  * @param[in] stride_y                      Stride of the image in Y dimension (in bytes)
  * @param[in] step_y                        stride_y * number of elements along Y processed per workitem(in bytes)
  * @param[in] stride_z                      Stride of the image in Z dimension (in bytes)
  * @param[in] step_z                        stride_z * number of elements along Z processed per workitem(in bytes)
  *
  * @return A 3D tensor object
  */
 inline Image update_image_from_tensor3D_workitem_ptr(__global uchar *ptr, uint offset_first_element_in_bytes, uint stride_x, uint step_x, uint stride_y, uint step_y, uint stride_z, uint step_z)
 {
     Image img =
     {
         .ptr                           = ptr,
         .offset_first_element_in_bytes = offset_first_element_in_bytes,
         .stride_x                      = stride_x,
         .stride_y                      = stride_y
     };
     img.ptr += img.offset_first_element_in_bytes + get_global_id(0) * step_x + get_global_id(1) * step_y + get_global_id(2) * step_z;
     return img;
 }

 /** Wrap 3D tensor information into an tensor structure, and make the pointer point at this workitem's data.
  *
  * @param[in] ptr                           Pointer to the starting postion of the buffer
  * @param[in] offset_first_element_in_bytes The offset of the first element in the source image
  * @param[in] stride_x                      Stride of the image in X dimension (in bytes)
  * @param[in] step_x                        stride_x * number of elements along X processed per workitem(in bytes)
  * @param[in] stride_y                      Stride of the image in Y dimension (in bytes)
  * @param[in] step_y                        stride_y * number of elements along Y processed per workitem(in bytes)
  * @param[in] stride_z                      Stride of the image in Z dimension (in bytes)
  * @param[in] step_z                        stride_z * number of elements along Z processed per workitem(in bytes)
  *
  * @return A 3D tensor object
  */
 inline Tensor3D update_tensor3D_workitem_ptr(__global uchar *ptr, uint offset_first_element_in_bytes, uint stride_x, uint step_x, uint stride_y, uint step_y, uint stride_z, uint step_z)
 {
     Tensor3D tensor =
     {
         .ptr                           = ptr,
         .offset_first_element_in_bytes = offset_first_element_in_bytes,
         .stride_x                      = stride_x,
         .stride_y                      = stride_y,
         .stride_z                      = stride_z
     };
     tensor.ptr += tensor.offset_first_element_in_bytes + get_global_id(0) * step_x + get_global_id(1) * step_y + get_global_id(2) * step_z;
     return tensor;
 }

 /** Wrap 3D tensor information into an tensor structure.
  *
  * @param[in] ptr                           Pointer to the starting postion of the buffer
  * @param[in] offset_first_element_in_bytes The offset of the first element in the source image
  * @param[in] stride_x                      Stride of the image in X dimension (in bytes)
  * @param[in] step_x                        stride_x * number of elements along X processed per workitem(in bytes)
  * @param[in] stride_y                      Stride of the image in Y dimension (in bytes)
  * @param[in] step_y                        stride_y * number of elements along Y processed per workitem(in bytes)
  * @param[in] stride_z                      Stride of the image in Z dimension (in bytes)
  * @param[in] step_z                        stride_z * number of elements along Z processed per workitem(in bytes)
  *
  * @return A 3D tensor object
  */
 inline Tensor3D tensor3D_ptr_no_update(__global uchar *ptr, uint offset_first_element_in_bytes, uint stride_x, uint step_x, uint stride_y, uint step_y, uint stride_z, uint step_z)
 {
     Tensor3D tensor =
     {
         .ptr                           = ptr,
         .offset_first_element_in_bytes = offset_first_element_in_bytes,
         .stride_x                      = stride_x,
         .stride_y                      = stride_y,
         .stride_z                      = stride_z
     };
     return tensor;
 }

 inline Tensor4D update_tensor4D_workitem_ptr(__global uchar *ptr, uint offset_first_element_in_bytes, uint stride_x, uint step_x, uint stride_y, uint step_y, uint stride_z, uint step_z, uint stride_w,
                                              uint step_w,
                                              uint mod_size)
 {
     Tensor4D tensor =
     {
         .ptr                           = ptr,
         .offset_first_element_in_bytes = offset_first_element_in_bytes,
         .stride_x                      = stride_x,
         .stride_y                      = stride_y,
         .stride_z                      = stride_z,
         .stride_w                      = stride_w
     };

     tensor.ptr += tensor.offset_first_element_in_bytes + get_global_id(0) * step_x + get_global_id(1) * step_y + (get_global_id(2) % mod_size) * step_z + (get_global_id(2) / mod_size) * step_w;
     return tensor;
 }

 /** Get the pointer position of a Vector
  *
  * @param[in] vec Pointer to the starting position of the buffer
  * @param[in] x   Relative X position
  */
 inline __global const uchar *vector_offset(const Vector *vec, int x)
 {
     return vec->ptr + x * vec->stride_x;
 }

 /** Get the pointer position of a Image
  *
  * @param[in] img Pointer to the starting position of the buffer
  * @param[in] x   Relative X position
  * @param[in] y   Relative Y position
  */
 inline __global uchar *offset(const Image *img, int x, int y)
 {
     return img->ptr + x * img->stride_x + y * img->stride_y;
 }

 /** Get the pointer position of a Tensor3D
  *
  * @param[in] tensor Pointer to the starting position of the buffer
  * @param[in] x      Relative X position
  * @param[in] y      Relative Y position
  * @param[in] z      Relative Z position
  */
 inline __global const uchar *tensor3D_offset(const Tensor3D *tensor, int x, int y, int z)
 {
     return tensor->ptr + x * tensor->stride_x + y * tensor->stride_y + z * tensor->stride_z;
 }

 /** Get the pointer position of a Tensor4D
  *
  * @param[in] tensor Pointer to the starting position of the buffer
  * @param[in] x      Relative X position
  * @param[in] y      Relative Y position
  * @param[in] z      Relative Z position
  * @param[in] w      Relative W position
  */
 inline __global const uchar *tensor4D_offset(const Tensor4D *tensor, int x, int y, int z, int w)
 {
     return tensor->ptr + x * tensor->stride_x + y * tensor->stride_y + z * tensor->stride_z + w * tensor->stride_w;
 }

 /** Get the offset for a given linear index of a Tensor3D
  *
  * @param[in] tensor Pointer to the starting position of the buffer
  * @param[in] width  Width of the input tensor
  * @param[in] height Height of the input tensor
  * @param[in] depth  Depth of the input tensor
  * @param[in] index  Linear index
  */
 inline __global const uchar *tensor3D_index2ptr(const Tensor3D *tensor, uint width, uint height, uint depth, uint index)
 {
     uint num_elements = width * height;

     const uint z = index / num_elements;

     index %= num_elements;

     const uint y = index / width;

     index %= width;

     const uint x = index;

     return tensor->ptr + x * tensor->stride_x + y * tensor->stride_y + z * tensor->stride_z + tensor->offset_first_element_in_bytes;
 }

 #endif // _HELPER_H

 /*
  * Copyright (c) 2019-2020 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to
  * deal in the Software without restriction, including without limitation the
  * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
  * The above copyright notice and this permission notice shall be included in all
  * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  * SOFTWARE.
  */

 /*
  * Copyright (c) 2016-2020 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to
  * deal in the Software without restriction, including without limitation the
  * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
  * The above copyright notice and this permission notice shall be included in all
  * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  * SOFTWARE.
  */
 #ifndef ARM_COMPUTE_HELPER_H
 #define ARM_COMPUTE_HELPER_H

 /*
  * Copyright (c) 2020 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to
  * deal in the Software without restriction, including without limitation the
  * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
  * The above copyright notice and this permission notice shall be included in all
  * copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  * SOFTWARE.
  */

 /** Store the 0 to (n-1)th rows of the given variables
  * @name STORE_ROW_n
  *
  * @param[in] N0        The width of the passed in vector. Supported: 1, 2, 3, 4, 8, 16
  * @param[in] DATA_TYPE The data type of the vectors
  * @param[in] BASENAME  The basename of the variables
  * @param[in] PTR       The base pointer
  * @param[in] STRIDE_Y  The stride value in y-axis direction
  * @param[in] Z         The offset in z-axis direction
  * @{
  */
 #define STORE_ROW_1(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     VSTORE(N0)                                                 \
     (BASENAME##0, 0, (__global DATA_TYPE *)(PTR + 0 * STRIDE_Y + Z##0));

 #define STORE_ROW_2(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_1(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE(N0)                                                 \
     (BASENAME##1, 0, (__global DATA_TYPE *)(PTR + 1 * STRIDE_Y + Z##1));

 #define STORE_ROW_3(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_2(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE(N0)                                                 \
     (BASENAME##2, 0, (__global DATA_TYPE *)(PTR + 2 * STRIDE_Y + Z##2));

 #define STORE_ROW_4(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_3(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE(N0)                                                 \
     (BASENAME##3, 0, (__global DATA_TYPE *)(PTR + 3 * STRIDE_Y + Z##3));

 #define STORE_ROW_5(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_4(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE(N0)                                                 \
     (BASENAME##4, 0, (__global DATA_TYPE *)(PTR + 4 * STRIDE_Y + Z##4));

 #define STORE_ROW_6(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_5(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE(N0)                                                 \
     (BASENAME##5, 0, (__global DATA_TYPE *)(PTR + 5 * STRIDE_Y + Z##5));

 #define STORE_ROW_7(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_6(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE(N0)                                                 \
     (BASENAME##6, 0, (__global DATA_TYPE *)(PTR + 6 * STRIDE_Y + Z##6));

 #define STORE_ROW_8(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_7(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE(N0)                                                 \
     (BASENAME##7, 0, (__global DATA_TYPE *)(PTR + 7 * STRIDE_Y + Z##7));

 #define STORE_ROW_9(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_8(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE(N0)                                                 \
     (BASENAME##8, 0, (__global DATA_TYPE *)(PTR + 8 * STRIDE_Y + Z##8));

 #define STORE_ROW_10(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_9(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)      \
     VSTORE(N0)                                                  \
     (BASENAME##9, 0, (__global DATA_TYPE *)(PTR + 9 * STRIDE_Y + Z##9));

 #define STORE_ROW_11(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_10(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE(N0)                                                  \
     (BASENAME##A, 0, (__global DATA_TYPE *)(PTR + 10 * STRIDE_Y + Z##A));

 #define STORE_ROW_12(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_11(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE(N0)                                                  \
     (BASENAME##B, 0, (__global DATA_TYPE *)(PTR + 11 * STRIDE_Y + Z##B));

 #define STORE_ROW_13(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_12(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE(N0)                                                  \
     (BASENAME##C, 0, (__global DATA_TYPE *)(PTR + 12 * STRIDE_Y + Z##C));

 #define STORE_ROW_14(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_13(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE(N0)                                                  \
     (BASENAME##D, 0, (__global DATA_TYPE *)(PTR + 13 * STRIDE_Y + Z##D));

 #define STORE_ROW_15(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_14(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE(N0)                                                  \
     (BASENAME##E, 0, (__global DATA_TYPE *)(PTR + 14 * STRIDE_Y + Z##E));

 #define STORE_ROW_16(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_15(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE(N0)                                                  \
     (BASENAME##F, 0, (__global DATA_TYPE *)(PTR + 15 * STRIDE_Y + Z##F));
 /** @} */ // end of groupd STORE_ROW_n

 /** Convert and store the 0th to (n-1)th rows of the given variables
  * @name CONVERT_STORE_ROW_n
  *
  * @param[in] N0        The size of the vectors
  * @param[in] DATA_TYPE The data type of the vectors
  * @param[in] BASENAME  The basename of the variables
  * @param[in] PTR       The base pointer
  * @param[in] STRIDE_Y  The stride value in y-axis direction
  * @param[in] Z         The offset in z-axis direction
  * @{
  */
 #define CONVERT_STORE_ROW_1(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     VSTORE(N0)                                                         \
     (CONVERT_SAT((BASENAME##0), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 0 * STRIDE_Y + Z##0));

 #define CONVERT_STORE_ROW_2(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     CONVERT_STORE_ROW_1(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE(N0)                                                         \
     (CONVERT_SAT((BASENAME##1), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 1 * STRIDE_Y + Z##1));

 #define CONVERT_STORE_ROW_3(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     CONVERT_STORE_ROW_2(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE(N0)                                                         \
     (CONVERT_SAT((BASENAME##2), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 2 * STRIDE_Y + Z##2));

 #define CONVERT_STORE_ROW_4(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     CONVERT_STORE_ROW_3(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE(N0)                                                         \
     (CONVERT_SAT((BASENAME##3), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 3 * STRIDE_Y + Z##3));

 #define CONVERT_STORE_ROW_5(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     CONVERT_STORE_ROW_4(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE(N0)                                                         \
     (CONVERT_SAT((BASENAME##4), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 4 * STRIDE_Y + Z##4));

 #define CONVERT_STORE_ROW_6(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     CONVERT_STORE_ROW_5(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE(N0)                                                         \
     (CONVERT_SAT((BASENAME##5), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 5 * STRIDE_Y + Z##5));

 #define CONVERT_STORE_ROW_7(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     CONVERT_STORE_ROW_6(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE(N0)                                                         \
     (CONVERT_SAT((BASENAME##6), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 6 * STRIDE_Y + Z##6));

 #define CONVERT_STORE_ROW_8(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     CONVERT_STORE_ROW_7(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE(N0)                                                         \
     (CONVERT_SAT((BASENAME##7), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 7 * STRIDE_Y + Z##7));

 #define CONVERT_STORE_ROW_9(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     CONVERT_STORE_ROW_8(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE(N0)                                                         \
     (CONVERT_SAT((BASENAME##8), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 8 * STRIDE_Y + Z##8));

 #define CONVERT_STORE_ROW_10(N0, DATA, BASENAME, PTR, STRIDE_Y, Z) \
     CONVERT_STORE_ROW_9(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     VSTORE(N0)                                                     \
     (CONVERT_SAT((BASENAME##9), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 9 * STRIDE_Y + Z##9));

 #define CONVERT_STORE_ROW_11(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     CONVERT_STORE_ROW_10(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE(N0)                                                          \
     (CONVERT_SAT((BASENAME##A), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 10 * STRIDE_Y + Z##A));

 #define CONVERT_STORE_ROW_12(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     CONVERT_STORE_ROW_11(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE(N0)                                                          \
     (CONVERT_SAT((BASENAME##B), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 11 * STRIDE_Y + Z##B));

 #define CONVERT_STORE_ROW_13(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     CONVERT_STORE_ROW_12(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE(N0)                                                          \
     (CONVERT_SAT((BASENAME##C), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 12 * STRIDE_Y + Z##C));

 #define CONVERT_STORE_ROW_14(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     CONVERT_STORE_ROW_13(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE(N0)                                                          \
     (CONVERT_SAT((BASENAME##D), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 13 * STRIDE_Y + Z##D));

 #define CONVERT_STORE_ROW_15(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     CONVERT_STORE_ROW_14(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE(N0)                                                          \
     (CONVERT_SAT((BASENAME##E), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 14 * STRIDE_Y + Z##E));

 #define CONVERT_STORE_ROW_16(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     CONVERT_STORE_ROW_15(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE(N0)                                                          \
     (CONVERT_SAT((BASENAME##F), VEC_DATA_TYPE(DATA_TYPE, N0)), 0, (__global DATA_TYPE *)(PTR + 15 * STRIDE_Y + Z##F));

 /** @} */ // end of groupd CONVERT_STORE_ROW_n

 /** Store a block of the given size M0xN0
  * @name STORE_BLOCK
  *
  * Supported cases are M0=1,2,3,...,16 and N0=2,3,4,8,16.
  * The data to store is expected to have consecutive names for each row.
  * E.g., for M0=3 and basename=c, the expected names are c0, c1 and c2.
  * The Z offset is expected to have consecutive names.
  * E.g., for M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2.
  *
  * @param[in] M0        The number of rows to store
  * @param[in] N0        The size of each vector
  * @param[in] DATA_TYPE The data type of the vectors
  * @param[in] BASENAME  The basename of the variables
  * @param[in] PTR       The base pointer
  * @param[in] STRIDE_Y  The stride value in y-axis direction
  * @param[in] Z         The offset in z-axis direction
  * @{
  */
 #define STORE_BLOCK_STR(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) STORE_ROW_##M0(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)
 #define STORE_BLOCK(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) STORE_BLOCK_STR(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)
 /** @} */ // end of group STORE_BLOCK

 /** Convert and store a block of the given size M0xN0
  * @name CONVERT_STORE_BLOCK
  *
  * Supported cases are M0=1,2,3,...,16 and N0=2,3,4,8,16.
  * The data to store is expected to have consecutive names for each row.
  * E.g., for M0=3 and basename=c, the expected names are c0, c1 and c2.
  * The Z offset is expected to have consecutive names.
  * E.g., for M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2.
  *
  * @param[in] M0        The number of rows to store
  * @param[in] N0        The size of each vector
  * @param[in] DATA_TYPE The data type of the vectors
  * @param[in] BASENAME  The basename of the variables
  * @param[in] PTR       The base pointer
  * @param[in] STRIDE_Y  The stride value in y-axis direction
  * @param[in] Z         The offset in z-axis direction
  * @{
  */
 #define CONVERT_STORE_BLOCK_STR(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) CONVERT_STORE_ROW_##M0(N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)
 #define CONVERT_STORE_BLOCK(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) CONVERT_STORE_BLOCK_STR(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)
 /** @} */ // end of group CONVERT_STORE_BLOCK

 /** Partially store the 0 to (n-1)th rows of the given variables
  * @name STORE_ROW_PARTIAL_n
  * Within each row, store the lower @p STORE_N0 elements of vectors of width @p N0
  *
  * @note in case @p STORE_N0 != 1, 2, 3, 4, 8, 16, extra vstore(s) will be invoked, thus incurring small performance penalty.
  *
  * @param[in] N0        The width of the passed in vector. Supported: 1, 2, 3, 4, 8, 16
  * @param[in] STORE_N0  The **lower** size of the vectors to store. Supported: [1-16 and <= @p N0
  * @param[in] DATA_TYPE The data type of the vectors
  * @param[in] BASENAME  The basename of the variables
  * @param[in] PTR       The base pointer
  * @param[in] STRIDE_Y  The stride value in y-axis direction
  * @param[in] Z         The offset in z-axis direction
  * @{
  */
 #define STORE_ROW_PARTIAL_1(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     VSTORE_PARTIAL(N0, STORE_N0)                                                 \
     (BASENAME##0, 0, (__global DATA_TYPE *)(PTR + 0 * STRIDE_Y + Z##0));

 #define STORE_ROW_PARTIAL_2(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_PARTIAL_1(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE_PARTIAL(N0, STORE_N0)                                                 \
     (BASENAME##1, 0, (__global DATA_TYPE *)(PTR + 1 * STRIDE_Y + Z##1));

 #define STORE_ROW_PARTIAL_3(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_PARTIAL_2(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE_PARTIAL(N0, STORE_N0)                                                 \
     (BASENAME##2, 0, (__global DATA_TYPE *)(PTR + 2 * STRIDE_Y + Z##2));

 #define STORE_ROW_PARTIAL_4(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_PARTIAL_3(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE_PARTIAL(N0, STORE_N0)                                                 \
     (BASENAME##3, 0, (__global DATA_TYPE *)(PTR + 3 * STRIDE_Y + Z##3));

 #define STORE_ROW_PARTIAL_5(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_PARTIAL_4(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE_PARTIAL(N0, STORE_N0)                                                 \
     (BASENAME##4, 0, (__global DATA_TYPE *)(PTR + 4 * STRIDE_Y + Z##4));

 #define STORE_ROW_PARTIAL_6(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_PARTIAL_5(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE_PARTIAL(N0, STORE_N0)                                                 \
     (BASENAME##5, 0, (__global DATA_TYPE *)(PTR + 5 * STRIDE_Y + Z##5));

 #define STORE_ROW_PARTIAL_7(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_PARTIAL_6(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE_PARTIAL(N0, STORE_N0)                                                 \
     (BASENAME##6, 0, (__global DATA_TYPE *)(PTR + 6 * STRIDE_Y + Z##6));

 #define STORE_ROW_PARTIAL_8(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_PARTIAL_7(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE_PARTIAL(N0, STORE_N0)                                                 \
     (BASENAME##7, 0, (__global DATA_TYPE *)(PTR + 7 * STRIDE_Y + Z##7));

 #define STORE_ROW_PARTIAL_9(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_PARTIAL_8(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE_PARTIAL(N0, STORE_N0)                                                 \
     (BASENAME##8, 0, (__global DATA_TYPE *)(PTR + 8 * STRIDE_Y + Z##8));

 #define STORE_ROW_PARTIAL_10(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_PARTIAL_9(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)      \
     VSTORE_PARTIAL(N0, STORE_N0)                                                  \
     (BASENAME##9, 0, (__global DATA_TYPE *)(PTR + 9 * STRIDE_Y + Z##9));

 #define STORE_ROW_PARTIAL_11(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_PARTIAL_10(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE_PARTIAL(N0, STORE_N0)                                                  \
     (BASENAME##A, 0, (__global DATA_TYPE *)(PTR + 10 * STRIDE_Y + Z##A));

 #define STORE_ROW_PARTIAL_12(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_PARTIAL_11(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE_PARTIAL(N0, STORE_N0)                                                  \
     (BASENAME##B, 0, (__global DATA_TYPE *)(PTR + 11 * STRIDE_Y + Z##B));

 #define STORE_ROW_PARTIAL_13(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_PARTIAL_12(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE_PARTIAL(N0, STORE_N0)                                                  \
     (BASENAME##C, 0, (__global DATA_TYPE *)(PTR + 12 * STRIDE_Y + Z##C));

 #define STORE_ROW_PARTIAL_14(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_PARTIAL_13(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE_PARTIAL(N0, STORE_N0)                                                  \
     (BASENAME##D, 0, (__global DATA_TYPE *)(PTR + 13 * STRIDE_Y + Z##D));

 #define STORE_ROW_PARTIAL_15(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_PARTIAL_14(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE_PARTIAL(N0, STORE_N0)                                                  \
     (BASENAME##E, 0, (__global DATA_TYPE *)(PTR + 14 * STRIDE_Y + Z##E));

 #define STORE_ROW_PARTIAL_16(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \
     STORE_ROW_PARTIAL_15(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)     \
     VSTORE_PARTIAL(N0, STORE_N0)                                                  \
     (BASENAME##F, 0, (__global DATA_TYPE *)(PTR + 15 * STRIDE_Y + Z##F));
 /** @} */ // end of groupd STORE_ROW_PARTIAL_n

 /** Partially store a block of the given size STORE_M0xSTORE_N0
  * @name STORE_BLOCK_PARTIAL
  *
  * @note The vector width @p N0 is also required for correct partial storing behaviour.
  * @note in case @p STORE_N0 != 1, 2, 3, 4, 8, 16, extra vstore(s) will be invoked, thus incurring small performance penalty.
  *
  * The data to store is expected to have consecutive names for each row.
  * E.g., for STORE_M0=3 and basename=c, the expected names are c0, c1 and c2.
  * The Z offset is expected to have consecutive names.
  * E.g., for STORE_M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2.
  *
  * @param[in] STORE_M0  The number of rows to store. Supported: 1-16
  * @param[in] STORE_N0  The lower number of elements of vectors to store. Supported: 1-16 and <= @p N0
  * @param[in] N0        The size of each vector. Supported: 1, 2, 3, 4, 8, 16
  * @param[in] DATA_TYPE The data type of the vectors
  * @param[in] BASENAME  The basename of the variables
  * @param[in] PTR       The base pointer
  * @param[in] STRIDE_Y  The stride value in y-axis direction
  * @param[in] Z         The offset in z-axis direction
  * @{
  */
 #define STORE_BLOCK_PARTIAL_STR(STORE_M0, STORE_N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) STORE_ROW_PARTIAL_##STORE_M0(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)
 #define STORE_BLOCK_PARTIAL(STORE_M0, STORE_N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) STORE_BLOCK_PARTIAL_STR(STORE_M0, STORE_N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)
 /** Store a block that can be partial in both x and y dimensions
  *
  * @note in cases @p PARTIAL_STORE_N0 != 1, 2, 3, 4, 8, 16, extra vstore(s) will be invoked, thus incurring small performance penalty.
  *
  * The data to store is expected to have consecutive names for each row.
  * E.g., for M0=3 and basename=c, the expected names are c0, c1 and c2.
  * The Z offset is expected to have consecutive names.
  * E.g., for M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2.
  *
  * @param[in] M0               The number of rows to store, for non-partial blocks. Supported: 1-16
  * @param[in] N0               The size of each vector, for non-partial blocks. Supported: 1, 2, 3, 4, 8, 16
  * @param[in] DATA_TYPE        The data type of the vectors
  * @param[in] BASENAME         The basename of the variables
  * @param[in] PTR              The base pointer
  * @param[in] STRIDE_Y         The stride value in y-axis direction
  * @param[in] Z                The offset in z-axis direction
  * @param[in] PARTIAL_STORE_M0 The partial size in y, for partial blocks. Supported range: [1, @p M0)
  * @param[in] PARTIAL_STORE_N0 The partial size in x, for partial blocks. Supported range: [1, @p N0)
  * @param[in] PARTIAL_COND_Y   Condition on the y axis to perform the partial store Y. True to use PARTIAL_STORE_M0 rather than M0.
  * @param[in] PARTIAL_COND_X   Condition on the x axis to perform the partial store X. True to use PARTIAL_STORE_N0 rather than N0.
  */
 #define STORE_BLOCK_PARTIAL_IN_X_AND_Y(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_STORE_N0, PARTIAL_COND_Y, PARTIAL_COND_X) \
     if(!(PARTIAL_COND_X) && !(PARTIAL_COND_Y))                                                                                                            \
     {                                                                                                                                                     \
         STORE_BLOCK_PARTIAL(M0, N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z);                                                                           \
     }                                                                                                                                                     \
     else if((PARTIAL_COND_Y) && !(PARTIAL_COND_X))                                                                                                        \
     {                                                                                                                                                     \
         STORE_BLOCK_PARTIAL(PARTIAL_STORE_M0, N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z);                                                             \
     }                                                                                                                                                     \
     else if(!(PARTIAL_COND_Y) && (PARTIAL_COND_X))                                                                                                        \
     {                                                                                                                                                     \
         STORE_BLOCK_PARTIAL(M0, PARTIAL_STORE_N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z);                                                             \
     }                                                                                                                                                     \
     else                                                                                                                                                  \
     {                                                                                                                                                     \
         STORE_BLOCK_PARTIAL(PARTIAL_STORE_M0, PARTIAL_STORE_N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z);                                               \
     }
 /** Store a block that can only be partial in x but not y.
  *
  * @note in case @p N0 or @p PARTIAL_STORE_N0 != 1, 2, 3, 4, 8, 16, extra vstore(s) will be invoked, thus incurring small performance penalty.
  *
  * The data to store is expected to have consecutive names for each row.
  * E.g., for M0=3 and basename=c, the expected names are c0, c1 and c2.
  * The Z offset is expected to have consecutive names.
  * E.g., for M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2.
  *
  * @param[in] M0               The number of rows to store, for non-partial blocks. Supported: 1-16
  * @param[in] N0               The size of each vector, for non-partial blocks. Supported: 1, 2, 3, 4, 8, 16
  * @param[in] DATA_TYPE        The data type of the vectors
  * @param[in] BASENAME         The basename of the variables
  * @param[in] PTR              The base pointer
  * @param[in] STRIDE_Y         The stride value in y-axis direction
  * @param[in] Z                The offset in z-axis direction
  * @param[in] PARTIAL_STORE_N0 The partial size in x, for partial blocks. Supported range: [1, @p N0)
  * @param[in] PARTIAL_COND_X   Condition on the x axis to perform the partial store X. True to use PARTIAL_STORE_N0 rather than N0.
  */
 #define STORE_BLOCK_PARTIAL_IN_X(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_N0, PARTIAL_COND_X) \
     if(!(PARTIAL_COND_X))                                                                                         \
     {                                                                                                             \
         STORE_BLOCK_PARTIAL(M0, N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z);                                   \
     }                                                                                                             \
     else                                                                                                          \
     {                                                                                                             \
         STORE_BLOCK_PARTIAL(M0, PARTIAL_STORE_N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z);                     \
     }
 /** Store a block that can only be partial in y but not x.
  *
  * @note in case @p N0 or @p PARTIAL_STORE_N0 != 1, 2, 3, 4, 8, 16, extra vstore(s) will be invoked, thus incurring small performance penalty.
  *
  * The data to store is expected to have consecutive names for each row.
  * E.g., for M0=3 and basename=c, the expected names are c0, c1 and c2.
  * The Z offset is expected to have consecutive names.
  * E.g., for M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2.
  *
  * @param[in] M0               The number of rows to store, for non-partial blocks. Supported: 1-16
  * @param[in] N0               The size of each vector, for non-partial blocks. Supported: 1, 2, 3, 4, 8, 16
  * @param[in] DATA_TYPE        The data type of the vectors
  * @param[in] BASENAME         The basename of the variables
  * @param[in] PTR              The base pointer
  * @param[in] STRIDE_Y         The stride value in y-axis direction
  * @param[in] Z                The offset in z-axis direction
  * @param[in] PARTIAL_STORE_M0 The partial size in y, for partial blocks. Supported range: [1, @p M0)
  * @param[in] PARTIAL_COND_Y   Condition on the y axis to perform the partial store Y. True to use PARTIAL_STORE_M0 rather than M0.
  */
 #define STORE_BLOCK_PARTIAL_IN_Y(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_COND_Y) \
     if(!(PARTIAL_COND_Y))                                                                                         \
     {                                                                                                             \
         STORE_BLOCK_PARTIAL(M0, N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z);                                   \
     }                                                                                                             \
     else                                                                                                          \
     {                                                                                                             \
         STORE_BLOCK_PARTIAL(PARTIAL_STORE_M0, N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z);                     \
     }
 /** @} */ // end of group STORE_BLOCK_PARTIAL

 #if defined(PARTIAL_STORE_M0) && defined(PARTIAL_STORE_N0)

 /** Boundary-aware GEMM block store
  * @name STORE_BLOCK_BOUNDARY_AWARE
  * This macro assumes the following schemes to achieve boundary-awareness:
  *  - Overlapping load in Y axis from lhs tensor. This implies lhs has no padding along y dim.
  *  - Non-Overlapping(normal) load from rhs tensor. This imples rhs can have paddings.
  *  - Overlapping load in Y axis from bias tensor. This implies rhs has no padding along y dim.
  * The macro then ensures that the dst tensor can be stored without any paddings in both x and y dim.
  *
  * In the y dimension, we place the partial blocks **at the beginning** while in the x dimension, we place the partial
  * blocks **at the end**.
  * Say, the dst tensor is of shape MxN and we have M0 and N0 as the block size, this is how we define "partial blocks"/
  * "boundary block" (we use the 2 terms "partial blocks" and "boundary blocks" interchangeably) and its various parameters:
  *
  *  *--x-->                         x == 0                        x == 1
  *  |                  |<------------------------------N-------------------------->|
  *  y                  |<--------------N0------------->|<----PARTIAL_STORE_N0----->|
  *  |     -------------#############################################################
  *  *     |          | |...............................|...........................|
  * y == 0 | PAR_..._M0 |......Boundary block in y......|.Boundary block in x and y.|
  *        |          | |...............................|...........................|
  *        M          --#############################################################
  *        |          | |                               |...........................|
  * y == 1 |         M0 |      Non-boundary block       |....Boundary block in x....|
  *        |          | |                               |...........................|
  *        |------------#############################################################
  *
  * Then @p PARTIAL_STORE_M0 = M % M0      and @p PARTIAL_STORE_N0 = N % N0
  *
  * @note in cases @p PARTIAL_STORE_N0 != 1, 2, 3, 4, 8, 16, extra vstore(s) will be invoked, thus incurring small performance penalty.
  *
  * It automatically detects if a giving M,N,M0,N0 combination can yield partial blocks in either X and Y dimension,
  * and select corresponding store methods such that the boundary detection logic is only added when needed.
  *
  * The data to store is expected to have consecutive names for each row.
  * E.g., for M0=3 and basename=c, the expected names are c0, c1 and c2.
  * The Z offset is expected to have consecutive names.
  * E.g., for M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2.
  *
  * @param[in] M0               The number of rows to store, for non-partial blocks. Supported: 1-16
  * @param[in] N0               The size of each vector, for non-partial blocks. Supported: 1, 2, 3, 4, 8, 16
  * @param[in] DATA_TYPE        The data type of the vectors
  * @param[in] BASENAME         The basename of the variables
  * @param[in] PTR              The base pointer
  * @param[in] STRIDE_Y         The stride value in y-axis direction
  * @param[in] Z                The offset in z-axis direction
  * @param[in] PARTIAL_STORE_M0 The partial size in y, for partial blocks. Supported: [0, @p M0)
  * @param[in] PARTIAL_STORE_N0 The partial size in x, for partial blocks. Supported: [0, @p N0)
  * @param[in] PARTIAL_COND_Y   Condition on the y axis to perform the partial store Y. True to use PARTIAL_STORE_M0 rather than M0.
  * @param[in] PARTIAL_COND_X   Condition on the x axis to perform the partial store X. True to use PARTIAL_STORE_N0 rather than N0.
  * @{
  */
 #if PARTIAL_STORE_M0 == 0 && PARTIAL_STORE_N0 == 0
 // Case1: No partial blocks in either x or y
 #define STORE_BLOCK_BOUNDARY_AWARE(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_STORE_N0, PARTIAL_COND_Y, PARTIAL_COND_X) \
     STORE_BLOCK(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z)

 #elif PARTIAL_STORE_M0 > 0 && PARTIAL_STORE_N0 == 0
 // Case2: Partial blocks in y
 #define STORE_BLOCK_BOUNDARY_AWARE(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_STORE_N0, PARTIAL_COND_Y, PARTIAL_COND_X) \
     STORE_BLOCK_PARTIAL_IN_Y(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_COND_Y)

 #elif PARTIAL_STORE_M0 == 0 && PARTIAL_STORE_N0 > 0
 // Case3: Partial blocks in x
 #define STORE_BLOCK_BOUNDARY_AWARE(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_STORE_N0, PARTIAL_COND_Y, PARTIAL_COND_X) \
     STORE_BLOCK_PARTIAL_IN_X(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_N0, PARTIAL_COND_X)

 #else // PARTIAL_STORE_M0 == 0 && PARTIAL_STORE_N0 == 0
 // Case4: Partial blocks in both x and y
 #define STORE_BLOCK_BOUNDARY_AWARE(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_STORE_N0, PARTIAL_COND_Y, PARTIAL_COND_X) \
     STORE_BLOCK_PARTIAL_IN_X_AND_Y(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_STORE_N0, PARTIAL_COND_Y, PARTIAL_COND_X)

 #endif // PARTIAL_STORE_M0 == 0 && PARTIAL_STORE_N0 == 0

 #endif    // defined(PARTIAL_STORE_M0) && defined(PARTIAL_STORE_N0)
 /** @} */ // end of group STORE_BLOCK_BOUNDARY_AWARE

 #if defined(PARTIAL_STORE_M0)
 /** Compute the start m0 row (LHS, BIAS and DST) in a boundary-aware way so as to avoid padding
  * @name COMPUTE_M0_START_ROW
  * If there're any partial blocks in y dimension, they are placed at the beginning of the rows.
  * This shift amount is added to all rows such that the partial block (at the beginning) overlaps with the subsequent
  * blocks in the y dimension to avoid any padding.
  * EG: M0=4, PARTIAL_STORE_M0=1:
  *                  | Non-overlapping | +M0_ROW_SHIFT (Overlapping)
  * block 0 (partial)| start row = 0   | start row = 0
  * block 1 (full)   | start row = 4   | start row = 1
  * block 2 (full)   | start row = 8   | start row = 5
  *
  * @param[in] y                Global id of current block in y.
  * @param[in] M0               The number of rows to store, for non-partial blocks. Supported: 1-16
  * @param[in] PARTIAL_STORE_M0 The partial size in y, for partial blocks. Supported: [0, @p M0)
  * @{
  */
 #define COMPUTE_M0_START_ROW(y, M0, PARTIAL_STORE_M0) \
     ((uint)(max(0, (int)(y * M0) - (int)((M0 - PARTIAL_STORE_M0) % M0))))
 #else // defined(PARTIAL_STORE_M0)
 #define COMPUTE_M0_START_ROW(y, M0, PARTIAL_STORE_M0) \
     ((uint)(y * M0))
 #endif    // defined(PARTIAL_STORE_M0)
 /** @} */ // end of group COMPUTE_M0_START_ROW

 /** Store a vector that can only be partial in x.
  *
  * @note in case @p vec_size or @p leftover != 1, 2, 3, 4, 8, 16, extra vstore(s) will be invoked, thus incurring small performance penalty.
  *
  * The data to store is expected to end in a 0.
  * E.g., for basename=c, the expected name is c0.
  *
  * @param[in] basename  The name of the variable without trailing 0
  * @param[in] data_type The data type of the vector
  * @param[in] ptr       The base pointer
  * @param[in] vec_size  The vector size if cond = false. Supported: 1, 2, 3, 4, 8, 16
  * @param[in] leftover  The vector size if cond = true. Supported range: [1, @p vec_size0)
  * @param[in] cond      Condition to select either vec_size0 or vec_size1
  * @{
  */
 #define STORE_VECTOR_SELECT(basename, data_type, ptr, vec_size, leftover, cond) \
     STORE_BLOCK_PARTIAL_IN_X(1, vec_size, data_type, basename, ptr, 0, 0, leftover, cond)
 /** @} */ // end of group STORE_VECTOR_SELECT

 #if defined(ARM_COMPUTE_OPENCL_FP16_ENABLED) && defined(cl_khr_fp16)
 #pragma OPENCL EXTENSION cl_khr_fp16 : enable
 #endif // defined(ARM_COMPUTE_OPENCL_FP16_ENABLED) && defined(cl_khr_fp16)

 #if defined(ARM_COMPUTE_OPENCL_DOT8_ENABLED) && defined(cl_arm_integer_dot_product_int8)
 #pragma OPENCL EXTENSION cl_arm_integer_dot_product_int8 : enable
 #endif // defined(ARM_COMPUTE_OPENCL_DOT8_ENABLED) && defined(cl_arm_integer_dot_product_int8)

 #if defined(ARM_COMPUTE_OPENCL_DOT8_ACC_ENABLED) && defined(cl_arm_integer_dot_product_accumulate_int8)
 #pragma OPENCL EXTENSION cl_arm_integer_dot_product_accumulate_int8 : enable
 #endif // defined(ARM_COMPUTE_OPENCL_DOT8_ACC_ENABLED) && defined(cl_arm_integer_dot_product_accumulate_int8)

 #if defined(ARM_COMPUTE_DEBUG_ENABLED) && defined(cl_arm_printf)
 #pragma OPENCL EXTENSION cl_arm_printf : enable
 #endif // defined(ARM_COMPUTE_DEBUG_ENABLED) && defined(cl_arm_printf)

 #define GPU_ARCH_MIDGARD 0x100
 #define GPU_ARCH_BIFROST 0x200

 /** Concatenate two inputs.
  *
  * @param[in] a The first input to be concatenated
  * @param[in] b The second input to be concatenated
  *
  * @return The concatenated output
  */
 #define CONCAT(a, b) a##b

 /** Expand the given vector
  *
  * @param[in] x The vector to be expanded
  *
  * @return The expanded output
  */
 #define EXPAND(x) x

 /** Clamp the given value between an upper and lower bound.
  *
  * @param[in] x       The value to be clamped
  * @param[in] min_val The lower bound
  * @param[in] max_val The upper bound
  *
  * @return The clamped value.
  */
 #define CLAMP(x, min_val, max_val) min(max(x, min_val), max_val)

 /** REVn reverses the given vector whose size is n.
  * @name REVn
  *
  * @param[in] x The vector to be reversed
  *
  * @return The reversed vector
  * @{
  */
 #define REV1(x) ((x))
 #define REV2(x) ((x).s10)
 #define REV3(x) ((x).s210)
 #define REV4(x) ((x).s3210)
 #define REV8(x) ((x).s76543210)
 #define REV16(x) ((x).sFEDCBA9876543210)
 /** @} */ // end of group REVn

 /** Reverse the given vector.
  * @name REVERSE
  *
  * @param[in] x The vector to be reversed
  * @param[in] s The size of the vector
  *
  * @return The reversed vector
  * @{
  */
 #define REVERSE_STR(x, s) REV##s((x))
 #define REVERSE(x, s) REVERSE_STR(x, s)
 /** @} */ // end of group REVERSE

 /** Circular-right-shift (rotate-right) the vector of size s by the amount of n.
  * @name ROTs_n
  *
  * @param[in] x The vector to be shifted
  *
  * @return The shifted vector
  * @{
  */
 #define ROT1_0(x) ((x))

 #define ROT2_0(x) ((x))
 #define ROT2_1(x) ((x).s10)

 #define ROT3_0(x) ((x))
 #define ROT3_1(x) ((x).s201)
 #define ROT3_2(x) ((x).s120)

 #define ROT4_0(x) ((x))
 #define ROT4_1(x) ((x).s3012)
 #define ROT4_2(x) ((x).s2301)
 #define ROT4_3(x) ((x).s1230)

 #define ROT8_0(x) ((x))
 #define ROT8_1(x) ((x).s70123456)
 #define ROT8_2(x) ((x).s67012345)
 #define ROT8_3(x) ((x).s56701234)
 #define ROT8_4(x) ((x).s45670123)
 #define ROT8_5(x) ((x).s34567012)
 #define ROT8_6(x) ((x).s23456701)
 #define ROT8_7(x) ((x).s12345670)

 #define ROT16_0(x) ((x))
 #define ROT16_1(x) ((x).sF0123456789ABCDE)
 #define ROT16_2(x) ((x).sEF0123456789ABCD)
 #define ROT16_3(x) ((x).sDEF0123456789ABC)
 #define ROT16_4(x) ((x).sCDEF0123456789AB)
 #define ROT16_5(x) ((x).sBCDEF0123456789A)
 #define ROT16_6(x) ((x).sABCDEF0123456789)
 #define ROT16_7(x) ((x).s9ABCDEF012345678)
 #define ROT16_8(x) ((x).s89ABCDEF01234567)
 #define ROT16_9(x) ((x).s789ABCDEF0123456)
 #define ROT16_10(x) ((x).s6789ABCDEF012345)
 #define ROT16_11(x) ((x).s56789ABCDEF01234)
 #define ROT16_12(x) ((x).s456789ABCDEF0123)
 #define ROT16_13(x) ((x).s3456789ABCDEF012)
 #define ROT16_14(x) ((x).s23456789ABCDEF01)
 #define ROT16_15(x) ((x).s123456789ABCDEF0)
 /** @} */ // end of group ROTs_n

 /** Circular-right-shift (rotate-right) the given vector by the given amount.
  * @name ROTATE
  *
  * @param[in] x The vector to be shifted
  * @param[in] s The size of the vector
  * @param[in] n The amount to be shifted
  *
  * @return The shifted vector
  * @{
  */
 #define ROTATE_STR(x, s, n) ROT##s##_##n(x)
 #define ROTATE(x, s, n) ROTATE_STR(x, s, n)
 /** @} */ // end of group ROTATE

 /** Creates a vector of size n filled with offset values corresponding to the location of each element.
  * @name V_OFFSn
  *
  * @param[in] dt The data type of the output vector
  *
  * @return The vector filled with offset values
  * @{
  */
 #define V_OFFS1(dt) (dt##1)(0)
 #define V_OFFS2(dt) (dt##2)(0, 1)
 #define V_OFFS3(dt) (dt##3)(0, 1, 2)
 #define V_OFFS4(dt) (dt##4)(0, 1, 2, 3)
 #define V_OFFS8(dt) (dt##8)(0, 1, 2, 3, 4, 5, 6, 7)
 #define V_OFFS16(dt) (dt##16)(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)
 /** @} */ // end of group V_OFFSn

 /** Create a vector filled with offset values corresponding to the location of each element.
  * @name VEC_OFFS
  *
  * @param[in] dt The data type of the output vector
  * @param[in] s  The size of the output vector
  *
  * @return The vector filled with offset values
  * @{
  */
 #define VEC_OFFS_STR(dt, s) V_OFFS##s(dt)
 #define VEC_OFFS(dt, s) VEC_OFFS_STR(dt, s)
 /** @} */ // end of group VEC_OFFS

 #define VLOAD_STR(size) vload##size
 #define VLOAD(size) VLOAD_STR(size)

 #define PIXEL_UNIT4 1
 #define PIXEL_UNIT8 2
 #define PIXEL_UNIT16 4

 /** Utility macro to convert a vector size in pixel unit.
  *
  * @name CONVERT_VECTOR_SIZE_TO_PIXEL_UNIT
  *
  * @param[in] vec_size Vector size. Only 4,8 and 16 is supported
  *
  * @return The pixel unit (number of pixels)
  * @{
  */
 #define CONVERT_VECTOR_SIZE_TO_PIXEL_UNIT_STR(vec_size) PIXEL_UNIT##vec_size
 #define CONVERT_VECTOR_SIZE_TO_PIXEL_UNIT(vec_size) CONVERT_VECTOR_SIZE_TO_PIXEL_UNIT_STR(vec_size)
 /** @} */ // end of group CONVERT_VECTOR_SIZE_TO_PIXEL_UNIT

 #define read_image2d_floatx1(img, x_coord, y_coord) (float4)(read_imagef(img, (int2)(x_coord, y_coord)));
 #define read_image2d_floatx2(img, x_coord, y_coord) (float8)(read_imagef(img, (int2)(x_coord, y_coord)), read_imagef(img, (int2)(x_coord + 1, y_coord)));
 #define read_image2d_floatx4(img, x_coord, y_coord) (float16)(read_imagef(img, (int2)(x_coord, y_coord)), read_imagef(img, (int2)(x_coord + 1, y_coord)), read_imagef(img, (int2)(x_coord + 2, y_coord)), read_imagef(img, (int2)(x_coord + 3, y_coord)));

 #if defined(ARM_COMPUTE_OPENCL_FP16_ENABLED) && defined(cl_khr_fp16)
 #define read_image2d_halfx1(img, x_coord, y_coord) (half4)(read_imageh(img, (int2)(x_coord, y_coord)));
 #define read_image2d_halfx2(img, x_coord, y_coord) (half8)(read_imageh(img, (int2)(x_coord, y_coord)), read_imageh(img, (int2)(x_coord + 1, y_coord)));
 #define read_image2d_halfx4(img, x_coord, y_coord) (half16)(read_imageh(img, (int2)(x_coord, y_coord)), read_imageh(img, (int2)(x_coord + 1, y_coord)), read_imageh(img, (int2)(x_coord + 2, y_coord)), read_imageh(img, (int2)(x_coord + 3, y_coord)));
 #endif // defined(ARM_COMPUTE_OPENCL_FP16_ENABLED) && defined(cl_khr_fp16)

 /** Utility macro to read a 2D OpenCL image object.
  *
  * @note Coordinates are not normalized
  *
  * @param[in] data_type Data type
  * @param[in] n0        Number of pixel to read. Only 1,2 and 4 is supported
  * @param[in] img       OpenCL image object
  * @param[in] x_coord   The x coordinate for the top-left pixel
  * @param[in] y_coord   The y coordinate for the top-left pixel
  *
  * @return Pixels from the 2D OpenCL image object
  * @{
  */
 #define READ_IMAGE2D_STR(data_type, n0, img, x_coord, y_coord) read_image2d_##data_type##x##n0(img, x_coord, y_coord)
 #define READ_IMAGE2D(data_type, n0, img, x_coord, y_coord) READ_IMAGE2D_STR(data_type, n0, img, x_coord, y_coord)

 #define VSTORE_STR(size) vstore##size
 #define VSTORE(size) VSTORE_STR(size)

 #define float1 float
 #define half1 half
 #define char1 char
 #define uchar1 uchar
 #define short1 short
 #define ushort1 ushort
 #define int1 int
 #define uint1 uint
 #define long1 long
 #define ulong1 ulong
 #define double1 double

 #define vload1(OFFSET, PTR) *(OFFSET + PTR)
 #define vstore1(DATA, OFFSET, PTR) *(OFFSET + PTR) = DATA

 /** Extended partial vstore that correctly handles scalar values as well.
  * Store the **lower** 0 to (n-1)th elements of the given vector while minimising the amount of vstore ops
  * @name VSTORE_PARTIAL
  *
  * @note With this macro, the passed data can be both a vector and a scalar
  * @note @p store_size needs to be <= @p size
  * eg 1: Valid
  * VSTORE_PARTIAL(16, 15) ...;
  * eg 2: Invalid
  * VSTORE_PARTIAL(4, 7) ...;
  *
  * @param[in] size       The width of @p DATA. Supported values: 1(scalar), 2, 3, 4, 8, 16
  * @param[in] store_size The number of lower elements to store. Supported values: 1-16, but has to be <= @p size
  * @{
  */
 #define VSTORE_PARTIAL_STR(size, store_size) vstore_partial_##size##_##store_size
 #define VSTORE_PARTIAL(size, store_size) VSTORE_PARTIAL_STR(size, store_size)

 #define NO_STORE(data, offs, ptr) \
     {                             \
     }

 // Size == 1 (scalar)
 #define vstore_partial_1_0 NO_STORE
 #define vstore_partial_1_1 vstore1
 #define vstore_partial_1_2 NO_STORE
 #define vstore_partial_1_3 NO_STORE
 #define vstore_partial_1_4 NO_STORE
 #define vstore_partial_1_5 NO_STORE
 #define vstore_partial_1_6 NO_STORE
 #define vstore_partial_1_7 NO_STORE
 #define vstore_partial_1_8 NO_STORE
 #define vstore_partial_1_9 NO_STORE
 #define vstore_partial_1_10 NO_STORE
 #define vstore_partial_1_11 NO_STORE
 #define vstore_partial_1_12 NO_STORE
 #define vstore_partial_1_13 NO_STORE
 #define vstore_partial_1_14 NO_STORE
 #define vstore_partial_1_15 NO_STORE
 #define vstore_partial_1_16 NO_STORE
 // Size == 2
 #define vstore_partial_2_0 NO_STORE
 #define vstore_partial_2_1 vstore_partial_1
 #define vstore_partial_2_2 vstore_partial_2
 #define vstore_partial_2_3 NO_STORE
 #define vstore_partial_2_4 NO_STORE
 #define vstore_partial_2_5 NO_STORE
 #define vstore_partial_2_6 NO_STORE
 #define vstore_partial_2_7 NO_STORE
 #define vstore_partial_2_8 NO_STORE
 #define vstore_partial_2_9 NO_STORE
 #define vstore_partial_2_10 NO_STORE
 #define vstore_partial_2_11 NO_STORE
 #define vstore_partial_2_12 NO_STORE
 #define vstore_partial_2_13 NO_STORE
 #define vstore_partial_2_14 NO_STORE
 #define vstore_partial_2_15 NO_STORE
 #define vstore_partial_2_16 NO_STORE
 // Size == 3
 #define vstore_partial_3_0 NO_STORE
 #define vstore_partial_3_1 vstore_partial_1
 #define vstore_partial_3_2 vstore_partial_2
 #define vstore_partial_3_3 vstore_partial_3
 #define vstore_partial_3_4 NO_STORE
 #define vstore_partial_3_5 NO_STORE
 #define vstore_partial_3_6 NO_STORE
 #define vstore_partial_3_7 NO_STORE
 #define vstore_partial_3_8 NO_STORE
 #define vstore_partial_3_9 NO_STORE
 #define vstore_partial_3_10 NO_STORE
 #define vstore_partial_3_11 NO_STORE
 #define vstore_partial_3_12 NO_STORE
 #define vstore_partial_3_13 NO_STORE
 #define vstore_partial_3_14 NO_STORE
 #define vstore_partial_3_15 NO_STORE
 #define vstore_partial_3_16 NO_STORE
 // Size == 4
 #define vstore_partial_4_0 NO_STORE
 #define vstore_partial_4_1 vstore_partial_1
 #define vstore_partial_4_2 vstore_partial_2
 #define vstore_partial_4_3 vstore_partial_3
 #define vstore_partial_4_4 vstore_partial_4
 #define vstore_partial_4_5 NO_STORE
 #define vstore_partial_4_6 NO_STORE
 #define vstore_partial_4_7 NO_STORE
 #define vstore_partial_4_8 NO_STORE
 #define vstore_partial_4_9 NO_STORE
 #define vstore_partial_4_10 NO_STORE
 #define vstore_partial_4_11 NO_STORE
 #define vstore_partial_4_12 NO_STORE
 #define vstore_partial_4_13 NO_STORE
 #define vstore_partial_4_14 NO_STORE
 #define vstore_partial_4_15 NO_STORE
 #define vstore_partial_4_16 NO_STORE
 // Size == 8
 #define vstore_partial_8_0 NO_STORE
 #define vstore_partial_8_1 vstore_partial_1
 #define vstore_partial_8_2 vstore_partial_2
 #define vstore_partial_8_3 vstore_partial_3
 #define vstore_partial_8_4 vstore_partial_4
 #define vstore_partial_8_5 vstore_partial_5
 #define vstore_partial_8_6 vstore_partial_6
 #define vstore_partial_8_7 vstore_partial_7
 #define vstore_partial_8_8 vstore_partial_8
 #define vstore_partial_8_9 NO_STORE
 #define vstore_partial_8_10 NO_STORE
 #define vstore_partial_8_11 NO_STORE
 #define vstore_partial_8_12 NO_STORE
 #define vstore_partial_8_13 NO_STORE
 #define vstore_partial_8_14 NO_STORE
 #define vstore_partial_8_15 NO_STORE
 #define vstore_partial_8_16 NO_STORE
 // Size == 16
 #define vstore_partial_16_0 NO_STORE
 #define vstore_partial_16_1 vstore_partial_1
 #define vstore_partial_16_2 vstore_partial_2
 #define vstore_partial_16_3 vstore_partial_3
 #define vstore_partial_16_4 vstore_partial_4
 #define vstore_partial_16_5 vstore_partial_5
 #define vstore_partial_16_6 vstore_partial_6
 #define vstore_partial_16_7 vstore_partial_7
 #define vstore_partial_16_8 vstore_partial_8
 #define vstore_partial_16_9 vstore_partial_9
 #define vstore_partial_16_10 vstore_partial_10
 #define vstore_partial_16_11 vstore_partial_11
 #define vstore_partial_16_12 vstore_partial_12
 #define vstore_partial_16_13 vstore_partial_13
 #define vstore_partial_16_14 vstore_partial_14
 #define vstore_partial_16_15 vstore_partial_15
 #define vstore_partial_16_16 vstore_partial_16

 /** Partial vstore. Store the **lower** 0 to (n-1)th elements of the given vector while minimising the amount of vstore ops
  * @name vstore_partial_n
  *
  * @note @p DATA needs to be a vector not a scalar
  * @note n needs to be <= the vector width of the input variable @p DATA
  * eg 1: Valid
  * vstore_partial_15(var:float16, 0, 0xabcd);
  * eg 2: Invalid
  * vstore_partial_7(var:float4, 0, 0xabcd);
  *
  * @note in cases n == 1, 2, 3, 4, 8, 16, no extra vstore is invoked, thus there's no performance penalty.
  *
  * @param[in] DATA   The name of the variable
  * @param[in] OFFSET Offset in n
  * @param[in] PTR    The base pointer
  * @{
  */
 #define vstore_partial_1(DATA, OFFSET, PTR) \
     vstore1(DATA.s0, OFFSET, PTR);

 #define vstore_partial_2(DATA, OFFSET, PTR) \
     vstore2(DATA.s01, OFFSET, PTR);

 #define vstore_partial_3(DATA, OFFSET, PTR) \
     vstore3(DATA.s012, OFFSET, PTR);

 #define vstore_partial_4(DATA, OFFSET, PTR) \
     vstore4(DATA.s0123, OFFSET, PTR);

 #define vstore_partial_5(DATA, OFFSET, PTR)    \
     vstore_partial_4(DATA.s0123, OFFSET, PTR); \
     vstore1(DATA.s4, OFFSET, PTR + 4);

 #define vstore_partial_6(DATA, OFFSET, PTR)    \
     vstore_partial_4(DATA.s0123, OFFSET, PTR); \
     vstore_partial_2(DATA.s45, OFFSET, PTR + 4);

 #define vstore_partial_7(DATA, OFFSET, PTR)    \
     vstore_partial_4(DATA.s0123, OFFSET, PTR); \
     vstore_partial_3(DATA.s456, OFFSET, PTR + 4);

 #define vstore_partial_8(DATA, OFFSET, PTR) \
     vstore8(DATA.s01234567, OFFSET, PTR);

 #define vstore_partial_9(DATA, OFFSET, PTR)        \
     vstore_partial_8(DATA.s01234567, OFFSET, PTR); \
     vstore1(DATA.s8, OFFSET, PTR + 8);

 #define vstore_partial_10(DATA, OFFSET, PTR)       \
     vstore_partial_8(DATA.s01234567, OFFSET, PTR); \
     vstore_partial_2(DATA.s89, OFFSET, PTR + 8);

 #define vstore_partial_11(DATA, OFFSET, PTR)       \
     vstore_partial_8(DATA.s01234567, OFFSET, PTR); \
     vstore_partial_3(DATA.s89a, OFFSET, PTR + 8);

 #define vstore_partial_12(DATA, OFFSET, PTR)       \
     vstore_partial_8(DATA.s01234567, OFFSET, PTR); \
     vstore_partial_4(DATA.s89ab, OFFSET, PTR + 8);

 #define vstore_partial_13(DATA, OFFSET, PTR)       \
     vstore_partial_8(DATA.s01234567, OFFSET, PTR); \
     vstore_partial_5(DATA.s89abcdef, OFFSET, PTR + 8);

 #define vstore_partial_14(DATA, OFFSET, PTR)       \
     vstore_partial_8(DATA.s01234567, OFFSET, PTR); \
     vstore_partial_6(DATA.s89abcdef, OFFSET, PTR + 8);

 #define vstore_partial_15(DATA, OFFSET, PTR)       \
     vstore_partial_8(DATA.s01234567, OFFSET, PTR); \
     vstore_partial_7(DATA.s89abcdef, OFFSET, PTR + 8);

 #define vstore_partial_16(DATA, OFFSET, PTR) \
     vstore16(DATA, OFFSET, PTR);
 /** @} */ // end of groupd vstore_partial_n
 /** @} */ // end of groupd VSTORE_PARTIAL

 // Convert built-in functions with _sat modifier are not supported in floating point so we create defines
 // without _sat to overcome this issue
 #define convert_float_sat convert_float
 #define convert_float1_sat convert_float
 #define convert_float2_sat convert_float2
 #define convert_float3_sat convert_float3
 #define convert_float4_sat convert_float4
 #define convert_float8_sat convert_float8
 #define convert_float16_sat convert_float16
 #define convert_half_sat convert_float
 #define convert_half1_sat convert_half
 #define convert_half2_sat convert_half2
 #define convert_half3_sat convert_half3
 #define convert_half4_sat convert_half4
 #define convert_half8_sat convert_half8
 #define convert_half16_sat convert_half16

 #define convert_float1 convert_float
 #define convert_half1 convert_half
 #define convert_char1 convert_char
 #define convert_uchar1 convert_uchar
 #define convert_short1 convert_short
 #define convert_ushort1 convert_ushort
 #define convert_int1 convert_int
 #define convert_uint1 convert_uint
 #define convert_long1 convert_long
 #define convert_ulong1 convert_ulong
 #define convert_double1 convert_double

 #define convert_char1_sat convert_char_sat
 #define convert_uchar1_sat convert_uchar_sat
 #define convert_short1_sat convert_short_sat
 #define convert_ushort1_sat convert_ushort_sat
 #define convert_int1_sat convert_int_sat
 #define convert_uint1_sat convert_uint_sat
 #define convert_long1_sat convert_long_sat
 #define convert_ulong1_sat convert_ulong_sat
 #define convert_double1_sat convert_double_sat

 #define VEC_DATA_TYPE_STR(type, size) type##size
 #define VEC_DATA_TYPE(type, size) VEC_DATA_TYPE_STR(type, size)

 #define CONVERT_STR(x, type) (convert_##type((x)))
 #define CONVERT(x, type) CONVERT_STR(x, type)

 #define CONVERT_SAT_STR(x, type) (convert_##type##_sat((x)))
 #define CONVERT_SAT(x, type) CONVERT_SAT_STR(x, type)

 #define CONVERT_SAT_ROUND_STR(x, type, round) (convert_##type##_sat_##round((x)))
 #define CONVERT_SAT_ROUND(x, type, round) CONVERT_SAT_ROUND_STR(x, type, round)

 #define select_vec_dt_uchar(size) uchar##size
 #define select_vec_dt_char(size) char##size
 #define select_vec_dt_ushort(size) ushort##size
 #define select_vec_dt_short(size) short##size
 #define select_vec_dt_half(size) short##size
 #define select_vec_dt_uint(size) uint##size
 #define select_vec_dt_int(size) int##size
 #define select_vec_dt_float(size) int##size
 #define select_vec_dt_ulong(size) ulong##size
 #define select_vec_dt_long(size) long##size

 #define SELECT_VEC_DATA_TYPE_STR(type, size) select_vec_dt_##type(size)
 #define SELECT_VEC_DATA_TYPE(type, size) SELECT_VEC_DATA_TYPE_STR(type, size)
 #define SELECT_DATA_TYPE(type) SELECT_VEC_DATA_TYPE_STR(type, 1)

 #define sum_reduce_1(x) (x)
 #define sum_reduce_2(x) ((x).s0) + ((x).s1)
 #define sum_reduce_3(x) sum_reduce_2((x).s01) + ((x).s2)
 #define sum_reduce_4(x) sum_reduce_2((x).s01) + sum_reduce_2((x).s23)
 #define sum_reduce_8(x) sum_reduce_4((x).s0123) + sum_reduce_4((x).s4567)
 #define sum_reduce_16(x) sum_reduce_8((x).s01234567) + sum_reduce_8((x).s89ABCDEF)

 #define SUM_REDUCE_STR(x, size) sum_reduce_##size(x)
 #define SUM_REDUCE(x, size) SUM_REDUCE_STR(x, size)

 #define max_reduce_1(x) (x)
 #define max_reduce_2(x) max(((x).s0), ((x).s1))
 #define max_reduce_3(x) max(max_reduce_2((x).s01), ((x).s2))
 #define max_reduce_4(x) max(max_reduce_2((x).s01), max_reduce_2((x).s23))
 #define max_reduce_8(x) max(max_reduce_4((x).s0123), max_reduce_4((x).s4567))
 #define max_reduce_16(x) max(max_reduce_8((x).s01234567), max_reduce_8((x).s89ABCDEF))

 #define MAX_REDUCE_STR(x, size) max_reduce_##size(x)
 #define MAX_REDUCE(x, size) MAX_REDUCE_STR(x, size)

 #define VECTOR_DECLARATION(name)     \
     __global uchar *name##_ptr,      \
     uint        name##_stride_x, \
     uint        name##_step_x,   \
     uint        name##_offset_first_element_in_bytes

 #define IMAGE_DECLARATION(name)      \
     __global uchar *name##_ptr,      \
     uint        name##_stride_x, \
     uint        name##_step_x,   \
     uint        name##_stride_y, \
     uint        name##_step_y,   \
     uint        name##_offset_first_element_in_bytes

 #define TENSOR3D_DECLARATION(name)   \
     __global uchar *name##_ptr,      \
     uint        name##_stride_x, \
     uint        name##_step_x,   \
     uint        name##_stride_y, \
     uint        name##_step_y,   \
     uint        name##_stride_z, \
     uint        name##_step_z,   \
     uint        name##_offset_first_element_in_bytes

 #define TENSOR4D_DECLARATION(name)   \
     __global uchar *name##_ptr,      \
     uint        name##_stride_x, \
     uint        name##_step_x,   \
     uint        name##_stride_y, \
     uint        name##_step_y,   \
     uint        name##_stride_z, \
     uint        name##_step_z,   \
     uint        name##_stride_w, \
     uint        name##_step_w,   \
     uint        name##_offset_first_element_in_bytes

 #define CONVERT_TO_VECTOR_STRUCT(name) \
     update_vector_workitem_ptr(name##_ptr, name##_offset_first_element_in_bytes, name##_stride_x, name##_step_x)

 #define CONVERT_TO_VECTOR_STRUCT_NO_STEP(name) \
     update_vector_workitem_ptr(name##_ptr, name##_offset_first_element_in_bytes, name##_stride_x, 0)

 #define CONVERT_TO_IMAGE_STRUCT(name) \
     update_image_workitem_ptr(name##_ptr, name##_offset_first_element_in_bytes, name##_stride_x, name##_step_x, name##_stride_y, name##_step_y)

 #define CONVERT_TO_IMAGE_STRUCT_NO_STEP(name) \
     update_image_workitem_ptr(name##_ptr, name##_offset_first_element_in_bytes, name##_stride_x, 0, name##_stride_y, 0)

 #define CONVERT_TENSOR3D_TO_IMAGE_STRUCT(name) \
     update_image_from_tensor3D_workitem_ptr(name##_ptr, name##_offset_first_element_in_bytes, name##_stride_x, name##_step_x, name##_stride_y, name##_step_y, name##_stride_z, name##_step_z)

 #define CONVERT_TENSOR3D_TO_IMAGE_STRUCT_NO_STEP(name) \
     update_image_from_tensor3D_workitem_ptr(name##_ptr, name##_offset_first_element_in_bytes, name##_stride_x, 0, name##_stride_y, 0, name##_stride_z, name##_step_z)

 #define CONVERT_TENSOR3D_TO_IMAGE_STRUCT(name) \
     update_image_from_tensor3D_workitem_ptr(name##_ptr, name##_offset_first_element_in_bytes, name##_stride_x, name##_step_x, name##_stride_y, name##_step_y, name##_stride_z, name##_step_z)

 #define CONVERT_TO_TENSOR3D_STRUCT(name)                                                                                                           \
     update_tensor3D_workitem_ptr(name##_ptr, name##_offset_first_element_in_bytes, name##_stride_x, name##_step_x, name##_stride_y, name##_step_y, \
                                  name##_stride_z, name##_step_z)

 #define CONVERT_TO_TENSOR3D_STRUCT_NO_STEP(name) \
     update_tensor3D_workitem_ptr(name##_ptr, name##_offset_first_element_in_bytes, name##_stride_x, 0, name##_stride_y, 0, name##_stride_z, 0)

 #define CONVERT_TO_TENSOR4D_STRUCT(name, mod_size)                                                                                                 \
     update_tensor4D_workitem_ptr(name##_ptr, name##_offset_first_element_in_bytes, name##_stride_x, name##_step_x, name##_stride_y, name##_step_y, \
                                  name##_stride_z, name##_step_z, name##_stride_w, name##_step_w, mod_size)

 #define CONVERT_TO_TENSOR4D_STRUCT_NO_STEP(name, mod_size) \
     update_tensor4D_workitem_ptr(name##_ptr, name##_offset_first_element_in_bytes, name##_stride_x, 0, name##_stride_y, 0, name##_stride_z, 0, name##_stride_w, 0, mod_size)

 #define CONVERT_TO_TENSOR3D_STRUCT_NO_UPDATE_PTR(name)                                                                                       \
     tensor3D_ptr_no_update(name##_ptr, name##_offset_first_element_in_bytes, name##_stride_x, name##_step_x, name##_stride_y, name##_step_y, \
                            name##_stride_z, name##_step_z)

 /** Structure to hold Vector information */
 typedef struct Vector
 {
     __global uchar *ptr;                           /**< Pointer to the starting postion of the buffer */
     int             offset_first_element_in_bytes; /**< The offset of the first element in the source image */
     int             stride_x;                      /**< Stride of the image in X dimension (in bytes) */
 } Vector;

 /** Structure to hold Image information */
 typedef struct Image
 {
     __global uchar *ptr;                           /**< Pointer to the starting postion of the buffer */
     int             offset_first_element_in_bytes; /**< The offset of the first element in the source image */
     int             stride_x;                      /**< Stride of the image in X dimension (in bytes) */
     int             stride_y;                      /**< Stride of the image in Y dimension (in bytes) */
 } Image;

 /** Structure to hold 3D tensor information */
 typedef struct Tensor3D
 {
     __global uchar *ptr;                           /**< Pointer to the starting postion of the buffer */
     int             offset_first_element_in_bytes; /**< The offset of the first element in the source image */
     int             stride_x;                      /**< Stride of the image in X dimension (in bytes) */
     int             stride_y;                      /**< Stride of the image in Y dimension (in bytes) */
     int             stride_z;                      /**< Stride of the image in Z dimension (in bytes) */
 } Tensor3D;

 /** Structure to hold 4D tensor information */
 typedef struct Tensor4D
 {
     __global uchar *ptr;                           /**< Pointer to the starting postion of the buffer */
     int             offset_first_element_in_bytes; /**< The offset of the first element in the source image */
     int             stride_x;                      /**< Stride of the image in X dimension (in bytes) */
     int             stride_y;                      /**< Stride of the image in Y dimension (in bytes) */
     int             stride_z;                      /**< Stride of the image in Z dimension (in bytes) */
     int             stride_w;                      /**< Stride of the image in W dimension (in bytes) */
 } Tensor4D;

 /** Wrap vector information into an Vector structure, and make the pointer point at this workitem's data.
  *
  * @param[in] ptr                           Pointer to the starting postion of the buffer
  * @param[in] offset_first_element_in_bytes The offset of the first element in the source vector
  * @param[in] stride_x                      Stride of the vector in X dimension (in bytes)
  * @param[in] step_x                        stride_x * number of elements along X processed per workitem(in bytes)
  *
  * @return An image object
  */
 inline Vector update_vector_workitem_ptr(__global uchar *ptr, uint offset_first_element_in_bytes, uint stride_x, uint step_x)
 {
     Vector vector =
     {
         .ptr                           = ptr,
         .offset_first_element_in_bytes = offset_first_element_in_bytes,
         .stride_x                      = stride_x,
     };
     vector.ptr += vector.offset_first_element_in_bytes + get_global_id(0) * step_x;
     return vector;
 }

 /** Wrap image information into an Image structure, and make the pointer point at this workitem's data.
  *
  * @param[in] ptr                           Pointer to the starting postion of the buffer
  * @param[in] offset_first_element_in_bytes The offset of the first element in the source image
  * @param[in] stride_x                      Stride of the image in X dimension (in bytes)
  * @param[in] step_x                        stride_x * number of elements along X processed per workitem(in bytes)
  * @param[in] stride_y                      Stride of the image in Y dimension (in bytes)
  * @param[in] step_y                        stride_y * number of elements along Y processed per workitem(in bytes)
  *
  * @return An image object
  */
 inline Image update_image_workitem_ptr(__global uchar *ptr, uint offset_first_element_in_bytes, uint stride_x, uint step_x, uint stride_y, uint step_y)
 {
     Image img =
     {
         .ptr                           = ptr,
         .offset_first_element_in_bytes = offset_first_element_in_bytes,
         .stride_x                      = stride_x,
         .stride_y                      = stride_y
     };
     img.ptr += img.offset_first_element_in_bytes + get_global_id(0) * step_x + get_global_id(1) * step_y;
     return img;
 }

 /** Wrap 3D tensor information into an image structure, and make the pointer point at this workitem's data.
  *
  * @param[in] ptr                           Pointer to the starting postion of the buffer
  * @param[in] offset_first_element_in_bytes The offset of the first element in the source image
  * @param[in] stride_x                      Stride of the image in X dimension (in bytes)
  * @param[in] step_x                        stride_x * number of elements along X processed per workitem(in bytes)
  * @param[in] stride_y                      Stride of the image in Y dimension (in bytes)
  * @param[in] step_y                        stride_y * number of elements along Y processed per workitem(in bytes)
  * @param[in] stride_z                      Stride of the image in Z dimension (in bytes)
  * @param[in] step_z                        stride_z * number of elements along Z processed per workitem(in bytes)
  *
  * @return A 3D tensor object
  */
 inline Image update_image_from_tensor3D_workitem_ptr(__global uchar *ptr, uint offset_first_element_in_bytes, uint stride_x, uint step_x, uint stride_y, uint step_y, uint stride_z, uint step_z)
 {
     Image img =
     {
         .ptr                           = ptr,
         .offset_first_element_in_bytes = offset_first_element_in_bytes,
         .stride_x                      = stride_x,
         .stride_y                      = stride_y
     };
     img.ptr += img.offset_first_element_in_bytes + get_global_id(0) * step_x + get_global_id(1) * step_y + get_global_id(2) * step_z;
     return img;
 }

 /** Wrap 3D tensor information into an tensor structure, and make the pointer point at this workitem's data.
  *
  * @param[in] ptr                           Pointer to the starting postion of the buffer
  * @param[in] offset_first_element_in_bytes The offset of the first element in the source image
  * @param[in] stride_x                      Stride of the image in X dimension (in bytes)
  * @param[in] step_x                        stride_x * number of elements along X processed per workitem(in bytes)
  * @param[in] stride_y                      Stride of the image in Y dimension (in bytes)
  * @param[in] step_y                        stride_y * number of elements along Y processed per workitem(in bytes)
  * @param[in] stride_z                      Stride of the image in Z dimension (in bytes)
  * @param[in] step_z                        stride_z * number of elements along Z processed per workitem(in bytes)
  *
  * @return A 3D tensor object
  */
 inline Tensor3D update_tensor3D_workitem_ptr(__global uchar *ptr, uint offset_first_element_in_bytes, uint stride_x, uint step_x, uint stride_y, uint step_y, uint stride_z, uint step_z)
 {
     Tensor3D tensor =
     {
         .ptr                           = ptr,
         .offset_first_element_in_bytes = offset_first_element_in_bytes,
         .stride_x                      = stride_x,
         .stride_y                      = stride_y,
         .stride_z                      = stride_z
     };
     tensor.ptr += tensor.offset_first_element_in_bytes + get_global_id(0) * step_x + get_global_id(1) * step_y + get_global_id(2) * step_z;
     return tensor;
 }

 /** Wrap 3D tensor information into an tensor structure.
  *
  * @param[in] ptr                           Pointer to the starting postion of the buffer
  * @param[in] offset_first_element_in_bytes The offset of the first element in the source image
  * @param[in] stride_x                      Stride of the image in X dimension (in bytes)
  * @param[in] step_x                        stride_x * number of elements along X processed per workitem(in bytes)
  * @param[in] stride_y                      Stride of the image in Y dimension (in bytes)
  * @param[in] step_y                        stride_y * number of elements along Y processed per workitem(in bytes)
  * @param[in] stride_z                      Stride of the image in Z dimension (in bytes)
  * @param[in] step_z                        stride_z * number of elements along Z processed per workitem(in bytes)
  *
  * @return A 3D tensor object
  */
 inline Tensor3D tensor3D_ptr_no_update(__global uchar *ptr, uint offset_first_element_in_bytes, uint stride_x, uint step_x, uint stride_y, uint step_y, uint stride_z, uint step_z)
 {
     Tensor3D tensor =
     {
         .ptr                           = ptr,
         .offset_first_element_in_bytes = offset_first_element_in_bytes,
         .stride_x                      = stride_x,
         .stride_y                      = stride_y,
         .stride_z                      = stride_z
     };
     return tensor;
 }

 inline Tensor4D update_tensor4D_workitem_ptr(__global uchar *ptr, uint offset_first_element_in_bytes, uint stride_x, uint step_x, uint stride_y, uint step_y, uint stride_z, uint step_z, uint stride_w,
                                              uint step_w,
                                              uint mod_size)
 {
     Tensor4D tensor =
     {
         .ptr                           = ptr,
         .offset_first_element_in_bytes = offset_first_element_in_bytes,
         .stride_x                      = stride_x,
         .stride_y                      = stride_y,
         .stride_z                      = stride_z,
         .stride_w                      = stride_w
     };

     tensor.ptr += tensor.offset_first_element_in_bytes + get_global_id(0) * step_x + get_global_id(1) * step_y + (get_global_id(2) % mod_size) * step_z + (get_global_id(2) / mod_size) * step_w;
     return tensor;
 }

 /** Get the pointer position of a Vector
  *
  * @param[in] vec Pointer to the starting position of the buffer
  * @param[in] x   Relative X position
  */
 inline __global const uchar *vector_offset(const Vector *vec, int x)
 {
     return vec->ptr + x * vec->stride_x;
 }

 /** Get the pointer position of a Image
  *
  * @param[in] img Pointer to the starting position of the buffer
  * @param[in] x   Relative X position
  * @param[in] y   Relative Y position
  */
 inline __global uchar *offset(const Image *img, int x, int y)
 {
     return img->ptr + x * img->stride_x + y * img->stride_y;
 }

 /** Get the pointer position of a Tensor3D
  *
  * @param[in] tensor Pointer to the starting position of the buffer
  * @param[in] x      Relative X position
  * @param[in] y      Relative Y position
  * @param[in] z      Relative Z position
  */
 inline __global const uchar *tensor3D_offset(const Tensor3D *tensor, int x, int y, int z)
 {
     return tensor->ptr + x * tensor->stride_x + y * tensor->stride_y + z * tensor->stride_z;
 }

 /** Get the pointer position of a Tensor4D
  *
  * @param[in] tensor Pointer to the starting position of the buffer
  * @param[in] x      Relative X position
  * @param[in] y      Relative Y position
  * @param[in] z      Relative Z position
  * @param[in] w      Relative W position
  */
 inline __global const uchar *tensor4D_offset(const Tensor4D *tensor, int x, int y, int z, int w)
 {
     return tensor->ptr + x * tensor->stride_x + y * tensor->stride_y + z * tensor->stride_z + w * tensor->stride_w;
 }

 /** Get the offset for a given linear index of a Tensor3D
  *
  * @param[in] tensor Pointer to the starting position of the buffer
  * @param[in] width  Width of the input tensor
  * @param[in] height Height of the input tensor
  * @param[in] depth  Depth of the input tensor
  * @param[in] index  Linear index
  */
 inline __global const uchar *tensor3D_index2ptr(const Tensor3D *tensor, uint width, uint height, uint depth, uint index)
 {
     uint num_elements = width * height;

     const uint z = index / num_elements;

     index %= num_elements;

     const uint y = index / width;

     index %= width;

     const uint x = index;

     return tensor->ptr + x * tensor->stride_x + y * tensor->stride_y + z * tensor->stride_z + tensor->offset_first_element_in_bytes;
 }

 #endif // _HELPER_H

 #if GPU_ARCH == GPU_ARCH_BIFROST
 #define MLA(a, b, c) (fma(c, b, a))
 #else // GPU_ARCH == GPU_ARCH_BIFROST
 #define MLA(a, b, c) ((b) * (c) + (a))
 #endif // GPU_ARCH == GPU_ARCH_BIFROST

 // Hard-Swish
 #define hard_swish_op(DATA_TYPE, VEC_SIZE, x, A_VAL, B_VAL) (x * ((min(max((x + (DATA_TYPE)3.0), (DATA_TYPE)0.0), (DATA_TYPE)6.0)) * (DATA_TYPE)0.166666667))

 // Logistic Activation
 #define logistic_op(DATA_TYPE, VEC_SIZE, x, A_VAL, B_VAL) ((DATA_TYPE)1.0 / ((DATA_TYPE)1.0 + exp(-x)))

 // Hyperbolic Tangent Activation
 #define tanh_op(DATA_TYPE, VEC_SIZE, x, A_VAL, B_VAL) ((DATA_TYPE)A_VAL * tanh((DATA_TYPE)B_VAL * x))

 // RELU Tangent Activation
 #define relu_op(DATA_TYPE, VEC_SIZE, x, A_VAL, B_VAL) (max((DATA_TYPE)0.0, x))

 // Bounded RELU Activation
 #define brelu_op(DATA_TYPE, VEC_SIZE, x, A_VAL, B_VAL) (min((DATA_TYPE)A_VAL, max((DATA_TYPE)0.0, x)))

 // Lower Upper Bounded RELU Activation
 #define lu_brelu_op(DATA_TYPE, VEC_SIZE, x, A_VAL, B_VAL) (min(max(x, (DATA_TYPE)B_VAL), (DATA_TYPE)A_VAL))

 // Leaky RELU Activation
 #define lrelu_op(DATA_TYPE, VEC_SIZE, x, A_VAL, B_VAL) ((min(x, (DATA_TYPE)0.0) * (DATA_TYPE)A_VAL) + max(x, (DATA_TYPE)0.0))

 // Soft RELU Activation
 #define srelu_op(DATA_TYPE, VEC_SIZE, x, A_VAL, B_VAL) (log((DATA_TYPE)1.0 + exp(x)))

 // ELU Activation
 #define elu_op(DATA_TYPE, VEC_SIZE, x, A_VAL, B_VAL) (select(((DATA_TYPE)A_VAL * (exp(x) - (DATA_TYPE)1.0)), x, (SELECT_VEC_DATA_TYPE(DATA_TYPE, VEC_SIZE))isgreaterequal(x, (DATA_TYPE)0.0)))

 // Absolute Activation
 #define abs_op(DATA_TYPE, VEC_SIZE, x, A_VAL, B_VAL) (fabs(x))

 // Square Activation
 #define square_op(DATA_TYPE, VEC_SIZE, x, A_VAL, B_VAL) (x * x)

 // Square-root Activation
 #define sqrt_op(DATA_TYPE, VEC_SIZE, x, A_VAL, B_VAL) (sqrt(x))

 // Linear Activation
 #define linear_op(DATA_TYPE, VEC_SIZE, x, A_VAL, B_VAL) (MLA((DATA_TYPE)B_VAL, (DATA_TYPE)A_VAL, x))

 // Identity Activation
 #define identity_op(DATA_TYPE, VEC_SIZE, x, A_VAL, B_VAL) (x)

 #define ACT_OP(op, DATA_TYPE, VEC_SIZE, x, A_VAL, B_VAL) op##_op(DATA_TYPE, VEC_SIZE, x, A_VAL, B_VAL)

 #define ACTIVATION(op, DATA_TYPE, VEC_SIZE, x, A_VAL, B_VAL) ACT_OP(op, DATA_TYPE, VEC_SIZE, x, A_VAL, B_VAL)

 /** Get the pointer position at a certain offset in x and y direction.
  *
  * @param[in] ptr      Pointer to the starting position of the buffer
  * @param[in] x        Relative X position
  * @param[in] y        Relative Y position
  * @param[in] stride_x Stride of the source tensor in X dimension (in bytes)
  * @param[in] stride_y Stride of the source tensor in Y dimension (in bytes)
  *
  * @return a uchar
  */
 inline __global uchar *ptr_offset(__global uchar *ptr, const int x, const int y, const int stride_x, const int stride_y)
 {
     return ptr + x * stride_x + y * stride_y;
 }

 #if(DILATION_X == 1 && DILATION_Y == 1)

 #define CONVOLUTION1x3_BIFROST2X1_STRIDE1(acc, src0, weights_row0) \
     ({                                                             \
         acc.s0 = fma(src0.s0, weights_row0.s0, acc.s0);            \
         acc.s0 = fma(src0.s1, weights_row0.s1, acc.s0);            \
         acc.s0 = fma(src0.s2, weights_row0.s2, acc.s0);            \
         acc.s1 = fma(src0.s1, weights_row0.s0, acc.s1);            \
         acc.s1 = fma(src0.s2, weights_row0.s1, acc.s1);            \
         acc.s1 = fma(src0.s3, weights_row0.s2, acc.s1);            \
     })

 #define CONVOLUTION1x3_BIFROST4X1_STRIDE1(acc, src0, weights_row0) \
     ({                                                             \
         acc.s0 = fma(src0.s0, weights_row0.s0, acc.s0);            \
         acc.s0 = fma(src0.s1, weights_row0.s1, acc.s0);            \
         acc.s0 = fma(src0.s2, weights_row0.s2, acc.s0);            \
         acc.s1 = fma(src0.s1, weights_row0.s0, acc.s1);            \
         acc.s1 = fma(src0.s2, weights_row0.s1, acc.s1);            \
         acc.s1 = fma(src0.s3, weights_row0.s2, acc.s1);            \
         acc.s2 = fma(src0.s2, weights_row0.s0, acc.s2);            \
         acc.s2 = fma(src0.s3, weights_row0.s1, acc.s2);            \
         acc.s2 = fma(src0.s4, weights_row0.s2, acc.s2);            \
         acc.s3 = fma(src0.s3, weights_row0.s0, acc.s3);            \
         acc.s3 = fma(src0.s4, weights_row0.s1, acc.s3);            \
         acc.s3 = fma(src0.s5, weights_row0.s2, acc.s3);            \
     })

 #define CONVOLUTION1x3_BIFROST2X1_STRIDE2(acc, src0, src1, weights_row0) \
     ({                                                                   \
         acc.s0 = fma(src0.s0, weights_row0.s0, acc.s0);                  \
         acc.s0 = fma(src0.s1, weights_row0.s1, acc.s0);                  \
         acc.s0 = fma(src0.s2, weights_row0.s2, acc.s0);                  \
         acc.s1 = fma(src0.s2, weights_row0.s0, acc.s1);                  \
         acc.s1 = fma(src0.s3, weights_row0.s1, acc.s1);                  \
         acc.s1 = fma(src1.s0, weights_row0.s2, acc.s1);                  \
     })

 #define CONVOLUTION1x3_BIFROST4X1_STRIDE2(acc, src0, src1, weights_row0) \
     ({                                                                   \
         acc.s0 = fma(src0.s0, weights_row0.s0, acc.s0);                  \
         acc.s0 = fma(src0.s1, weights_row0.s1, acc.s0);                  \
         acc.s0 = fma(src0.s2, weights_row0.s2, acc.s0);                  \
         acc.s1 = fma(src0.s2, weights_row0.s0, acc.s1);                  \
         acc.s1 = fma(src0.s3, weights_row0.s1, acc.s1);                  \
         acc.s1 = fma(src0.s4, weights_row0.s2, acc.s1);                  \
         acc.s2 = fma(src0.s4, weights_row0.s0, acc.s2);                  \
         acc.s2 = fma(src0.s5, weights_row0.s1, acc.s2);                  \
         acc.s2 = fma(src0.s6, weights_row0.s2, acc.s2);                  \
         acc.s3 = fma(src0.s6, weights_row0.s0, acc.s3);                  \
         acc.s3 = fma(src0.s7, weights_row0.s1, acc.s3);                  \
         acc.s3 = fma(src1.s0, weights_row0.s2, acc.s3);                  \
     })

 #else /* DILATION_X==1 && DILATION_Y==1 */

 #define CONVOLUTION1x3_BIFROST2X1_STRIDE1(acc, src0_left, src0_mid, src0_right, weights_row0) \
     ({                                                                                        \
         acc.s0 = fma(src0_left.s0, weights_row0.s0, acc.s0);                                  \
         acc.s0 = fma(src0_mid.s0, weights_row0.s1, acc.s0);                                   \
         acc.s0 = fma(src0_right.s0, weights_row0.s2, acc.s0);                                 \
         acc.s1 = fma(src0_left.s1, weights_row0.s0, acc.s1);                                  \
         acc.s1 = fma(src0_mid.s1, weights_row0.s1, acc.s1);                                   \
         acc.s1 = fma(src0_right.s1, weights_row0.s2, acc.s1);                                 \
     })

 #define CONVOLUTION1x3_BIFROST2X1_STRIDE2(acc, src0_left, src0_mid, src0_right, weights_row0) \
     ({                                                                                        \
         acc.s0 = fma(src0_left.s0, weights_row0.s0, acc.s0);                                  \
         acc.s0 = fma(src0_mid.s0, weights_row0.s1, acc.s0);                                   \
         acc.s0 = fma(src0_right.s0, weights_row0.s2, acc.s0);                                 \
         acc.s1 = fma(src0_left.s2, weights_row0.s0, acc.s1);                                  \
         acc.s1 = fma(src0_mid.s2, weights_row0.s1, acc.s1);                                   \
         acc.s1 = fma(src0_right.s2, weights_row0.s2, acc.s1);                                 \
     })

 #define CONVOLUTION1x3_BIFROST4X1_STRIDE1(acc, src0_left, src0_mid, src0_right, weights_row0) \
     ({                                                                                        \
         acc.s0 = fma(src0_left.s0, weights_row0.s0, acc.s0);                                  \
         acc.s0 = fma(src0_mid.s0, weights_row0.s1, acc.s0);                                   \
         acc.s0 = fma(src0_right.s0, weights_row0.s2, acc.s0);                                 \
         acc.s1 = fma(src0_left.s1, weights_row0.s0, acc.s1);                                  \
         acc.s1 = fma(src0_mid.s1, weights_row0.s1, acc.s1);                                   \
         acc.s1 = fma(src0_right.s1, weights_row0.s2, acc.s1);                                 \
         acc.s2 = fma(src0_left.s2, weights_row0.s0, acc.s2);                                  \
         acc.s2 = fma(src0_mid.s2, weights_row0.s1, acc.s2);                                   \
         acc.s2 = fma(src0_right.s2, weights_row0.s2, acc.s2);                                 \
         acc.s3 = fma(src0_left.s3, weights_row0.s0, acc.s3);                                  \
         acc.s3 = fma(src0_mid.s3, weights_row0.s1, acc.s3);                                   \
         acc.s3 = fma(src0_right.s3, weights_row0.s2, acc.s3);                                 \
     })

 #define CONVOLUTION1x3_BIFROST4X1_STRIDE2(acc, src0_left, src0_mid, src0_right, weights_row0) \
     ({                                                                                        \
         acc.s0 = fma(src0_left.s0, weights_row0.s0, acc.s0);                                  \
         acc.s0 = fma(src0_mid.s0, weights_row0.s1, acc.s0);                                   \
         acc.s0 = fma(src0_right.s0, weights_row0.s2, acc.s0);                                 \
         acc.s1 = fma(src0_left.s2, weights_row0.s0, acc.s1);                                  \
         acc.s1 = fma(src0_mid.s2, weights_row0.s1, acc.s1);                                   \
         acc.s1 = fma(src0_right.s2, weights_row0.s2, acc.s1);                                 \
         acc.s2 = fma(src0_left.s4, weights_row0.s0, acc.s2);                                  \
         acc.s2 = fma(src0_mid.s4, weights_row0.s1, acc.s2);                                   \
         acc.s2 = fma(src0_right.s4, weights_row0.s2, acc.s2);                                 \
         acc.s3 = fma(src0_left.s6, weights_row0.s0, acc.s3);                                  \
         acc.s3 = fma(src0_mid.s6, weights_row0.s1, acc.s3);                                   \
         acc.s3 = fma(src0_right.s6, weights_row0.s2, acc.s3);                                 \
     })

 #endif /* DILATION_X==1 && DILATION_Y==1 */

 #if defined(DEPTH_MULTIPLIER) && defined(DST_CHANNELS) && defined(IS_F32)
 #if defined(CONV_STRIDE_X)

 #if CONV_STRIDE_X == 1
 #define convolution1x3 convolution1x3_stride_1
 #elif CONV_STRIDE_X == 2
 #define convolution1x3 convolution1x3_stride_2
 #elif CONV_STRIDE_X == 3
 #define convolution1x3 convolution1x3_stride_3
 #else /* CONV_STRIDE_X */
 #error "Stride not supported"
 #endif /* CONV_STRIDE_X */

 /** Compute a 1D horizontal convolution of size 3 and stride 1 for floating point type.
  *
  * @param[in] left_pixel   Pointer to the left pixel.
  * @param[in] left_coeff   Weight of the left pixel
  * @param[in] middle_coeff Weight of the middle pixel
  * @param[in] right_coeff  Weight of the right pixel
  *
  * @return a float2 containing 2 convoluted values.
  */
 inline float2 convolution1x3_stride_1(__global const uchar *left_pixel,
                                       const float           left_coeff,
                                       const float           middle_coeff,
                                       const float           right_coeff)
 {
 #if(DILATION_X == 1 && DILATION_Y == 1)
     float4 temp = vload4(0, (__global float *)left_pixel);

     float2 left   = CONVERT(temp.s01, float2);
     float2 middle = CONVERT(temp.s12, float2);
     float2 right  = CONVERT(temp.s23, float2);
     return left * (float2)left_coeff + middle * (float2)middle_coeff + right * (float2)right_coeff;
 #else  /* DILATION_X==1 && DILATION_Y==1 */
     return vload2(0, (__global float *)left_pixel) * (float2)left_coeff
            + vload2(0, (__global float *)(left_pixel) + DILATION_X) * (float2)middle_coeff
            + vload2(0, (__global float *)(left_pixel) + 2 * DILATION_X) * (float2)right_coeff;
 #endif /* DILATION_X==1 && DILATION_Y==1 */
 }

 /** Compute a 1D horizontal convolution of size 3 and stride 2 for floating point type.
  *
  * @param[in] left_pixel   Pointer to the left pixel.
  * @param[in] left_coeff   Weight of the left pixel
  * @param[in] middle_coeff Weight of the middle pixel
  * @param[in] right_coeff  Weight of the right pixel
  *
  * @return a float2 containing 2 convoluted values.
  */
 inline float2 convolution1x3_stride_2(__global const uchar *left_pixel,
                                       const float           left_coeff,
                                       const float           middle_coeff,
                                       const float           right_coeff)
 {
 #if(DILATION_X == 1 && DILATION_Y == 1)
     float4 temp0 = vload4(0, (__global float *)left_pixel);
     float  temp1 = *((__global float *)(left_pixel + 4 * sizeof(float)));

     float2 left   = CONVERT(temp0.s02, float2);
     float2 middle = CONVERT(temp0.s13, float2);
     float2 right  = CONVERT((float2)(temp0.s2, temp1), float2);

     return left * (float2)left_coeff + middle * (float2)middle_coeff + right * (float2)right_coeff;
 #else /* DILATION_X==1 && DILATION_Y==1 */
     __global float *left_pixel_float = (__global float *)left_pixel;

     return vload4(0, left_pixel_float).s02 * (float2)left_coeff
            + vload4(0, left_pixel_float + DILATION_X).s02 * (float2)middle_coeff
            + vload4(0, left_pixel_float + DILATION_X * 2).s02 * (float2)right_coeff;

 #endif /* DILATION_X==1 && DILATION_Y==1 */
 }

 /** Compute a 1D horizontal convolution of size 3 and stride 3 for floating point type.
  *
  * @param[in] left_pixel   Pointer to the left pixel.
  * @param[in] left_coeff   Weight of the left pixel
  * @param[in] middle_coeff Weight of the middle pixel
  * @param[in] right_coeff  Weight of the right pixel
  *
  * @return a float2 containing 2 convoluted values.
  */
 inline float2 convolution1x3_stride_3(__global const uchar *left_pixel,
                                       const float           left_coeff,
                                       const float           middle_coeff,
                                       const float           right_coeff)
 {
 #if(DILATION_X == 1 && DILATION_Y == 1)
     float4 temp0 = vload4(0, (__global float *)left_pixel);
     float2 temp1 = vload2(0, (__global float *)(left_pixel + 4 * sizeof(float)));

     float2 left   = CONVERT(temp0.s03, float2);
     float2 middle = CONVERT((float2)(temp0.s1, temp1.s0), float2);
     float2 right  = CONVERT((float2)(temp0.s2, temp1.s1), float2);

     return left * (float2)left_coeff + middle * (float2)middle_coeff + right * (float2)right_coeff;
 #else  /* DILATION_X==1 && DILATION_Y==1 */
     __global float *left_pixel_float = (__global float *)left_pixel;

     return (float2)(*left_pixel_float, *(left_pixel_float + 3)) * (float2)left_coeff
            + (float2)(*(left_pixel_float + DILATION_X), *(left_pixel_float + DILATION_X + 3)) * (float2)middle_coeff
            + (float2)(*(left_pixel_float + DILATION_X * 2), *(left_pixel_float + DILATION_X * 2 + 3)) * (float2)right_coeff;
 #endif /* DILATION_X==1 && DILATION_Y==1 */
 }

 /** Apply a 3x3 convolution matrix to a single channel F32 input image and return the result.
  *
  * Convolution matrix layout:
  *
  * [ mat0, mat1, mat2 ]\n
  * [ mat3, mat4, mat5 ]\n
  * [ mat6, mat7, mat8 ]\n
  *
  * @param[in] src  A pointer to source Image structure
  * @param[in] mat0 Coefficient from the convolution matrix
  * @param[in] mat1 Coefficient from the convolution matrix
  * @param[in] mat2 Coefficient from the convolution matrix
  * @param[in] mat3 Coefficient from the convolution matrix
  * @param[in] mat4 Coefficient from the convolution matrix
  * @param[in] mat5 Coefficient from the convolution matrix
  * @param[in] mat6 Coefficient from the convolution matrix
  * @param[in] mat0 Coefficient from the convolution matrix
  * @param[in] mat7 Coefficient from the convolution matrix
  * @param[in] mat8 Coefficient from the convolution matrix
  *
  * @return a float2 containing 2 convoluted values.
  */
 inline float2 convolution3x3(
     __global const uchar *src,
     unsigned int          src_stride_y,
     const float mat0, const float mat1, const float mat2,
     const float mat3, const float mat4, const float mat5,
     const float mat6, const float mat7, const float mat8)
 {
     float2 pixels;

     pixels = convolution1x3((src + 0 * DILATION_Y * src_stride_y), mat0, mat1, mat2);
     pixels += convolution1x3((src + 1 * DILATION_Y * src_stride_y), mat3, mat4, mat5);
     pixels += convolution1x3((src + 2 * DILATION_Y * src_stride_y), mat6, mat7, mat8);

     return pixels;
 }

 /** This OpenCL kernel computes the depthwise convolution 3x3
  *
  * @note It is possible to select the activation function to apply using -DACTIVATION_TYPE e.g. -DACTIVATION_TYPE=relu
  * @note A, B variables required by some activation functions are set using -DA_VAL= and -DB_VAL= respectively
  *
  * @param[in] src_ptr                               Pointer to the source tensor. Supported data types: F32
  * @param[in] src_stride_x                          Stride of the source tensor in X dimension (in bytes)
  * @param[in] src_step_x                            src_stride_x * number of elements along X processed per workitem(in bytes)
  * @param[in] src_stride_y                          Stride of the source tensor in Y dimension (in bytes)
  * @param[in] src_step_y                            src_stride_y * number of elements along Y processed per workitem(in bytes)
  * @param[in] src_offset_first_element_in_bytes     The offset of the first element in the source tensor
  * @param[in] src_stride_z                          Stride of the source tensor in Z dimension (in bytes)
  * @param[in] src_step_z                            src_stride_z * number of elements along Y processed per workitem(in bytes)
  * @param[in] dst_ptr                               Pointer to the destination tensor. Supported data types: F32
  * @param[in] dst_stride_x                          Stride of the destination tensor in X dimension (in bytes)
  * @param[in] dst_step_x                            dst_stride_x * number of elements along X processed per workitem(in bytes)
  * @param[in] dst_stride_y                          Stride of the destination tensor in Y dimension (in bytes)
  * @param[in] dst_step_y                            dst_stride_y * number of elements along Y processed per workitem(in bytes)
  * @param[in] dst_stride_z                          Stride of the destination tensor in Z dimension (in bytes)
  * @param[in] dst_step_z                            dst_stride_z * number of elements along Y processed per workitem(in bytes)
  * @param[in] dst_offset_first_element_in_bytes     The offset of the first element in the destination tensor
  * @param[in] weights_ptr                           Pointer to the weights tensor. Supported data types: F32
  * @param[in] weights_stride_x                      Stride of the weights tensor in X dimension (in bytes)
  * @param[in] weights_step_x                        weights_stride_x * number of elements along X processed per workitem(in bytes)
  * @param[in] weights_stride_y                      Stride of the weights tensor in Y dimension (in bytes)
  * @param[in] weights_step_y                        weights_stride_y * number of elements along Y processed per workitem(in bytes)
  * @param[in] weights_stride_z                      Stride of the weights tensor in Z dimension (in bytes)
  * @param[in] weights_step_z                        weights_stride_z * number of elements along Y processed per workitem(in bytes)
  * @param[in] weights_offset_first_element_in_bytes The offset of the first element in the biases vector
  * @param[in] biases_ptr                            (Optional) Pointer to the biases vector. Supported data types: F16/F32
  * @param[in] biases_stride_x                       (Optional) Stride of the biases vector in X dimension (in bytes)
  * @param[in] biases_step_x                         (Optional) biases_stride_x * number of elements along X processed per workitem(in bytes)
  * @param[in] biases_offset_first_element_in_bytes  (Optional) The offset of the first element in the biases vector
  */
 __kernel void depthwise_convolution_3x3(
     TENSOR3D_DECLARATION(src),
     TENSOR3D_DECLARATION(dst),
     TENSOR3D_DECLARATION(weights)
 #if defined(HAS_BIAS)
     ,
     VECTOR_DECLARATION(biases)
 #endif //defined(HAS_BIAS)
 )
 {
     Image    src     = CONVERT_TENSOR3D_TO_IMAGE_STRUCT(src);
     Image    dst     = CONVERT_TENSOR3D_TO_IMAGE_STRUCT(dst);
     Tensor3D weights = CONVERT_TO_TENSOR3D_STRUCT_NO_STEP(weights);

     float2 pixels = 0.0f;

     // Extract channel and linearized batch indices
     const int channel = get_global_id(2) % DST_CHANNELS;
     const int batch   = get_global_id(2) / DST_CHANNELS;
     // Load relevant input and weights data (Accounts depth multiplier when indexing input, OFM = IFM * DEPTH_MULTIPLIER)

     __global uchar *weights_addr = weights.ptr + get_global_id(0) * weights_step_x + get_global_id(1) * weights_step_y + channel * weights_step_z;

     __global uchar *src_addr = src.ptr - batch * (DST_CHANNELS / DEPTH_MULTIPLIER) * (DEPTH_MULTIPLIER - 1) * src_step_z - (channel - (channel / DEPTH_MULTIPLIER)) * src_step_z;

     // Load the weights
     float3 weights_values0 = vload3(0, (__global float *)(weights_addr + 0 * weights_stride_y));
     float3 weights_values1 = vload3(0, (__global float *)(weights_addr + 1 * weights_stride_y));
     float3 weights_values2 = vload3(0, (__global float *)(weights_addr + 2 * weights_stride_y));

     pixels = convolution3x3(src_addr, src_stride_y,
                             weights_values0.s0, weights_values0.s1, weights_values0.s2,
                             weights_values1.s0, weights_values1.s1, weights_values1.s2,
                             weights_values2.s0, weights_values2.s1, weights_values2.s2);
 #if defined(HAS_BIAS)
     Vector biases = CONVERT_TO_VECTOR_STRUCT_NO_STEP(biases);

     float bias = *((__global float *)(vector_offset(&biases, channel)));

     pixels += (float2)bias;
 #endif //defined(HAS_BIAS)

     vstore2(ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, pixels, A_VAL, B_VAL), 0, (__global float *)dst.ptr);
 }
 #endif //defined(CONV_STRIDE_X)

 #if(DILATION_X > 1 || DILATION_Y > 1)

 /** Perform 3x3 convolution for stride_x=1 and stride_y=1 when DILATION_X>1 or DILATION_Y>1 for F32
  *
  * @param[in] src_addr         Pointer to the starting position of where to perform the convolution
  * @param[in] src_stride_x     Stride of the source tensor in X dimension (in bytes)
  * @param[in] src_stride_y     Stride of the source tensor in Y dimension (in bytes)
  * @param[in] y_offset         Offset from the source tensor from which to start convolution
  * @param[in] weights_addr     Pointer from where to get weights
  * @param[in] weights_stride_y Stride of weights tesnsor in Y dimension
  */
 inline float2 convolution_3x3_dilation_stridex1_stridey1_bifrost_f32(__global uchar *src_addr, const int stride_x_bytes, const int stride_y_bytes,
                                                                      const int y_offset, __global uchar *weights_addr, const int weights_stride_y)
 {
     // Load the weights
     float3 weights_row0 = vload3(0, (__global float *)(weights_addr + 0 * weights_stride_y));
     float3 weights_row1 = vload3(0, (__global float *)(weights_addr + 1 * weights_stride_y));
     float3 weights_row2 = vload3(0, (__global float *)(weights_addr + 2 * weights_stride_y));

     float2 pixels0 = 0.0f;

     float2 src00_left  = vload2(0, (__global float *)ptr_offset(src_addr, 0, y_offset, stride_x_bytes, stride_y_bytes)); // Row0
     float2 src00_mid   = vload2(0, (__global float *)ptr_offset(src_addr, DILATION_X, y_offset, stride_x_bytes, stride_y_bytes));
     float2 src00_right = vload2(0, (__global float *)ptr_offset(src_addr, 2 * DILATION_X, y_offset, stride_x_bytes, stride_y_bytes));

     float2 src10_left  = vload2(0, (__global float *)ptr_offset(src_addr, 0, y_offset + DILATION_Y, stride_x_bytes, stride_y_bytes)); // Row1
     float2 src10_mid   = vload2(0, (__global float *)ptr_offset(src_addr, DILATION_X, y_offset + DILATION_Y, stride_x_bytes, stride_y_bytes));
     float2 src10_right = vload2(0, (__global float *)ptr_offset(src_addr, 2 * DILATION_X, y_offset + DILATION_Y, stride_x_bytes, stride_y_bytes));

     float2 src20_left  = vload2(0, (__global float *)ptr_offset(src_addr, 0, y_offset + DILATION_Y * 2, stride_x_bytes, stride_y_bytes)); // Row2
     float2 src20_mid   = vload2(0, (__global float *)ptr_offset(src_addr, DILATION_X, y_offset + DILATION_Y * 2, stride_x_bytes, stride_y_bytes));
     float2 src20_right = vload2(0, (__global float *)ptr_offset(src_addr, 2 * DILATION_X, y_offset + DILATION_Y * 2, stride_x_bytes, stride_y_bytes));

     CONVOLUTION1x3_BIFROST2X1_STRIDE1(pixels0, src00_left, src00_mid, src00_right, weights_row0);
     CONVOLUTION1x3_BIFROST2X1_STRIDE1(pixels0, src10_left, src10_mid, src10_right, weights_row1);
     CONVOLUTION1x3_BIFROST2X1_STRIDE1(pixels0, src20_left, src20_mid, src20_right, weights_row2);

     return pixels0;
 }

 /** Perform 3x3 convolution for stride_x=2 and stride_y=2 when DILATION_X>1 or DILATION_Y>1 for F32
  *
  * @param[in] src_addr         Pointer to the starting position of where to perform the convolution
  * @param[in] src_stride_x     Stride of the source tensor in X dimension (in bytes)
  * @param[in] src_stride_y     Stride of the source tensor in Y dimension (in bytes)
  * @param[in] y_offset         Offset from the source tensor from which to start convolution
  * @param[in] weights_addr     Pointer from where to get weights
  * @param[in] weights_stride_y Stride of weights tesnsor in Y dimension
  */
 inline float2 convolution_3x3_dilation_stridex2_stridey2_bifrost_f32(__global uchar *src_addr, const int stride_x_bytes, const int stride_y_bytes,
                                                                      const int y_offset, __global uchar *weights_addr, const int weights_stride_y)
 {
     // Load the weights
     float3 weights_row0 = vload3(0, (__global float *)(weights_addr + 0 * weights_stride_y));
     float3 weights_row1 = vload3(0, (__global float *)(weights_addr + 1 * weights_stride_y));
     float3 weights_row2 = vload3(0, (__global float *)(weights_addr + 2 * weights_stride_y));

     float2 pixels0 = 0.0f;

     float3 src00_left  = vload3(0, (__global float *)ptr_offset(src_addr, 0, y_offset, stride_x_bytes, stride_y_bytes)); // Row0
     float3 src00_mid   = vload3(0, (__global float *)ptr_offset(src_addr, DILATION_X, y_offset, stride_x_bytes, stride_y_bytes));
     float3 src00_right = vload3(0, (__global float *)ptr_offset(src_addr, 2 * DILATION_X, y_offset, stride_x_bytes, stride_y_bytes));

     float3 src10_left  = vload3(0, (__global float *)ptr_offset(src_addr, 0, y_offset + DILATION_Y, stride_x_bytes, stride_y_bytes)); // Row1
     float3 src10_mid   = vload3(0, (__global float *)ptr_offset(src_addr, DILATION_X, y_offset + DILATION_Y, stride_x_bytes, stride_y_bytes));
     float3 src10_right = vload3(0, (__global float *)ptr_offset(src_addr, 2 * DILATION_X, y_offset + DILATION_Y, stride_x_bytes, stride_y_bytes));

     float3 src20_left  = vload3(0, (__global float *)ptr_offset(src_addr, 0, y_offset + DILATION_Y * 2, stride_x_bytes, stride_y_bytes)); // Row2
     float3 src20_mid   = vload3(0, (__global float *)ptr_offset(src_addr, DILATION_X, y_offset + DILATION_Y * 2, stride_x_bytes, stride_y_bytes));
     float3 src20_right = vload3(0, (__global float *)ptr_offset(src_addr, 2 * DILATION_X, y_offset + DILATION_Y * 2, stride_x_bytes, stride_y_bytes));

     CONVOLUTION1x3_BIFROST2X1_STRIDE2(pixels0, src00_left, src00_mid, src00_right, weights_row0);
     CONVOLUTION1x3_BIFROST2X1_STRIDE2(pixels0, src10_left, src10_mid, src10_right, weights_row1);
     CONVOLUTION1x3_BIFROST2X1_STRIDE2(pixels0, src20_left, src20_mid, src20_right, weights_row2);

     return pixels0;
 }

 #endif /* (DILATION_X > 1 || DILATION_Y > 1) */

 /** This OpenCL kernel is optimized for Bifrost architectures and computes the depthwise convolution 3x3 when both
  * stride_x and stride_y are equal to 1
  *
  * @note It is possible to select the activation function to apply using -DACTIVATION_TYPE e.g. -DACTIVATION_TYPE=relu
  * @note If activation function is enabled, the data type must be passed at compile time using -DDATA_TYPE e.g. -DDATA_TYPE=float. Supported data types: float.
  * @note A, B variables required by some activation functions are set using -DA_VAL= and -DB_VAL= respectively
  * @note Vector size should be given as a preprocessor argument using -DVEC_SIZE=size
  *
  * @param[in] src_ptr                               Pointer to the source tensor. Supported data types: F32
  * @param[in] src_stride_x                          Stride of the source tensor in X dimension (in bytes)
  * @param[in] src_step_x                            src_stride_x * number of elements along X processed per workitem(in bytes)
  * @param[in] src_stride_y                          Stride of the source tensor in Y dimension (in bytes)
  * @param[in] src_step_y                            src_stride_y * number of elements along Y processed per workitem(in bytes)
  * @param[in] src_offset_first_element_in_bytes     The offset of the first element in the source tensor
  * @param[in] src_stride_z                          Stride of the source tensor in Z dimension (in bytes)
  * @param[in] src_step_z                            src_stride_z * number of elements along Y processed per workitem(in bytes)
  * @param[in] dst_ptr                               Pointer to the destination tensor. Supported data types: F32
  * @param[in] dst_stride_x                          Stride of the destination tensor in X dimension (in bytes)
  * @param[in] dst_step_x                            dst_stride_x * number of elements along X processed per workitem(in bytes)
  * @param[in] dst_stride_y                          Stride of the destination tensor in Y dimension (in bytes)
  * @param[in] dst_step_y                            dst_stride_y * number of elements along Y processed per workitem(in bytes)
  * @param[in] dst_stride_z                          Stride of the destination tensor in Z dimension (in bytes)
  * @param[in] dst_step_z                            dst_stride_z * number of elements along Y processed per workitem(in bytes)
  * @param[in] dst_offset_first_element_in_bytes     The offset of the first element in the destination tensor
  * @param[in] weights_ptr                           Pointer to the weights tensor. Supported data types: F32
  * @param[in] weights_stride_x                      Stride of the weights tensor in X dimension (in bytes)
  * @param[in] weights_step_x                        weights_stride_x * number of elements along X processed per workitem(in bytes)
  * @param[in] weights_stride_y                      Stride of the weights tensor in Y dimension (in bytes)
  * @param[in] weights_step_y                        weights_stride_y * number of elements along Y processed per workitem(in bytes)
  * @param[in] weights_stride_z                      Stride of the weights tensor in Z dimension (in bytes)
  * @param[in] weights_step_z                        weights_stride_z * number of elements along Y processed per workitem(in bytes)
  * @param[in] weights_offset_first_element_in_bytes The offset of the first element in the biases vector
  * @param[in] biases_ptr                            (Optional) Pointer to the biases vector. Supported data types: F32
  * @param[in] biases_stride_x                       (Optional) Stride of the biases vector in X dimension (in bytes)
  * @param[in] biases_step_x                         (Optional) biases_stride_x * number of elements along X processed per workitem(in bytes)
  * @param[in] biases_offset_first_element_in_bytes  (Optional) The offset of the first element in the biases vector
  */
 __kernel void depthwise_convolution_3x3_stridex1_stridey1_bifrost_f32(
     TENSOR3D_DECLARATION(src),
     TENSOR3D_DECLARATION(dst),
     TENSOR3D_DECLARATION(weights)
 #if defined(HAS_BIAS)
     ,
     VECTOR_DECLARATION(biases)
 #endif //defined(HAS_BIAS)
 )
 {
     Image    src     = CONVERT_TENSOR3D_TO_IMAGE_STRUCT(src);
     Image    dst     = CONVERT_TENSOR3D_TO_IMAGE_STRUCT(dst);
     Tensor3D weights = CONVERT_TO_TENSOR3D_STRUCT_NO_STEP(weights);

     float2 pixels0 = 0.0f;
     float2 pixels1 = 0.0f;
     float2 pixels2 = 0.0f;
     float2 pixels3 = 0.0f;

     // Extract channel and linearized batch indices
     const int channel = get_global_id(2) % DST_CHANNELS;
     const int batch   = get_global_id(2) / DST_CHANNELS;
     // Load relevant input and weights data (Accounts depth multiplier when indexing input, OFM = IFM * DEPTH_MULTIPLIER)
     __global uchar *weights_addr = weights.ptr + get_global_id(0) * weights_step_x + get_global_id(1) * weights_step_y + channel * weights_step_z;
     __global uchar *src_addr     = src.ptr - batch * (DST_CHANNELS / DEPTH_MULTIPLIER) * (DEPTH_MULTIPLIER - 1) * src_step_z - (channel - (channel / DEPTH_MULTIPLIER)) * src_step_z;

 #if(DILATION_X == 1 && DILATION_Y == 1)
     // Load the weights
     float3 weights_row0 = vload3(0, (__global float *)(weights_addr + 0 * weights_stride_y));
     float3 weights_row1 = vload3(0, (__global float *)(weights_addr + 1 * weights_stride_y));
     float3 weights_row2 = vload3(0, (__global float *)(weights_addr + 2 * weights_stride_y));

     // Note: Since each work-item computes 4x2 elements, we need to load 6 rows from the input tensor
     float4 src00 = vload4(0, (__global float *)(src_addr + 0 * src_stride_y)); // Row0
     float4 src10 = vload4(0, (__global float *)(src_addr + 1 * src_stride_y)); // Row1
     float4 src20 = vload4(0, (__global float *)(src_addr + 2 * src_stride_y)); // Row2
     float4 src30 = vload4(0, (__global float *)(src_addr + 3 * src_stride_y)); // Row3
     float4 src40 = vload4(0, (__global float *)(src_addr + 4 * src_stride_y)); // Row4
     float4 src50 = vload4(0, (__global float *)(src_addr + 5 * src_stride_y)); // Row5

     CONVOLUTION1x3_BIFROST2X1_STRIDE1(pixels0, src00, weights_row0);
     CONVOLUTION1x3_BIFROST2X1_STRIDE1(pixels0, src10, weights_row1);
     CONVOLUTION1x3_BIFROST2X1_STRIDE1(pixels0, src20, weights_row2);
     CONVOLUTION1x3_BIFROST2X1_STRIDE1(pixels1, src10, weights_row0);
     CONVOLUTION1x3_BIFROST2X1_STRIDE1(pixels1, src20, weights_row1);
     CONVOLUTION1x3_BIFROST2X1_STRIDE1(pixels1, src30, weights_row2);
     CONVOLUTION1x3_BIFROST2X1_STRIDE1(pixels2, src20, weights_row0);
     CONVOLUTION1x3_BIFROST2X1_STRIDE1(pixels2, src30, weights_row1);
     CONVOLUTION1x3_BIFROST2X1_STRIDE1(pixels2, src40, weights_row2);
     CONVOLUTION1x3_BIFROST2X1_STRIDE1(pixels3, src30, weights_row0);
     CONVOLUTION1x3_BIFROST2X1_STRIDE1(pixels3, src40, weights_row1);
     CONVOLUTION1x3_BIFROST2X1_STRIDE1(pixels3, src50, weights_row2);

 #else /* DILATION_X==1 && DILATION_Y==1 */

     //3x3 Convolution of elements starting in 0th row
     pixels0 = convolution_3x3_dilation_stridex1_stridey1_bifrost_f32(src_addr, src.stride_x, src.stride_y, 0, weights_addr, weights_stride_y);
     //3x3 Convolution of elements starting in 1st row
     pixels1 = convolution_3x3_dilation_stridex1_stridey1_bifrost_f32(src_addr, src.stride_x, src.stride_y, 1, weights_addr, weights_stride_y);
     //3x3 Convolution of elements starting in 2nd row
     pixels2 = convolution_3x3_dilation_stridex1_stridey1_bifrost_f32(src_addr, src.stride_x, src.stride_y, 2, weights_addr, weights_stride_y);
     //3x3 Convolution of elements starting in 3rd row
     pixels3 = convolution_3x3_dilation_stridex1_stridey1_bifrost_f32(src_addr, src.stride_x, src.stride_y, 3, weights_addr, weights_stride_y);

 #endif /* DILATION_X==1 && DILATION_Y==1 */

 #ifdef HAS_BIAS
     Vector biases = CONVERT_TO_VECTOR_STRUCT_NO_STEP(biases);

     float bias = *((__global float *)(vector_offset(&biases, channel)));

     pixels0 += (float2)bias;
     pixels1 += (float2)bias;
     pixels2 += (float2)bias;
     pixels3 += (float2)bias;
 #endif /* defined(HAS_BIAS) */

     vstore2(ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, pixels0, A_VAL, B_VAL), 0, (__global float *)(dst.ptr + 0 * dst_stride_y));
     vstore2(ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, pixels1, A_VAL, B_VAL), 0, (__global float *)(dst.ptr + 1 * dst_stride_y));
     vstore2(ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, pixels2, A_VAL, B_VAL), 0, (__global float *)(dst.ptr + 2 * dst_stride_y));
     vstore2(ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, pixels3, A_VAL, B_VAL), 0, (__global float *)(dst.ptr + 3 * dst_stride_y));
 }

 /** This OpenCL kernel is optimized for Bifrost architectures and computes the depthwise convolution 3x3 when both
  * stride_x and stride_y are equal to 2
  *
  * @note It is possible to select the activation function to apply using -DACTIVATION_TYPE e.g. -DACTIVATION_TYPE=relu
  * @note If activation function is enabled, the data type must be passed at compile time using -DDATA_TYPE e.g. -DDATA_TYPE=float. Supported data types: float.
  * @note A, B variables required by some activation functions are set using -DA_VAL= and -DB_VAL= respectively
  * @note Vector size should be given as a preprocessor argument using -DVEC_SIZE=size
  *
  * @param[in] src_ptr                               Pointer to the source tensor. Supported data types: F32
  * @param[in] src_stride_x                          Stride of the source tensor in X dimension (in bytes)
  * @param[in] src_step_x                            src_stride_x * number of elements along X processed per workitem(in bytes)
  * @param[in] src_stride_y                          Stride of the source tensor in Y dimension (in bytes)
  * @param[in] src_step_y                            src_stride_y * number of elements along Y processed per workitem(in bytes)
  * @param[in] src_offset_first_element_in_bytes     The offset of the first element in the source tensor
  * @param[in] src_stride_z                          Stride of the source tensor in Z dimension (in bytes)
  * @param[in] src_step_z                            src_stride_z * number of elements along Y processed per workitem(in bytes)
  * @param[in] dst_ptr                               Pointer to the destination tensor. Supported data types: F32
  * @param[in] dst_stride_x                          Stride of the destination tensor in X dimension (in bytes)
  * @param[in] dst_step_x                            dst_stride_x * number of elements along X processed per workitem(in bytes)
  * @param[in] dst_stride_y                          Stride of the destination tensor in Y dimension (in bytes)
  * @param[in] dst_step_y                            dst_stride_y * number of elements along Y processed per workitem(in bytes)
  * @param[in] dst_stride_z                          Stride of the destination tensor in Z dimension (in bytes)
  * @param[in] dst_step_z                            dst_stride_z * number of elements along Y processed per workitem(in bytes)
  * @param[in] dst_offset_first_element_in_bytes     The offset of the first element in the destination tensor
  * @param[in] weights_ptr                           Pointer to the weights tensor. Supported data types: F32
  * @param[in] weights_stride_x                      Stride of the weights tensor in X dimension (in bytes)
  * @param[in] weights_step_x                        weights_stride_x * number of elements along X processed per workitem(in bytes)
  * @param[in] weights_stride_y                      Stride of the weights tensor in Y dimension (in bytes)
  * @param[in] weights_step_y                        weights_stride_y * number of elements along Y processed per workitem(in bytes)
  * @param[in] weights_stride_z                      Stride of the weights tensor in Z dimension (in bytes)
  * @param[in] weights_step_z                        weights_stride_z * number of elements along Y processed per workitem(in bytes)
  * @param[in] weights_offset_first_element_in_bytes The offset of the first element in the biases vector
  * @param[in] biases_ptr                            (Optional) Pointer to the biases vector. Supported data types: F32
  * @param[in] biases_stride_x                       (Optional) Stride of the biases vector in X dimension (in bytes)
  * @param[in] biases_step_x                         (Optional) biases_stride_x * number of elements along X processed per workitem(in bytes)
  * @param[in] biases_offset_first_element_in_bytes  (Optional) The offset of the first element in the biases vector
  */
 __kernel void depthwise_convolution_3x3_stridex2_stridey2_bifrost_f32(
     TENSOR3D_DECLARATION(src),
     TENSOR3D_DECLARATION(dst),
     TENSOR3D_DECLARATION(weights)
 #if defined(HAS_BIAS)
     ,
     VECTOR_DECLARATION(biases)
 #endif //defined(HAS_BIAS)
 )
 {
     Image    src     = CONVERT_TENSOR3D_TO_IMAGE_STRUCT(src);
     Image    dst     = CONVERT_TENSOR3D_TO_IMAGE_STRUCT(dst);
     Tensor3D weights = CONVERT_TO_TENSOR3D_STRUCT_NO_STEP(weights);

     float2 pixels0 = 0.0f;
     float2 pixels1 = 0.0f;

     // Extract channel and linearized batch indices
     const int channel = get_global_id(2) % DST_CHANNELS;
     const int batch   = get_global_id(2) / DST_CHANNELS;
     // Load relevant input and weights data (Accounts depth multiplier when indexing input, OFM = IFM * DEPTH_MULTIPLIER)
     __global uchar *weights_addr = weights.ptr + get_global_id(0) * weights_step_x + get_global_id(1) * weights_step_y + channel * weights_step_z;
     __global uchar *src_addr     = src.ptr - batch * (DST_CHANNELS / DEPTH_MULTIPLIER) * (DEPTH_MULTIPLIER - 1) * src_step_z - (channel - (channel / DEPTH_MULTIPLIER)) * src_step_z;

 #if(DILATION_X == 1 && DILATION_Y == 1)

     // Load the weights
     float3 weights_row0 = vload3(0, (__global float *)(weights_addr + 0 * weights_stride_y));
     float3 weights_row1 = vload3(0, (__global float *)(weights_addr + 1 * weights_stride_y));
     float3 weights_row2 = vload3(0, (__global float *)(weights_addr + 2 * weights_stride_y));

     // Note: Since each work-item computes 4x2 elements, we need to load 5 rows from the input tensor
     float4 src00 = vload4(0, (__global float *)(src_addr + 0 * src_stride_y)); // Row0
     float2 src01 = vload2(2, (__global float *)(src_addr + 0 * src_stride_y)); // Row0
     float4 src10 = vload4(0, (__global float *)(src_addr + 1 * src_stride_y)); // Row1
     float2 src11 = vload2(2, (__global float *)(src_addr + 1 * src_stride_y)); // Row1
     float4 src20 = vload4(0, (__global float *)(src_addr + 2 * src_stride_y)); // Row2
     float2 src21 = vload2(2, (__global float *)(src_addr + 2 * src_stride_y)); // Row2
     float4 src30 = vload4(0, (__global float *)(src_addr + 3 * src_stride_y)); // Row3
     float2 src31 = vload2(2, (__global float *)(src_addr + 3 * src_stride_y)); // Row3
     float4 src40 = vload4(0, (__global float *)(src_addr + 4 * src_stride_y)); // Row4
     float2 src41 = vload2(2, (__global float *)(src_addr + 4 * src_stride_y)); // Row4

     CONVOLUTION1x3_BIFROST2X1_STRIDE2(pixels0, src00, src01, weights_row0);
     CONVOLUTION1x3_BIFROST2X1_STRIDE2(pixels0, src10, src11, weights_row1);
     CONVOLUTION1x3_BIFROST2X1_STRIDE2(pixels0, src20, src21, weights_row2);
     CONVOLUTION1x3_BIFROST2X1_STRIDE2(pixels1, src20, src21, weights_row0);
     CONVOLUTION1x3_BIFROST2X1_STRIDE2(pixels1, src30, src31, weights_row1);
     CONVOLUTION1x3_BIFROST2X1_STRIDE2(pixels1, src40, src41, weights_row2);

 #else  /* DILATION_X==1 && DILATION_Y==1 */

     //3x3 Convolution of elements starting in 0th row
     pixels0 = convolution_3x3_dilation_stridex2_stridey2_bifrost_f32(src_addr, src.stride_x, src.stride_y, 0, weights_addr, weights_stride_y);
     //3x3 Convolution of elements starting in 2nd row
     pixels1 = convolution_3x3_dilation_stridex2_stridey2_bifrost_f32(src_addr, src.stride_x, src.stride_y, 2, weights_addr, weights_stride_y);
 #endif /* DILATION_X==1 && DILATION_Y==1 */

 #ifdef HAS_BIAS
     Vector biases = CONVERT_TO_VECTOR_STRUCT_NO_STEP(biases);

     float bias = *((__global float *)(vector_offset(&biases, channel)));

     pixels0 += (float2)bias;
     pixels1 += (float2)bias;
 #endif /* defined(HAS_BIAS) */

     vstore2(ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, pixels0, A_VAL, B_VAL), 0, (__global float *)(dst.ptr + 0 * dst_stride_y));
     vstore2(ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, pixels1, A_VAL, B_VAL), 0, (__global float *)(dst.ptr + 1 * dst_stride_y));
 }

 #endif // defined(DEPTH_MULTIPLIER) && defined(DST_CHANNELS) && defined(IS_F32)

 #if defined(VEC_SIZE) && defined(DATA_TYPE) && defined(DST_WIDTH)
 /** Reshape the weights for quantized depthwise convolution
  *
  * @note Datatype should be given as a preprocessor argument using -DDATA_TYPE=type, e.g. -DDATA_TYPE=uint8
  * @note Output width should be given as a preprocessor argument using -DDST_WIDTH=width, e.g. -DDST_WIDTH=128
  * @note Vector size should be given as a preprocessor argument using -DVEC_SIZE=vec_size, e.g., -DVEC_SIZE=4
  * @attention Input's height and width should be 3
  *
  * @param[in]  src_ptr                           Pointer to the source tensor. Supported data types: All
  * @param[in]  src_stride_x                      Stride of the source tensor in X dimension (in bytes)
  * @param[in]  src_step_x                        src_stride_x * number of elements along X processed per workitem(in bytes)
  * @param[in]  src_stride_y                      Stride of the source tensor in Y dimension (in bytes)
  * @param[in]  src_step_y                        src_stride_y * number of elements along Y processed per workitem(in bytes)
  * @param[in]  src_stride_z                      Stride of the source tensor in Z dimension (in bytes)
  * @param[in]  src_step_z                        src_stride_z * number of elements along Y processed per workitem(in bytes)
  * @param[in]  src_offset_first_element_in_bytes The offset of the first element in the source tensor
  * @param[out] dst_ptr                           Pointer to the destination tensor. Supported data types: same as @p src_ptr
  * @param[in]  dst_stride_x                      Stride of the destination tensor in X dimension (in bytes)
  * @param[in]  dst_step_x                        dst_stride_x * number of elements along X processed per workitem(in bytes)
  * @param[in]  dst_stride_y                      Stride of the destination tensor in Y dimension (in bytes)
  * @param[in]  dst_step_y                        dst_stride_y * number of elements along Y processed per workitem(in bytes)
  * @param[in]  dst_offset_first_element_in_bytes The offset of the first element in the destination tensor
  */
 __kernel void depthwise_convolution_reshape_weights(
     TENSOR3D_DECLARATION(src),
     IMAGE_DECLARATION(dst))
 {
     Vector    src = CONVERT_TO_VECTOR_STRUCT(src);
     const int x   = get_global_id(0);

     // Load 3x3xVEC_SIZE weights
     VEC_DATA_TYPE(DATA_TYPE, VEC_SIZE)
     w0 = VLOAD(VEC_SIZE)(0, src.ptr + 0 * src_stride_y + 0 * src_stride_z);
     VEC_DATA_TYPE(DATA_TYPE, VEC_SIZE)
     w1 = VLOAD(VEC_SIZE)(0, src.ptr + 1 * src_stride_y + 0 * src_stride_z);
     VEC_DATA_TYPE(DATA_TYPE, VEC_SIZE)
     w2 = VLOAD(VEC_SIZE)(0, src.ptr + 2 * src_stride_y + 0 * src_stride_z);
     VEC_DATA_TYPE(DATA_TYPE, VEC_SIZE)
     w3 = VLOAD(VEC_SIZE)(0, src.ptr + 0 * src_stride_y + 1 * src_stride_z);
     VEC_DATA_TYPE(DATA_TYPE, VEC_SIZE)
     w4 = VLOAD(VEC_SIZE)(0, src.ptr + 1 * src_stride_y + 1 * src_stride_z);
     VEC_DATA_TYPE(DATA_TYPE, VEC_SIZE)
     w5 = VLOAD(VEC_SIZE)(0, src.ptr + 2 * src_stride_y + 1 * src_stride_z);
     VEC_DATA_TYPE(DATA_TYPE, VEC_SIZE)
     w6 = VLOAD(VEC_SIZE)(0, src.ptr + 0 * src_stride_y + 2 * src_stride_z);
     VEC_DATA_TYPE(DATA_TYPE, VEC_SIZE)
     w7 = VLOAD(VEC_SIZE)(0, src.ptr + 1 * src_stride_y + 2 * src_stride_z);
     VEC_DATA_TYPE(DATA_TYPE, VEC_SIZE)
     w8 = VLOAD(VEC_SIZE)(0, src.ptr + 2 * src_stride_y + 2 * src_stride_z);

     __global uchar *dst_addr = dst_ptr + dst_offset_first_element_in_bytes + x * DST_WIDTH * sizeof(DATA_TYPE);

 #if defined(TRANSPOSE)
 #if VEC_SIZE != 4
 #error "VEC_SIZE not supported"
 #else  // VEC_SIZE != 4
     VSTORE(VEC_SIZE)
     ((VEC_DATA_TYPE(DATA_TYPE, VEC_SIZE))(w0.s0, w1.s0, w2.s0, w3.s0), 0, dst_addr + 0);
     VSTORE(VEC_SIZE)
     ((VEC_DATA_TYPE(DATA_TYPE, VEC_SIZE))(w4.s0, w5.s0, w6.s0, w7.s0), 0, dst_addr + 1 * sizeof(DATA_TYPE) * VEC_SIZE);
     VSTORE(VEC_SIZE)
     ((VEC_DATA_TYPE(DATA_TYPE, VEC_SIZE))(w8.s0, w0.s1, w1.s1, w2.s1), 0, dst_addr + 2 * sizeof(DATA_TYPE) * VEC_SIZE);
     VSTORE(VEC_SIZE)
     ((VEC_DATA_TYPE(DATA_TYPE, VEC_SIZE))(w3.s1, w4.s1, w5.s1, w6.s1), 0, dst_addr + 3 * sizeof(DATA_TYPE) * VEC_SIZE);
     VSTORE(VEC_SIZE)
     ((VEC_DATA_TYPE(DATA_TYPE, VEC_SIZE))(w7.s1, w8.s1, w0.s2, w1.s2), 0, dst_addr + 4 * sizeof(DATA_TYPE) * VEC_SIZE);
     VSTORE(VEC_SIZE)
     ((VEC_DATA_TYPE(DATA_TYPE, VEC_SIZE))(w2.s2, w3.s2, w4.s2, w5.s2), 0, dst_addr + 5 * sizeof(DATA_TYPE) * VEC_SIZE);
     VSTORE(VEC_SIZE)
     ((VEC_DATA_TYPE(DATA_TYPE, VEC_SIZE))(w6.s2, w7.s2, w8.s2, w0.s3), 0, dst_addr + 6 * sizeof(DATA_TYPE) * VEC_SIZE);
     VSTORE(VEC_SIZE)
     ((VEC_DATA_TYPE(DATA_TYPE, VEC_SIZE))(w1.s3, w2.s3, w3.s3, w4.s3), 0, dst_addr + 7 * sizeof(DATA_TYPE) * VEC_SIZE);
     VSTORE(VEC_SIZE)
     ((VEC_DATA_TYPE(DATA_TYPE, VEC_SIZE))(w5.s3, w6.s3, w7.s3, w8.s3), 0, dst_addr + 8 * sizeof(DATA_TYPE) * VEC_SIZE);
 #endif // VEC_SIZE != 4
 #else  // !defined(TRANSPOSE)
     VSTORE(VEC_SIZE)
     (w0, 0, dst_addr + 0);
     VSTORE(VEC_SIZE)
     (w1, 0, dst_addr + 1 * sizeof(DATA_TYPE) * VEC_SIZE);
     VSTORE(VEC_SIZE)
     (w2, 0, dst_addr + 2 * sizeof(DATA_TYPE) * VEC_SIZE);
     VSTORE(VEC_SIZE)
     (w3, 0, dst_addr + 3 * sizeof(DATA_TYPE) * VEC_SIZE);
     VSTORE(VEC_SIZE)
     (w4, 0, dst_addr + 4 * sizeof(DATA_TYPE) * VEC_SIZE);
     VSTORE(VEC_SIZE)
     (w5, 0, dst_addr + 5 * sizeof(DATA_TYPE) * VEC_SIZE);
     VSTORE(VEC_SIZE)
     (w6, 0, dst_addr + 6 * sizeof(DATA_TYPE) * VEC_SIZE);
     VSTORE(VEC_SIZE)
     (w7, 0, dst_addr + 7 * sizeof(DATA_TYPE) * VEC_SIZE);
     VSTORE(VEC_SIZE)
     (w8, 0, dst_addr + 8 * sizeof(DATA_TYPE) * VEC_SIZE);
 #endif // defined(TRANSPOSE)
 }
 #endif // defined(VEC_SIZE) && defined(DATA_TYPE) && defined(DST_WIDTH)

 #if defined(ARM_COMPUTE_OPENCL_FP16_ENABLED) && defined(DEPTH_MULTIPLIER) && defined(DST_CHANNELS) && defined(IS_F16)
 #if defined(CONV_STRIDE_X)
 #if CONV_STRIDE_X == 1
 #define convolution1x3_f16 convolution1x3_stride_1_f16
 #elif CONV_STRIDE_X == 2
 #define convolution1x3_f16 convolution1x3_stride_2_f16
 #elif CONV_STRIDE_X == 3
 #define convolution1x3_f16 convolution1x3_stride_3_f16
 #else /* CONV_STRIDE_X */
 #error "Stride not supported"
 #endif /* CONV_STRIDE_X */

 #if(DILATION_X > 1 || DILATION_Y > 1)

 /** Perform 3x3 convolution for stride_x=1 and stride_y=1 when DILATION_X>1 or DILATION_Y>1 for f16
  *
  * @param[in] src_addr         Pointer to the starting position of where to perform the convolution
  * @param[in] src_stride_x     Stride of the source tensor in X dimension (in bytes)
  * @param[in] src_stride_y     Stride of the source tensor in Y dimension (in bytes)
  * @param[in] y_offset         Offset from the source tensor from which to start convolution
  * @param[in] weights_addr     Pointer from where to get weights
  * @param[in] weights_stride_y Stride of weights tesnsor in Y dimension
  */
 inline half4 convolution_3x3_dilation_stridex1_stridey1_bifrost_f16(__global uchar *src_addr, const int stride_x_bytes, const int stride_y_bytes,
                                                                     const int y_offset, __global uchar *weights_addr, const int weights_stride_y)
 {
     // Load the weights
     half3 weights_row0 = vload3(0, (__global half *)(weights_addr + 0 * weights_stride_y));
     half3 weights_row1 = vload3(0, (__global half *)(weights_addr + 1 * weights_stride_y));
     half3 weights_row2 = vload3(0, (__global half *)(weights_addr + 2 * weights_stride_y));

     half4 pixels0 = 0.0f;

     half4 src00_left  = vload4(0, (__global half *)ptr_offset(src_addr, 0, y_offset, stride_x_bytes, stride_y_bytes)); // Row0
     half4 src00_mid   = vload4(0, (__global half *)ptr_offset(src_addr, DILATION_X, y_offset, stride_x_bytes, stride_y_bytes));
     half4 src00_right = vload4(0, (__global half *)ptr_offset(src_addr, 2 * DILATION_X, y_offset, stride_x_bytes, stride_y_bytes));

     half4 src10_left  = vload4(0, (__global half *)ptr_offset(src_addr, 0, y_offset + DILATION_Y, stride_x_bytes, stride_y_bytes)); // Row1
     half4 src10_mid   = vload4(0, (__global half *)ptr_offset(src_addr, DILATION_X, y_offset + DILATION_Y, stride_x_bytes, stride_y_bytes));
     half4 src10_right = vload4(0, (__global half *)ptr_offset(src_addr, 2 * DILATION_X, y_offset + DILATION_Y, stride_x_bytes, stride_y_bytes));

     half4 src20_left  = vload4(0, (__global half *)ptr_offset(src_addr, 0, y_offset + DILATION_Y * 2, stride_x_bytes, stride_y_bytes)); // Row2
     half4 src20_mid   = vload4(0, (__global half *)ptr_offset(src_addr, DILATION_X, y_offset + DILATION_Y * 2, stride_x_bytes, stride_y_bytes));
     half4 src20_right = vload4(0, (__global half *)ptr_offset(src_addr, 2 * DILATION_X, y_offset + DILATION_Y * 2, stride_x_bytes, stride_y_bytes));

     CONVOLUTION1x3_BIFROST4X1_STRIDE1(pixels0, src00_left, src00_mid, src00_right, weights_row0);
     CONVOLUTION1x3_BIFROST4X1_STRIDE1(pixels0, src10_left, src10_mid, src10_right, weights_row1);
     CONVOLUTION1x3_BIFROST4X1_STRIDE1(pixels0, src20_left, src20_mid, src20_right, weights_row2);

     return pixels0;
 }

 /** Perform 3x3 convolution for stride_x=2 and stride_y=2 when DILATION_X>1 or DILATION_Y>1 for F16
  *
  * @param[in] src_addr         Pointer to the starting position of where to perform the convolution
  * @param[in] src_stride_x     Stride of the source tensor in X dimension (in bytes)
  * @param[in] src_stride_y     Stride of the source tensor in Y dimension (in bytes)
  * @param[in] y_offset         Offset from the source tensor from which to start convolution
  * @param[in] weights_addr     Pointer from where to get weights
  * @param[in] weights_stride_y Stride of weights tesnsor in Y dimension
  */
 inline half4 convolution_3x3_dilation_stridex2_stridey2_bifrost_f16(__global uchar *src_addr, const int stride_x_bytes, const int stride_y_bytes,
                                                                     const int y_offset, __global uchar *weights_addr, const int weights_stride_y)
 {
     // Load the weights
     half3 weights_row0 = vload3(0, (__global half *)(weights_addr + 0 * weights_stride_y));
     half3 weights_row1 = vload3(0, (__global half *)(weights_addr + 1 * weights_stride_y));
     half3 weights_row2 = vload3(0, (__global half *)(weights_addr + 2 * weights_stride_y));

     half4 pixels0 = 0.0f;

     half8 src00_left  = vload8(0, (__global half *)ptr_offset(src_addr, 0, y_offset, stride_x_bytes, stride_y_bytes)); // Row0
     half8 src00_mid   = vload8(0, (__global half *)ptr_offset(src_addr, DILATION_X, y_offset, stride_x_bytes, stride_y_bytes));
     half8 src00_right = vload8(0, (__global half *)ptr_offset(src_addr, 2 * DILATION_X, y_offset, stride_x_bytes, stride_y_bytes));

     half8 src10_left  = vload8(0, (__global half *)ptr_offset(src_addr, 0, y_offset + DILATION_Y, stride_x_bytes, stride_y_bytes)); // Row1
     half8 src10_mid   = vload8(0, (__global half *)ptr_offset(src_addr, DILATION_X, y_offset + DILATION_Y, stride_x_bytes, stride_y_bytes));
     half8 src10_right = vload8(0, (__global half *)ptr_offset(src_addr, 2 * DILATION_X, y_offset + DILATION_Y, stride_x_bytes, stride_y_bytes));

     half8 src20_left  = vload8(0, (__global half *)ptr_offset(src_addr, 0, y_offset + DILATION_Y * 2, stride_x_bytes, stride_y_bytes)); // Row2
     half8 src20_mid   = vload8(0, (__global half *)ptr_offset(src_addr, DILATION_X, y_offset + DILATION_Y * 2, stride_x_bytes, stride_y_bytes));
     half8 src20_right = vload8(0, (__global half *)ptr_offset(src_addr, 2 * DILATION_X, y_offset + DILATION_Y * 2, stride_x_bytes, stride_y_bytes));

     CONVOLUTION1x3_BIFROST4X1_STRIDE2(pixels0, src00_left, src00_mid, src00_right, weights_row0);
     CONVOLUTION1x3_BIFROST4X1_STRIDE2(pixels0, src10_left, src10_mid, src10_right, weights_row1);
     CONVOLUTION1x3_BIFROST4X1_STRIDE2(pixels0, src20_left, src20_mid, src20_right, weights_row2);

     return pixels0;
 }

 #endif // (DILATION_X > 1 && DILATION_Y > 1)

 /** Compute a 1D horizontal convolution of size 3 and stride 1 for 16bit floating point type.
  *
  * @param[in] left_pixel   Pointer to the left pixel.
  * @param[in] left_coeff   Weight of the left pixel
  * @param[in] middle_coeff Weight of the middle pixel
  * @param[in] right_coeff  Weight of the right pixel
  *
  * @return a half4 containing 4 convoluted values.
  */
 inline half4 convolution1x3_stride_1_f16(__global const uchar *left_pixel,
                                          const half            left_coeff,
                                          const half            middle_coeff,
                                          const half            right_coeff)
 {
 #if(DILATION_X == 1 && DILATION_Y == 1)

     half8 temp = vload8(0, (__global half *)left_pixel);

     half4 left   = CONVERT(temp.s0123, half4);
     half4 middle = CONVERT(temp.s1234, half4);
     half4 right  = CONVERT(temp.s2345, half4);

     return left * (half4)left_coeff + middle * (half4)middle_coeff + right * (half4)right_coeff;
 #else /* DILATION_X==1 && DILATION_Y==1 */
     return vload4(0, (__global half *)left_pixel) * (half4)left_coeff
            + vload4(0, (__global half *)(left_pixel) + DILATION_X) * (half4)middle_coeff
            + vload4(0, (__global half *)(left_pixel) + 2 * DILATION_X) * (half4)right_coeff;

 #endif /* DILATION_X==1 && DILATION_Y==1 */
 }

 /** Compute a 1D horizontal convolution of size 3 and stride 2 for 16bit floating point type.
  *
  * @param[in] left_pixel   Pointer to the left pixel.
  * @param[in] left_coeff   Weight of the left pixel
  * @param[in] middle_coeff Weight of the middle pixel
  * @param[in] right_coeff  Weight of the right pixel
  *
  * @return a half4 containing 4 convoluted values.
  */
 inline half4 convolution1x3_stride_2_f16(__global const uchar *left_pixel,
                                          const half            left_coeff,
                                          const half            middle_coeff,
                                          const half            right_coeff)
 {
 #if(DILATION_X == 1 && DILATION_Y == 1)

     half8 temp0 = vload8(0, (__global half *)left_pixel);
     half temp1  = *((__global half *)(left_pixel + 8 * sizeof(half)));

     half4 left   = CONVERT(temp0.s0246, half4);
     half4 middle = CONVERT(temp0.s1357, half4);
     half4 right  = CONVERT((half4)(temp0.s246, temp1), half4);

     return left * (half4)left_coeff + middle * (half4)middle_coeff + right * (half4)right_coeff;
 #else /* DILATION_X==1 && DILATION_Y==1 */

     __global half *left_pixel_float = (__global half *)left_pixel;

     return (half4)(*left_pixel_float, *(left_pixel_float + 2), *(left_pixel_float + 4), *(left_pixel_float + 6)) * (half4)left_coeff
            + (half4)(*(left_pixel_float + DILATION_X), *(left_pixel_float + DILATION_X + 2), *(left_pixel_float + DILATION_X + 4), *(left_pixel_float + DILATION_X + 6)) * (half4)middle_coeff
            + (half4)(*(left_pixel_float + DILATION_X * 2), *(left_pixel_float + DILATION_X * 2 + 2), *(left_pixel_float + DILATION_X * 2 + 4), *(left_pixel_float + DILATION_X * 2 + 6)) * (half4)right_coeff;

 #endif /* DILATION_X==1 && DILATION_Y==1 */
 }

 /** Compute a 1D horizontal convolution of size 3 and stride 3 for 16bit floating point type.
  *
  * @param[in] left_pixel   Pointer to the left pixel.
  * @param[in] left_coeff   Weight of the left pixel
  * @param[in] middle_coeff Weight of the middle pixel
  * @param[in] right_coeff  Weight of the right pixel
  *
  * @return a half4 containing 4 convoluted values.
  */
 inline half4 convolution1x3_stride_3_f16(__global const uchar *left_pixel,
                                          const half            left_coeff,
                                          const half            middle_coeff,
                                          const half            right_coeff)
 {
 #if(DILATION_X == 1 && DILATION_Y == 1)

     half16 temp0 = vload16(0, (__global half *)left_pixel);

     half4 left   = CONVERT(temp0.s0369, half4);
     half4 middle = CONVERT(temp0.s147A, half4);
     half4 right  = CONVERT(temp0.s258B, half4);

     return left * (half4)left_coeff + middle * (half4)middle_coeff + right * (half4)right_coeff;
 #else /* DILATION_X==1 && DILATION_Y==1 */

     __global half *left_pixel_float = (__global half *)left_pixel;

     return (half4)(*left_pixel_float, *(left_pixel_float + 3), *(left_pixel_float + 6), *(left_pixel_float + 9)) * (half4)left_coeff
            + (half4)(*(left_pixel_float + DILATION_X), *(left_pixel_float + DILATION_X + 3), *(left_pixel_float + DILATION_X + 6), *(left_pixel_float + DILATION_X + 9)) * (half4)middle_coeff
            + (half4)(*(left_pixel_float + DILATION_X * 2), *(left_pixel_float + DILATION_X * 2 + 3), *(left_pixel_float + DILATION_X * 2 + 6), *(left_pixel_float + DILATION_X * 2 + 9)) * (half4)right_coeff;

 #endif /* DILATION_X==1 && DILATION_Y==1 */
 }

 /** Apply a 3x3 convolution matrix to a single channel F16 input image and return the result.
  *
  * Convolution matrix layout:
  *
  * [ mat0, mat1, mat2 ]\n
  * [ mat3, mat4, mat5 ]\n
  * [ mat6, mat7, mat8 ]\n
  *
  * @param[in] src  A pointer to source Image structure
  * @param[in] mat0 Coefficient from the convolution matrix
  * @param[in] mat1 Coefficient from the convolution matrix
  * @param[in] mat2 Coefficient from the convolution matrix
  * @param[in] mat3 Coefficient from the convolution matrix
  * @param[in] mat4 Coefficient from the convolution matrix
  * @param[in] mat5 Coefficient from the convolution matrix
  * @param[in] mat6 Coefficient from the convolution matrix
  * @param[in] mat0 Coefficient from the convolution matrix
  * @param[in] mat7 Coefficient from the convolution matrix
  * @param[in] mat8 Coefficient from the convolution matrix
  *
  * @return a half4 containing 4 convoluted values.
  */
 inline half4 convolution3x3_f16(
     Image     *src,
     const half mat0, const half mat1, const half mat2,
     const half mat3, const half mat4, const half mat5,
     const half mat6, const half mat7, const half mat8)
 {
     half4 pixels;

     pixels = convolution1x3_f16(offset(src, 0, 0), mat0, mat1, mat2);
     pixels += convolution1x3_f16(offset(src, 0, DILATION_Y), mat3, mat4, mat5);
     pixels += convolution1x3_f16(offset(src, 0, DILATION_Y * 2), mat6, mat7, mat8);

     return pixels;
 }

 #if defined(DEPTH_MULTIPLIER)

 /** This OpenCL kernel computes the depthwise convolution 3x3
  *
  * @note It is possible to select the activation function to apply using -DACTIVATION_TYPE e.g. -DACTIVATION_TYPE=relu
  * @note If activation function is enabled, the data type must be passed at compile time using -DDATA_TYPE e.g. -DDATA_TYPE=half. Supported data types: half.
  * @note A, B variables required by some activation functions are set using -DA_VAL= and -DB_VAL= respectively
  * @note Vector size should be given as a preprocessor argument using -DVEC_SIZE=size
  *
  * @param[in] src_ptr                               Pointer to the source tensor. Supported data types: F16
  * @param[in] src_stride_x                          Stride of the source tensor in X dimension (in bytes)
  * @param[in] src_step_x                            src_stride_x * number of elements along X processed per workitem(in bytes)
  * @param[in] src_stride_y                          Stride of the source tensor in Y dimension (in bytes)
  * @param[in] src_step_y                            src_stride_y * number of elements along Y processed per workitem(in bytes)
  * @param[in] src_stride_z                          Stride of the source tensor in Z dimension (in bytes)
  * @param[in] src_step_z                            src_stride_z * number of elements along Y processed per workitem(in bytes)
  * @param[in] src_offset_first_element_in_bytes     The offset of the first element in the source tensor
  * @param[in] dst_ptr                               Pointer to the destination tensor. Supported data types: same as @p src_ptr
  * @param[in] dst_stride_x                          Stride of the destination tensor in X dimension (in bytes)
  * @param[in] dst_step_x                            dst_stride_x * number of elements along X processed per workitem(in bytes)
  * @param[in] dst_stride_y                          Stride of the destination tensor in Y dimension (in bytes)
  * @param[in] dst_step_y                            dst_stride_y * number of elements along Y processed per workitem(in bytes)
  * @param[in] dst_stride_z                          Stride of the destination tensor in Z dimension (in bytes)
  * @param[in] dst_step_z                            dst_stride_z * number of elements along Y processed per workitem(in bytes)
  * @param[in] dst_offset_first_element_in_bytes     The offset of the first element in the destination tensor
  * @param[in] weights_ptr                           Pointer to the weights tensor. Supported data types: same as @p src_ptr
  * @param[in] weights_stride_x                      Stride of the weights tensor in X dimension (in bytes)
  * @param[in] weights_step_x                        weights_stride_x * number of elements along X processed per workitem(in bytes)
  * @param[in] weights_stride_y                      Stride of the weights tensor in Y dimension (in bytes)
  * @param[in] weights_step_y                        weights_stride_y * number of elements along Y processed per workitem(in bytes)
  * @param[in] weights_stride_z                      Stride of the weights tensor in Z dimension (in bytes)
  * @param[in] weights_step_z                        weights_stride_z * number of elements along Y processed per workitem(in bytes)
  * @param[in] weights_offset_first_element_in_bytes The offset of the first element in the biases vector
  * @param[in] biases_ptr                            (Optional) Pointer to the biases vector. Supported data types: F16
  * @param[in] biases_stride_x                       (Optional) Stride of the biases vector in X dimension (in bytes)
  * @param[in] biases_step_x                         (Optional) biases_stride_x * number of elements along X processed per workitem(in bytes)
  * @param[in] biases_offset_first_element_in_bytes  (Optional) The offset of the first element in the biases vector
  */
 __kernel void depthwise_convolution_3x3_f16(
     TENSOR3D_DECLARATION(src),
     TENSOR3D_DECLARATION(dst),
     TENSOR3D_DECLARATION(weights)
 #if defined(HAS_BIAS)
     ,
     VECTOR_DECLARATION(biases)
 #endif //defined(HAS_BIAS)
 )
 {
     Image    src     = CONVERT_TENSOR3D_TO_IMAGE_STRUCT(src);
     Image    dst     = CONVERT_TENSOR3D_TO_IMAGE_STRUCT(dst);
     Tensor3D weights = CONVERT_TO_TENSOR3D_STRUCT_NO_STEP(weights);
 #if defined(HAS_BIAS)
     Vector biases = CONVERT_TO_VECTOR_STRUCT_NO_STEP(biases);
 #endif //defined(HAS_BIAS)

     // Extract channel and linearized batch indices
     const int channel = get_global_id(2) % DST_CHANNELS;
     const int batch   = get_global_id(2) / DST_CHANNELS;
     // Load relevant input and weights data (Accounts depth multiplier when indexing input, OFM = IFM * DEPTH_MULTIPLIER)
     src.ptr -= batch * (DST_CHANNELS / DEPTH_MULTIPLIER) * (DEPTH_MULTIPLIER - 1) * src_step_z + (channel - (channel / DEPTH_MULTIPLIER)) * src_step_z;
     __global uchar *weights_addr = weights.ptr + get_global_id(0) * weights_step_x + get_global_id(1) * weights_step_y + channel * weights_step_z;

     uchar3 offset         = (uchar3)(0, 1, 2) * (uchar3)weights_stride_y;
     half3 weights_values0 = vload3(0, (__global half *)(weights_addr + offset.s0));
     half3 weights_values1 = vload3(0, (__global half *)(weights_addr + offset.s1));
     half3 weights_values2 = vload3(0, (__global half *)(weights_addr + offset.s2));

     half4 pixels = convolution3x3_f16(&src, weights_values0.s0, weights_values0.s1, weights_values0.s2,
                                       weights_values1.s0, weights_values1.s1, weights_values1.s2,
                                       weights_values2.s0, weights_values2.s1, weights_values2.s2);
 #if defined(HAS_BIAS)
     pixels += (half4)(*((__global half *)(biases.ptr + channel * biases_stride_x)));
 #endif //defined(HAS_BIAS)

     vstore4(ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, pixels, A_VAL, B_VAL), 0, (__global half *)dst.ptr);
 }
 #endif // defined(DEPTH_MULTIPLIER)
 #endif // defined(CONV_STRIDE_X)

 /** This OpenCL kernel is optimized for Bifrost architectures and computes the 16bit floating point depthwise convolution 3x3
  * when both stride_x and stride_y are equal to 1
  *
  * @note It is possible to select the activation function to apply using -DACTIVATION_TYPE e.g. -DACTIVATION_TYPE=relu
  * @note If activation function is enabled, the data type must be passed at compile time using -DDATA_TYPE e.g. -DDATA_TYPE=half. Supported data types: half.
  * @note A, B variables required by some activation functions are set using -DA_VAL= and -DB_VAL= respectively
  * @note Vector size should be given as a preprocessor argument using -DVEC_SIZE=size
  *
  * @param[in] src_ptr                               Pointer to the source tensor. Supported data types: F16
  * @param[in] src_stride_x                          Stride of the source tensor in X dimension (in bytes)
  * @param[in] src_step_x                            src_stride_x * number of elements along X processed per workitem(in bytes)
  * @param[in] src_stride_y                          Stride of the source tensor in Y dimension (in bytes)
  * @param[in] src_step_y                            src_stride_y * number of elements along Y processed per workitem(in bytes)
  * @param[in] src_stride_z                          Stride of the source tensor in Z dimension (in bytes)
  * @param[in] src_step_z                            src_stride_z * number of elements along Y processed per workitem(in bytes)
  * @param[in] src_offset_first_element_in_bytes     The offset of the first element in the source tensor
  * @param[in] dst_ptr                               Pointer to the destination tensor. Supported data types: same as @p src_ptr
  * @param[in] dst_stride_x                          Stride of the destination tensor in X dimension (in bytes)
  * @param[in] dst_step_x                            dst_stride_x * number of elements along X processed per workitem(in bytes)
  * @param[in] dst_stride_y                          Stride of the destination tensor in Y dimension (in bytes)
  * @param[in] dst_step_y                            dst_stride_y * number of elements along Y processed per workitem(in bytes)
  * @param[in] dst_stride_z                          Stride of the destination tensor in Z dimension (in bytes)
  * @param[in] dst_step_z                            dst_stride_z * number of elements along Y processed per workitem(in bytes)
  * @param[in] dst_offset_first_element_in_bytes     The offset of the first element in the destination tensor
  * @param[in] weights_ptr                           Pointer to the weights tensor. Supported data types: same as @p src_ptr
  * @param[in] weights_stride_x                      Stride of the weights tensor in X dimension (in bytes)
  * @param[in] weights_step_x                        weights_stride_x * number of elements along X processed per workitem(in bytes)
  * @param[in] weights_stride_y                      Stride of the weights tensor in Y dimension (in bytes)
  * @param[in] weights_step_y                        weights_stride_y * number of elements along Y processed per workitem(in bytes)
  * @param[in] weights_stride_z                      Stride of the weights tensor in Z dimension (in bytes)
  * @param[in] weights_step_z                        weights_stride_z * number of elements along Y processed per workitem(in bytes)
  * @param[in] weights_offset_first_element_in_bytes The offset of the first element in the biases vector
  * @param[in] biases_ptr                            (Optional) Pointer to the biases vector. Supported data types: same as @p src_ptr
  * @param[in] biases_stride_x                       (Optional) Stride of the biases vector in X dimension (in bytes)
  * @param[in] biases_step_x                         (Optional) biases_stride_x * number of elements along X processed per workitem(in bytes)
  * @param[in] biases_offset_first_element_in_bytes  (Optional) The offset of the first element in the biases vector
  */
 __kernel void depthwise_convolution_3x3_stridex1_stridey1_bifrost_f16(
     TENSOR3D_DECLARATION(src),
     TENSOR3D_DECLARATION(dst),
     TENSOR3D_DECLARATION(weights)
 #if defined(HAS_BIAS)
     ,
     VECTOR_DECLARATION(biases)
 #endif //defined(HAS_BIAS)
 )
 {
     Image    src     = CONVERT_TENSOR3D_TO_IMAGE_STRUCT(src);
     Image    dst     = CONVERT_TENSOR3D_TO_IMAGE_STRUCT(dst);
     Tensor3D weights = CONVERT_TO_TENSOR3D_STRUCT_NO_STEP(weights);

     // Extract channel and linearized batch indices
     const int channel = get_global_id(2) % DST_CHANNELS;
     const int batch   = get_global_id(2) / DST_CHANNELS;

 #ifdef HAS_BIAS
     Vector biases = CONVERT_TO_VECTOR_STRUCT_NO_STEP(biases);

     half bias = *((__global half *)(vector_offset(&biases, channel)));
 #endif /* defined(HAS_BIAS) */

     half4 pixels0 = 0.0f;
     half4 pixels1 = 0.0f;
     half4 pixels2 = 0.0f;
     half4 pixels3 = 0.0f;

     // Load relevant input and weights data (Accounts depth multiplier when indexing input, OFM = IFM * DEPTH_MULTIPLIER)
     __global uchar *weights_addr = weights.ptr + get_global_id(0) * weights_step_x + get_global_id(1) * weights_step_y + channel * weights_step_z;
     __global uchar *src_addr     = src.ptr - batch * (DST_CHANNELS / DEPTH_MULTIPLIER) * (DEPTH_MULTIPLIER - 1) * src_step_z - (channel - (channel / DEPTH_MULTIPLIER)) * src_step_z;

 #if(DILATION_X == 1 && DILATION_Y == 1)
     // Load the weights
     half3 weights_row0 = vload3(0, (__global half *)(weights_addr + 0 * weights_stride_y));
     half3 weights_row1 = vload3(0, (__global half *)(weights_addr + 1 * weights_stride_y));
     half3 weights_row2 = vload3(0, (__global half *)(weights_addr + 2 * weights_stride_y));

     // Note: Since each work-item computes 4x4 elements, we need to load 6 rows from the input tensor
     half8 src00 = vload8(0, (__global half *)(src_addr + 0 * src_stride_y)); // Row0
     half8 src10 = vload8(0, (__global half *)(src_addr + 1 * src_stride_y)); // Row1
     half8 src20 = vload8(0, (__global half *)(src_addr + 2 * src_stride_y)); // Row2
     half8 src30 = vload8(0, (__global half *)(src_addr + 3 * src_stride_y)); // Row3
     half8 src40 = vload8(0, (__global half *)(src_addr + 4 * src_stride_y)); // Row4
     half8 src50 = vload8(0, (__global half *)(src_addr + 5 * src_stride_y)); // Row5

     CONVOLUTION1x3_BIFROST4X1_STRIDE1(pixels0, src00, weights_row0);
     CONVOLUTION1x3_BIFROST4X1_STRIDE1(pixels0, src10, weights_row1);
     CONVOLUTION1x3_BIFROST4X1_STRIDE1(pixels0, src20, weights_row2);
     CONVOLUTION1x3_BIFROST4X1_STRIDE1(pixels1, src10, weights_row0);
     CONVOLUTION1x3_BIFROST4X1_STRIDE1(pixels1, src20, weights_row1);
     CONVOLUTION1x3_BIFROST4X1_STRIDE1(pixels1, src30, weights_row2);
     CONVOLUTION1x3_BIFROST4X1_STRIDE1(pixels2, src20, weights_row0);
     CONVOLUTION1x3_BIFROST4X1_STRIDE1(pixels2, src30, weights_row1);
     CONVOLUTION1x3_BIFROST4X1_STRIDE1(pixels2, src40, weights_row2);
     CONVOLUTION1x3_BIFROST4X1_STRIDE1(pixels3, src30, weights_row0);
     CONVOLUTION1x3_BIFROST4X1_STRIDE1(pixels3, src40, weights_row1);
     CONVOLUTION1x3_BIFROST4X1_STRIDE1(pixels3, src50, weights_row2);

 #else /* DILATION_X==1 && DILATION_Y==1 */

     //3x3 Convolution of elements starting in 0th row
     pixels0 = convolution_3x3_dilation_stridex1_stridey1_bifrost_f16(src_addr, src.stride_x, src.stride_y, 0, weights_addr, weights_stride_y);
     //3x3 Convolution of elements starting in 1st row
     pixels1 = convolution_3x3_dilation_stridex1_stridey1_bifrost_f16(src_addr, src.stride_x, src.stride_y, 1, weights_addr, weights_stride_y);
     //3x3 Convolution of elements starting in 2nd row
     pixels2 = convolution_3x3_dilation_stridex1_stridey1_bifrost_f16(src_addr, src.stride_x, src.stride_y, 2, weights_addr, weights_stride_y);
     //3x3 Convolution of elements starting in 3rd row
     pixels3 = convolution_3x3_dilation_stridex1_stridey1_bifrost_f16(src_addr, src.stride_x, src.stride_y, 3, weights_addr, weights_stride_y);

 #endif /* DILATION_X==1 && DILATION_Y==1 */

 #ifdef HAS_BIAS
     pixels0 += (half4)bias;
     pixels1 += (half4)bias;
     pixels2 += (half4)bias;
     pixels3 += (half4)bias;
 #endif /* defined(HAS_BIAS) */

     vstore4(ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, pixels0, A_VAL, B_VAL), 0, (__global half *)(dst.ptr + 0 * dst_stride_y));
     vstore4(ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, pixels1, A_VAL, B_VAL), 0, (__global half *)(dst.ptr + 1 * dst_stride_y));
     vstore4(ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, pixels2, A_VAL, B_VAL), 0, (__global half *)(dst.ptr + 2 * dst_stride_y));
     vstore4(ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, pixels3, A_VAL, B_VAL), 0, (__global half *)(dst.ptr + 3 * dst_stride_y));
 }

 /** This OpenCL kernel is optimized for Bifrost architectures and computes 16bit floating point the depthwise convolution 3x3
  * when both stride_x and stride_y are equal to 2
  *
  * @note It is possible to select the activation function to apply using -DACTIVATION_TYPE e.g. -DACTIVATION_TYPE=relu
  * @note If activation function is enabled, the data type must be passed at compile time using -DDATA_TYPE e.g. -DDATA_TYPE=half. Supported data types: half.
  * @note A, B variables required by some activation functions are set using -DA_VAL= and -DB_VAL= respectively
  * @note Vector size should be given as a preprocessor argument using -DVEC_SIZE=size
  *
  * @param[in] src_ptr                               Pointer to the source tensor. Supported data types: F16
  * @param[in] src_stride_x                          Stride of the source tensor in X dimension (in bytes)
  * @param[in] src_step_x                            src_stride_x * number of elements along X processed per workitem(in bytes)
  * @param[in] src_stride_y                          Stride of the source tensor in Y dimension (in bytes)
  * @param[in] src_step_y                            src_stride_y * number of elements along Y processed per workitem(in bytes)
  * @param[in] src_stride_z                          Stride of the source tensor in Z dimension (in bytes)
  * @param[in] src_step_z                            src_stride_y * number of elements along Z processed per workitem(in bytes)
  * @param[in] src_offset_first_element_in_bytes     The offset of the first element in the source tensor
  * @param[in] dst_ptr                               Pointer to the destination tensor. Supported data types: same as @p src_ptr
  * @param[in] dst_stride_x                          Stride of the destination tensor in X dimension (in bytes)
  * @param[in] dst_step_x                            dst_stride_x * number of elements along X processed per workitem(in bytes)
  * @param[in] dst_stride_y                          Stride of the destination tensor in Y dimension (in bytes)
  * @param[in] dst_step_y                            dst_stride_y * number of elements along Y processed per workitem(in bytes)
  * @param[in] dst_stride_z                          Stride of the destination tensor in Z dimension (in bytes)
  * @param[in] dst_step_z                            dst_stride_z * number of elements along Y processed per workitem(in bytes)
  * @param[in] dst_offset_first_element_in_bytes     The offset of the first element in the destination tensor
  * @param[in] weights_ptr                           Pointer to the weights tensor. Supported data types: same as @p src_ptr
  * @param[in] weights_stride_x                      Stride of the weights tensor in X dimension (in bytes)
  * @param[in] weights_step_x                        weights_stride_x * number of elements along X processed per workitem(in bytes)
  * @param[in] weights_stride_y                      Stride of the weights tensor in Y dimension (in bytes)
  * @param[in] weights_step_y                        weights_stride_y * number of elements along Y processed per workitem(in bytes)
  * @param[in] weights_stride_z                      Stride of the weights tensor in Z dimension (in bytes)
  * @param[in] weights_step_z                        weights_stride_z * number of elements along Y processed per workitem(in bytes)
  * @param[in] weights_offset_first_element_in_bytes The offset of the first element in the biases vector
  * @param[in] biases_ptr                            (Optional) Pointer to the biases vector. Supported data types: same as @p src_ptr
  * @param[in] biases_stride_x                       (Optional) Stride of the biases vector in X dimension (in bytes)
  * @param[in] biases_step_x                         (Optional) biases_stride_x * number of elements along X processed per workitem(in bytes)
  * @param[in] biases_offset_first_element_in_bytes  (Optional) The offset of the first element in the biases vector
  */
 __kernel void depthwise_convolution_3x3_stridex2_stridey2_bifrost_f16(
     TENSOR3D_DECLARATION(src),
     TENSOR3D_DECLARATION(dst),
     TENSOR3D_DECLARATION(weights)
 #if defined(HAS_BIAS)
     ,
     VECTOR_DECLARATION(biases)
 #endif //defined(HAS_BIAS)
 )
 {
     Image    src     = CONVERT_TENSOR3D_TO_IMAGE_STRUCT(src);
     Image    dst     = CONVERT_TENSOR3D_TO_IMAGE_STRUCT(dst);
     Tensor3D weights = CONVERT_TO_TENSOR3D_STRUCT_NO_STEP(weights);

     // Extract channel and linearized batch indices
     const int channel = get_global_id(2) % DST_CHANNELS;
     const int batch   = get_global_id(2) / DST_CHANNELS;

 #ifdef HAS_BIAS
     Vector biases = CONVERT_TO_VECTOR_STRUCT_NO_STEP(biases);

     half bias = *((__global half *)(vector_offset(&biases, channel)));
 #endif /* defined(HAS_BIAS) */

     half4 pixels0 = 0.0f;
     half4 pixels1 = 0.0f;

     // Load relevant input and weights data ( Accounts depth multiplier when indexing input, OFM = IFM * DEPTH_MULTIPLIER)
     __global uchar *weights_addr = weights.ptr + get_global_id(0) * weights_step_x + get_global_id(1) * weights_step_y + channel * weights_step_z;
     __global uchar *src_addr     = src.ptr - batch * (DST_CHANNELS / DEPTH_MULTIPLIER) * (DEPTH_MULTIPLIER - 1) * src_step_z - (channel - (channel / DEPTH_MULTIPLIER)) * src_step_z;

 #if(DILATION_X == 1 && DILATION_Y == 1)

     // Load the weights
     half3 weights_row0 = vload3(0, (__global half *)(weights_addr + 0 * weights_stride_y));
     half3 weights_row1 = vload3(0, (__global half *)(weights_addr + 1 * weights_stride_y));
     half3 weights_row2 = vload3(0, (__global half *)(weights_addr + 2 * weights_stride_y));

     // Note: Since each work-item computes 2x4 elements, we need to load 5 rows from the input tensor
     half8 src00 = vload8(0, (__global half *)(src_addr + 0 * src_stride_y)); // Row0
     half2 src01 = vload2(4, (__global half *)(src_addr + 0 * src_stride_y)); // Row0
     half8 src10 = vload8(0, (__global half *)(src_addr + 1 * src_stride_y)); // Row1
     half2 src11 = vload2(4, (__global half *)(src_addr + 1 * src_stride_y)); // Row1
     half8 src20 = vload8(0, (__global half *)(src_addr + 2 * src_stride_y)); // Row2
     half2 src21 = vload2(4, (__global half *)(src_addr + 2 * src_stride_y)); // Row2
     half8 src30 = vload8(0, (__global half *)(src_addr + 3 * src_stride_y)); // Row3
     half2 src31 = vload2(4, (__global half *)(src_addr + 3 * src_stride_y)); // Row3
     half8 src40 = vload8(0, (__global half *)(src_addr + 4 * src_stride_y)); // Row4
     half2 src41 = vload2(4, (__global half *)(src_addr + 4 * src_stride_y)); // Row4

     CONVOLUTION1x3_BIFROST4X1_STRIDE2(pixels0, src00, src01, weights_row0);
     CONVOLUTION1x3_BIFROST4X1_STRIDE2(pixels0, src10, src11, weights_row1);
     CONVOLUTION1x3_BIFROST4X1_STRIDE2(pixels0, src20, src21, weights_row2);
     CONVOLUTION1x3_BIFROST4X1_STRIDE2(pixels1, src20, src21, weights_row0);
     CONVOLUTION1x3_BIFROST4X1_STRIDE2(pixels1, src30, src31, weights_row1);
     CONVOLUTION1x3_BIFROST4X1_STRIDE2(pixels1, src40, src41, weights_row2);

 #else  /* DILATION_X==1 && DILATION_Y==1 */
     //3x3 Convolution of elements starting in 0th row
     pixels0 = convolution_3x3_dilation_stridex2_stridey2_bifrost_f16(src_addr, src.stride_x, src.stride_y, 0, weights_addr, weights_stride_y);
     //3x3 Convolution of elements starting in 2nd row
     pixels1                  = convolution_3x3_dilation_stridex2_stridey2_bifrost_f16(src_addr, src.stride_x, src.stride_y, 2, weights_addr, weights_stride_y);
 #endif /* DILATION_X==1 && DILATION_Y==1 */

 #ifdef HAS_BIAS
     pixels0 += (half4)bias;
     pixels1 += (half4)bias;
 #endif /* defined(HAS_BIAS) */

     vstore4(ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, pixels0, A_VAL, B_VAL), 0, (__global half *)(dst.ptr + 0 * dst_stride_y));
     vstore4(ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, pixels1, A_VAL, B_VAL), 0, (__global half *)(dst.ptr + 1 * dst_stride_y));
 }
 #endif // defined(ARM_COMPUTE_OPENCL_FP16_ENABLED) && defined(DEPTH_MULTIPLIER) && defined(DST_CHANNELS) && defined(IS_F16)

 #if defined(SRC_DIM1) && defined(SRC_DIM2) && defined(KERNEL_WIDTH) && defined(KERNEL_HEIGHT) && defined(N0) && defined(DATA_TYPE) && defined(DILATION_X) && defined(DILATION_Y) && defined(CONV_STRIDE_X) && defined(CONV_STRIDE_Y) && defined(CONV_PAD_LEFT) && defined(CONV_PAD_TOP) && defined(VEC_SIZE_LEFTOVER)
 /** This function computes the depthwise convolution for NHWC data layout. This kernel assumes that the weights tensor is NOT reshaped
  *
  * @note Datatype should be given as a preprocessor argument using -DDATA_TYPE=type. e.g. -DDATA_TYPE=float
  * @note The number of elements processed must be passed at compile time using -DN0 (e.g. -DN0=2)
  * @note The depth multiplier must be passed at compile time using -DDEPTH_MULTIPLIER (e.g. -DDEPTH_MULTIPLIER=1)
  * @note The first dimension of the input tensor must be passed at compile time using -DSRC_DIM1 (e.g. -DSRC_DIM1=112)
  * @note The second dimension of the input tensor must be passed at compile time using -DSRC_DIM2 (e.g. -DSRC_DIM2=80)
  * @note The kernel width must be passed at compile time using -DKERNEL_WIDTH (e.g. -DKERNEL_WIDTH=5)
  * @note The kernel height must be passed at compile time using -DKERNEL_HEIGHT (e.g. -DKERNEL_HEIGHT=5)
  * @note The convolution pad top must be passed at compile time using -DCONV_PAD_TOP (e.g. -DCONV_PAD_TOP=1)
  * @note The convolution pad top must be passed at compile time using -DCONV_PAD_LEFT (e.g. -DCONV_PAD_LEFT=1)
  * @note The convolution stride along the width must be passed at compile time using -DCONV_STRIDE_X (e.g. -DCONV_STRIDE_Y=X)
  * @note The convolution stride along the height must be passed at compile time using -DCONV_STRIDE_Y (e.g. -DCONV_STRIDE_Y=1)
  * @note Leftover vector size has to be passed at compile time using -DVEC_SIZE_LEFTOVER. e.g. -DVEC_SIZE=3. It is defined as the remainder between the input's first dimension and VEC_SIZE
  * @note It is possible to select the activation function to apply using -DACTIVATION_TYPE e.g. -DACTIVATION_TYPE=relu
  * @note A, B variables required by some activation functions are set using -DA_VAL= and -DB_VAL= respectively
  *
  * @param[in] src_ptr                               Pointer to the source tensor. Supported data types: F16/F32
  * @param[in] src_stride_x                          Stride of the source tensor in X dimension (in bytes)
  * @param[in] src_step_x                            src_stride_x * number of elements along X processed per workitem(in bytes)
  * @param[in] src_stride_y                          Stride of the source tensor in Y dimension (in bytes)
  * @param[in] src_step_y                            src_stride_y * number of elements along Y processed per workitem(in bytes)
  * @param[in] src_stride_z                          Stride of the source tensor in Z dimension (in bytes)
  * @param[in] src_step_z                            src_stride_y * number of elements along Z processed per workitem(in bytes)
  * @param[in] src_stride_w                          Stride of the source tensor in W dimension (in bytes)
  * @param[in] src_step_w                            src_stride_w * number of elements along W processed per workitem(in bytes)
  * @param[in] src_offset_first_element_in_bytes     The offset of the first element in the source tensor
  * @param[in] dst_ptr                               Pointer to the destination tensor. Supported data types: same as src_ptr
  * @param[in] dst_stride_x                          Stride of the destination tensor in X dimension (in bytes)
  * @param[in] dst_step_x                            dst_stride_x * number of elements along X processed per workitem(in bytes)
  * @param[in] dst_stride_y                          Stride of the destination tensor in Y dimension (in bytes)
  * @param[in] dst_step_y                            dst_stride_y * number of elements along Y processed per workitem(in bytes)
  * @param[in] dst_stride_z                          Stride of the destination tensor in Z dimension (in bytes)
  * @param[in] dst_step_z                            dst_stride_z * number of elements along Y processed per workitem(in bytes)
  * @param[in] dst_stride_w                          Stride of the destination tensor in W dimension (in bytes)
  * @param[in] dst_step_w                            dst_stride_w * number of elements along W processed per workitem(in bytes)
  * @param[in] dst_offset_first_element_in_bytes     The offset of the first element in the destination tensor
  * @param[in] weights_ptr                           Pointer to the weights tensor. Supported data types: F16/F32
  * @param[in] weights_stride_x                      Stride of the weights tensor in X dimension (in bytes)
  * @param[in] weights_step_x                        weights_stride_x * number of elements along X processed per workitem(in bytes)
  * @param[in] weights_stride_y                      Stride of the weights tensor in Y dimension (in bytes)
  * @param[in] weights_step_y                        weights_stride_y * number of elements along Y processed per workitem(in bytes)
  * @param[in] weights_stride_z                      Stride of the weights tensor in Z dimension (in bytes)
  * @param[in] weights_step_z                        weights_stride_z * number of elements along Y processed per workitem(in bytes)
  * @param[in] weights_offset_first_element_in_bytes The offset of the first element in the weights tensor
  * @param[in] biases_ptr                            (Optional) Pointer to the biases vector. Supported data types: same as src_ptr
  * @param[in] biases_stride_x                       (Optional) Stride of the biases vector in X dimension (in bytes)
  * @param[in] biases_step_x                         (Optional) biases_stride_x * number of elements along X processed per workitem(in bytes)
  * @param[in] biases_offset_first_element_in_bytes  (Optional) The offset of the first element in the biases vector
  */
 __kernel void dwc_MxN_native_fp_nhwc(
     TENSOR4D_DECLARATION(src),
     TENSOR4D_DECLARATION(dst),
     TENSOR3D_DECLARATION(weights)
 #if defined(HAS_BIAS)
     ,
     VECTOR_DECLARATION(biases)
 #endif // defined(HAS_BIAS)
 )
 {
     int x_offs = max((int)(get_global_id(0) * N0 - (N0 - VEC_SIZE_LEFTOVER) % N0), 0) * sizeof(DATA_TYPE);

     int x = get_global_id(0); // channels
     int y = get_global_id(1); // spatial coordinate x
 #if defined(DST_DEPTH)
     int z = get_global_id(2) % (int)DST_DEPTH; // spatial coordinate y
     int b = get_global_id(2) / (int)DST_DEPTH; // batch
 #else                                          // defined(DST_DEPTH)
     int z                    = get_global_id(2); // spatial coordinate y
 #endif                                         // defined(DST_DEPTH)

     __global uchar *s_addr = src_ptr + src_offset_first_element_in_bytes + x_offs;

     __global uchar *d_addr = dst_ptr + dst_offset_first_element_in_bytes + x_offs * (int)DEPTH_MULTIPLIER + y * dst_stride_y + z * dst_stride_z;

     __global uchar *w_addr = weights_ptr + weights_offset_first_element_in_bytes + x_offs * (int)DEPTH_MULTIPLIER;

 #if defined(HAS_BIAS)
     __global uchar *b_addr = biases_ptr + biases_offset_first_element_in_bytes + x_offs * (int)DEPTH_MULTIPLIER;
 #endif // defined(HAS_BIAS)

 #if defined(DST_DEPTH)
     s_addr += b * src_stride_w;
     d_addr += b * dst_stride_w;
 #endif // defined(DST_DEPTH)

     for(int d = 0; d < (int)DEPTH_MULTIPLIER; ++d)
     {
         // Each work-item computes N0x1x1 elements
         VEC_DATA_TYPE(DATA_TYPE, N0)
         res0 = 0;

         int x_coord = y * CONV_STRIDE_X - (int)CONV_PAD_LEFT;
         int y_coord = z * CONV_STRIDE_Y - (int)CONV_PAD_TOP;

         for(int yk = 0; yk < KERNEL_HEIGHT; ++yk)
         {
             if(y_coord >= 0 && y_coord < SRC_DIM2)
             {
                 int x_coord_tmp = x_coord;

                 for(int xk = 0; xk < KERNEL_WIDTH; ++xk)
                 {
                     if(x_coord_tmp >= 0 && x_coord_tmp < SRC_DIM1)
                     {
                         int s_offset = x_coord_tmp * (int)src_stride_y + y_coord * (int)src_stride_z;
                         int w_offset = xk * weights_stride_y + yk * weights_stride_z;

                         // Load input and weights values
                         VEC_DATA_TYPE(DATA_TYPE, N0)
                         i = VLOAD(N0)(0, (__global DATA_TYPE *)(s_addr + s_offset));
                         VEC_DATA_TYPE(DATA_TYPE, N0)
                         w = VLOAD(N0)(0, (__global DATA_TYPE *)(w_addr + w_offset));

 #if GPU_ARCH == GPU_ARCH_MIDGARD
                         res0 += i * w;
 #else  // GPU_ARCH == GPU_ARCH_MIDGARD
                         res0 = fma(i, w, res0);
 #endif // GPU_ARCH == GPU_ARCH_MIDGARD
                     }
                     x_coord_tmp += DILATION_X;
                 }
             }
             y_coord += DILATION_Y;
         }

 #if defined(HAS_BIAS)
         res0 += VLOAD(N0)(0, (__global DATA_TYPE *)(b_addr));
 #endif // defined(HAS_BIAS)

         res0 = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, N0, res0, A_VAL, B_VAL);

         STORE_VECTOR_SELECT(res, DATA_TYPE, d_addr, N0, VEC_SIZE_LEFTOVER, VEC_SIZE_LEFTOVER != 0 && get_global_id(0) == 0)

         w_addr += sizeof(DATA_TYPE);
         d_addr += sizeof(DATA_TYPE);
 #if defined(HAS_BIAS)
         b_addr += sizeof(DATA_TYPE);
 #endif // defined(HAS_BIAS)
     }
 }
 #endif // defined(SRC_DIM1) && defined(SRC_DIM2) && defined(KERNEL_WIDTH) && defined(KERNEL_HEIGHT) && defiend(N0) && defined(DATA_TYPE) && defined(DILATION_X) && defined(DILATION_Y) && defined(CONV_STRIDE_X) && defined(CONV_STRIDE_Y) && defined(CONV_PAD_LEFT) && defined(CONV_PAD_TOP) && defined(VEC_SIZE_LEFTOVER)

 #if defined(VEC_SIZE) && defined(SRC_DIM_2) && defined(CONV_PAD_TOP) && defined(CONV_PAD_LEFT) && defined(DATA_TYPE)

 #if DATA_TYPE != float || DATA_TYPE != half
 #error "Unsupported data type"
 #endif // DATA_TYPE != float || DATA_TYPE != half

 #define VEC_FLOAT VEC_DATA_TYPE(DATA_TYPE, VEC_SIZE)

 #define FILL_ZERO_OUT_OF_BOUND_3(data_type, vec_size, basename, cond)                                                                     \
     ({                                                                                                                                    \
         basename##0 = select(basename##0, (VEC_DATA_TYPE(data_type, vec_size))0, (SELECT_VEC_DATA_TYPE(data_type, vec_size))((cond).s0)); \
         basename##1 = select(basename##1, (VEC_DATA_TYPE(data_type, vec_size))0, (SELECT_VEC_DATA_TYPE(data_type, vec_size))((cond).s1)); \
         basename##2 = select(basename##2, (VEC_DATA_TYPE(data_type, vec_size))0, (SELECT_VEC_DATA_TYPE(data_type, vec_size))((cond).s2)); \
     })

 #define FILL_ZERO_OUT_OF_BOUND_4(data_type, vec_size, basename, cond)                                                                     \
     ({                                                                                                                                    \
         FILL_ZERO_OUT_OF_BOUND_3(data_type, vec_size, basename, cond);                                                                    \
         basename##3 = select(basename##3, (VEC_DATA_TYPE(data_type, vec_size))0, (SELECT_VEC_DATA_TYPE(data_type, vec_size))((cond).s3)); \
     })

 #if defined(CONV_STRIDE_X) && defined(CONV_STRIDE_Y)

 /** This function computes the depthwise convolution for NHWC data layout when the stride along the width or height is not 1.
  *
  * @note Datatype should be given as a preprocessor argument using -DDATA_TYPE=type. e.g. -DDATA_TYPE=float
  * @note The number of elements read per thread must be passed at compile time using -DVEC_SIZE (e.g. -DVEC_SIZE=2)
  * @note Dimension two of the input tensor (height for NHWC data layout) must be passed at compile time using -DSRC_DIM2 (e.g. -DSRC_DIM_2=112)
  * @note The convolution pad top must be passed at compile time using -DCONV_PAD_TOP (e.g. -DCONV_PAD_TOP=1)
  * @note The convolution pad top must be passed at compile time using -DCONV_PAD_LEFT (e.g. -DCONV_PAD_LEFT=1)
  * @note The convolution stride along the width must be passed at compile time using -DCONV_STRIDE_X (e.g. -DCONV_STRIDE_Y=X)
  * @note The convolution stride along the height must be passed at compile time using -DCONV_STRIDE_Y (e.g. -DCONV_STRIDE_Y=1)
  * @note The dilation_x and dilation_y must be passed at compile time using -DDILATION_X and -DDILATION_Y: e.g. -DDILATION_X=1, -DDILATION_Y=1
  * @note It is possible to select the activation function to apply using -DACTIVATION_TYPE e.g. -DACTIVATION_TYPE=relu
  * @note A, B variables required by some activation functions are set using -DA_VAL= and -DB_VAL= respectively
  * @note Vector size should be given as a preprocessor argument using -DVEC_SIZE=size
  * @note The size of the partial store block in x must be passed at compile time using -DPARTIAL_STORE_N0 (e.g. -DPARTIAL_STORE_N0=1)
  * @note In case of biases, -DHAS_BIAS must to be passed at compile
  * @note If the output tensor has more than three dimensions, its third dimension must be passed at compile time using -DDST_DEPTH (e.g. -DDST_DEPTH=32)
  *
  * @param[in] src_ptr                               Pointer to the source tensor. Supported data types: F16/F32
  * @param[in] src_stride_x                          Stride of the source tensor in X dimension (in bytes)
  * @param[in] src_step_x                            src_stride_x * number of elements along X processed per workitem(in bytes)
  * @param[in] src_stride_y                          Stride of the source tensor in Y dimension (in bytes)
  * @param[in] src_step_y                            src_stride_y * number of elements along Y processed per workitem(in bytes)
  * @param[in] src_stride_z                          Stride of the source tensor in Z dimension (in bytes)
  * @param[in] src_step_z                            src_stride_y * number of elements along Z processed per workitem(in bytes)
  * @param[in] src_stride_w                          Stride of the source tensor in W dimension (in bytes)
  * @param[in] src_step_w                            src_stride_w * number of elements along W processed per workitem(in bytes)
  * @param[in] src_offset_first_element_in_bytes     The offset of the first element in the source tensor
  * @param[in] dst_ptr                               Pointer to the destination tensor. Supported data types: same as src_ptr
  * @param[in] dst_stride_x                          Stride of the destination tensor in X dimension (in bytes)
  * @param[in] dst_step_x                            dst_stride_x * number of elements along X processed per workitem(in bytes)
  * @param[in] dst_stride_y                          Stride of the destination tensor in Y dimension (in bytes)
  * @param[in] dst_step_y                            dst_stride_y * number of elements along Y processed per workitem(in bytes)
  * @param[in] dst_stride_z                          Stride of the destination tensor in Z dimension (in bytes)
  * @param[in] dst_step_z                            dst_stride_z * number of elements along Y processed per workitem(in bytes)
  * @param[in] dst_stride_w                          Stride of the destination tensor in W dimension (in bytes)
  * @param[in] dst_step_w                            dst_stride_w * number of elements along W processed per workitem(in bytes)
  * @param[in] dst_offset_first_element_in_bytes     The offset of the first element in the destination tensor
  * @param[in] weights_ptr                           Pointer to the weights tensor. Supported data types: F16/F32
  * @param[in] weights_stride_x                      Stride of the weights tensor in X dimension (in bytes)
  * @param[in] weights_step_x                        weights_stride_x * number of elements along X processed per workitem(in bytes)
  * @param[in] weights_stride_y                      Stride of the weights tensor in Y dimension (in bytes)
  * @param[in] weights_step_y                        weights_stride_y * number of elements along Y processed per workitem(in bytes)
  * @param[in] weights_stride_z                      Stride of the weights tensor in Z dimension (in bytes)
  * @param[in] weights_step_z                        weights_stride_z * number of elements along Y processed per workitem(in bytes)
  * @param[in] weights_offset_first_element_in_bytes The offset of the first element in the weights tensor
  * @param[in] max_offset                            Max offset for the input tensor
  * @param[in] biases_ptr                            (Optional) Pointer to the biases vector. Supported data types: same as src_ptr
  * @param[in] biases_stride_x                       (Optional) Stride of the biases vector in X dimension (in bytes)
  * @param[in] biases_step_x                         (Optional) biases_stride_x * number of elements along X processed per workitem(in bytes)
  * @param[in] biases_offset_first_element_in_bytes  (Optional) The offset of the first element in the biases vector
  */
 __kernel void depthwise_convolution_3x3_nhwc(
     TENSOR4D_DECLARATION(src),
     TENSOR4D_DECLARATION(dst),
     TENSOR3D_DECLARATION(weights)
 #if defined(HAS_BIAS)
     ,
     VECTOR_DECLARATION(biases)
 #endif /* defined(HAS_BIAS) */
 )
 {
     int x_offset = max((int)(get_global_id(0) * VEC_SIZE - (VEC_SIZE - PARTIAL_STORE_N0) % VEC_SIZE), 0) * sizeof(DATA_TYPE);
     int y        = get_global_id(1); // spatial coordinate x
 #if defined(DST_DEPTH)
     int z = get_global_id(2) % (int)DST_DEPTH; // spatial coordinate y
     int b = get_global_id(2) / (int)DST_DEPTH; // batch
 #else                                          // defined(DST_DEPTH)
     int      z               = get_global_id(2); // spatial coordinate y
 #endif                                         // defined(DST_DEPTH)

     __global uchar *weights_addr = weights_ptr + weights_offset_first_element_in_bytes + x_offset;

 #if defined(DST_DEPTH)
     __global uchar *src_addr = src_ptr + src_offset_first_element_in_bytes + x_offset + b * src_stride_w;
 #else  /* defined(DST_DEPTH) */
     __global uchar *src_addr = src_ptr + src_offset_first_element_in_bytes + x_offset;
 #endif /* defined(DST_DEPTH) */

     int3 src_coord_y = (int3)(y * CONV_STRIDE_X - CONV_PAD_LEFT) + (int3)(0, DILATION_X, 2 * DILATION_X);
     int3 src_coord_z = (int3)(z * CONV_STRIDE_Y - CONV_PAD_TOP) + (int3)(0, DILATION_Y, 2 * DILATION_Y);

     int3 src_offset_y = clamp(src_coord_y, (int3)0, (int3)(SRC_DIM_1 - 1));
     int3 src_offset_z = clamp(src_coord_z, (int3)0, (int3)(SRC_DIM_2 - 1));

     // Use these vectors to check whether the unclamped load would have been out of bounds
     src_coord_y = (src_offset_y != src_coord_y);
     src_coord_z = (src_offset_z != src_coord_z);

     src_offset_y *= (int3)src_stride_y;
     src_offset_z *= (int3)src_stride_z;

     // We compute VEC_SIZEx1x1 [C,W,H] elements
     VEC_FLOAT acc0 = 0;

     // Load weights
     VEC_FLOAT w0 = VLOAD(VEC_SIZE)(0, (__global DATA_TYPE *)(weights_addr + 0 * weights_stride_y + 0 * weights_stride_z));
     VEC_FLOAT w1 = VLOAD(VEC_SIZE)(0, (__global DATA_TYPE *)(weights_addr + 1 * weights_stride_y + 0 * weights_stride_z));
     VEC_FLOAT w2 = VLOAD(VEC_SIZE)(0, (__global DATA_TYPE *)(weights_addr + 2 * weights_stride_y + 0 * weights_stride_z));
     VEC_FLOAT w3 = VLOAD(VEC_SIZE)(0, (__global DATA_TYPE *)(weights_addr + 0 * weights_stride_y + 1 * weights_stride_z));
     VEC_FLOAT w4 = VLOAD(VEC_SIZE)(0, (__global DATA_TYPE *)(weights_addr + 1 * weights_stride_y + 1 * weights_stride_z));
     VEC_FLOAT w5 = VLOAD(VEC_SIZE)(0, (__global DATA_TYPE *)(weights_addr + 2 * weights_stride_y + 1 * weights_stride_z));
     VEC_FLOAT w6 = VLOAD(VEC_SIZE)(0, (__global DATA_TYPE *)(weights_addr + 0 * weights_stride_y + 2 * weights_stride_z));
     VEC_FLOAT w7 = VLOAD(VEC_SIZE)(0, (__global DATA_TYPE *)(weights_addr + 1 * weights_stride_y + 2 * weights_stride_z));
     VEC_FLOAT w8 = VLOAD(VEC_SIZE)(0, (__global DATA_TYPE *)(weights_addr + 2 * weights_stride_y + 2 * weights_stride_z));

     // Load input values
     // z == 0
     VEC_FLOAT values0 = VLOAD(VEC_SIZE)(0, (__global DATA_TYPE *)(src_addr + src_offset_z.s0 + src_offset_y.s0));
     VEC_FLOAT values1 = VLOAD(VEC_SIZE)(0, (__global DATA_TYPE *)(src_addr + src_offset_z.s0 + src_offset_y.s1));
     VEC_FLOAT values2 = VLOAD(VEC_SIZE)(0, (__global DATA_TYPE *)(src_addr + src_offset_z.s0 + src_offset_y.s2));

     FILL_ZERO_OUT_OF_BOUND_3(DATA_TYPE, VEC_SIZE, values, src_coord_y | (int3)src_coord_z.s0);

     acc0 = fma(values0, w0, acc0);
     acc0 = fma(values1, w1, acc0);
     acc0 = fma(values2, w2, acc0);

     // z == 1
     values0 = VLOAD(VEC_SIZE)(0, (__global DATA_TYPE *)(src_addr + src_offset_z.s1 + src_offset_y.s0));
     values1 = VLOAD(VEC_SIZE)(0, (__global DATA_TYPE *)(src_addr + src_offset_z.s1 + src_offset_y.s1));
     values2 = VLOAD(VEC_SIZE)(0, (__global DATA_TYPE *)(src_addr + src_offset_z.s1 + src_offset_y.s2));

     FILL_ZERO_OUT_OF_BOUND_3(DATA_TYPE, VEC_SIZE, values, src_coord_y | (int3)src_coord_z.s1);

     acc0 = fma(values0, w3, acc0);
     acc0 = fma(values1, w4, acc0);
     acc0 = fma(values2, w5, acc0);

     // z == 2
     values0 = VLOAD(VEC_SIZE)(0, (__global DATA_TYPE *)(src_addr + src_offset_z.s2 + src_offset_y.s0));
     values1 = VLOAD(VEC_SIZE)(0, (__global DATA_TYPE *)(src_addr + src_offset_z.s2 + src_offset_y.s1));
     values2 = VLOAD(VEC_SIZE)(0, (__global DATA_TYPE *)(src_addr + src_offset_z.s2 + src_offset_y.s2));

     FILL_ZERO_OUT_OF_BOUND_3(DATA_TYPE, VEC_SIZE, values, src_coord_y | (int3)src_coord_z.s2);

     acc0 = fma(values0, w6, acc0);
     acc0 = fma(values1, w7, acc0);
     acc0 = fma(values2, w8, acc0);

 #if defined(HAS_BIAS)
     __global uchar *biases_addr = biases_ptr + biases_offset_first_element_in_bytes + x_offset;
     VEC_FLOAT bias_values       = VLOAD(VEC_SIZE)(0, (__global DATA_TYPE *)biases_addr);
     acc0 += bias_values;
 #endif // defined(HAS_BIAS)

 #if defined(DST_DEPTH)
     __global uchar *dst_addr = dst_ptr + dst_offset_first_element_in_bytes + x_offset + y * dst_step_y + z * dst_step_z + b * dst_stride_w;
 #else  /* defined(DST_DEPTH) */
     __global uchar *dst_addr = dst_ptr + dst_offset_first_element_in_bytes + x_offset + y * dst_step_y + z * dst_step_z;
 #endif /* defined(DST_DEPTH) */

     acc0 = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, acc0, A_VAL, B_VAL);
     STORE_VECTOR_SELECT(acc, DATA_TYPE, dst_addr, VEC_SIZE, PARTIAL_STORE_N0, PARTIAL_STORE_N0 != 0 && get_global_id(0) == 0)
 }
 #endif // defined(CONV_STRIDE_X) && defined(CONV_STRIDE_Y)

 #if defined(NUM_ROWS_PROCESSED) && defined(NUM_PLANES_PROCESSED)
 /** This function computes the depthwise convolution for NHWC data layout when the stride along the width and height is 1.
  *
  * @note Datatype should be given as a preprocessor argument using -DDATA_TYPE=type. e.g. -DDATA_TYPE=float
  * @note The number of elements read per thread must be passed at compile time using -DVEC_SIZE (e.g. -DVEC_SIZE=2)
  * @note Dimension two of the input tensor (height for NHWC data layout) must be passed at compile time using -DSRC_DIM2 (e.g. -DSRC_DIM_2=112)
  * @note The number of rows processed per thread must be passed at compile time using -DNUM_ROWS_PROCESSED (i.e. -DNUM_ROWS_PROCESSED=2)
  * @note The number of planes processed per thread must be passed at compile time using -DNUM_PLANES_PROCESSED (i.e. -DNUM_PLANES_PROCESSED=2)
  * @note The convolution pad top must be passed at compile time using -DCONV_PAD_TOP (e.g. -DCONV_PAD_TOP=1)
  * @note The convolution pad top must be passed at compile time using -DCONV_PAD_LEFT (e.g. -DCONV_PAD_LEFT=1)
  * @note It is possible to select the activation function to apply using -DACTIVATION_TYPE e.g. -DACTIVATION_TYPE=relu
  * @note A, B variables required by some activation functions are set using -DA_VAL= and -DB_VAL= respectively
  * @note Vector size should be given as a preprocessor argument using -DVEC_SIZE=size
  * @note The size of the partial store block in y must be passed at compile time using -DPARTIAL_STORE_M0 (e.g. -DPARTIAL_STORE_M0=1)
  * @note The size of the partial store block in x must be passed at compile time using -DPARTIAL_STORE_N0 (e.g. -DPARTIAL_STORE_N0=1)
  * @note The size of the output's second dimension must be passed at compile time using -DDST_DIM_1 (e.g. -DDST_DIM_1=64)
  * @note The size of the output's third dimension must be passed at compile time using -DDST_DIM_2 (e.g. -DDST_DIM_2=32)
  * @note In case of biases, -DHAS_BIAS must to be passed at compile
  * @note If the output tensor has more than three dimensions, its third dimension must be passed at compile time using -DDST_DEPTH (e.g. -DDST_DEPTH=32)
  *
  * @param[in] src_ptr                               Pointer to the source tensor. Supported data types: F16/F32
  * @param[in] src_stride_x                          Stride of the source tensor in X dimension (in bytes)
  * @param[in] src_step_x                            src_stride_x * number of elements along X processed per workitem(in bytes)
  * @param[in] src_stride_y                          Stride of the source tensor in Y dimension (in bytes)
  * @param[in] src_step_y                            src_stride_y * number of elements along Y processed per workitem(in bytes)
  * @param[in] src_stride_z                          Stride of the source tensor in Z dimension (in bytes)
  * @param[in] src_step_z                            src_stride_y * number of elements along Z processed per workitem(in bytes)
  * @param[in] src_stride_w                          Stride of the source tensor in W dimension (in bytes)
  * @param[in] src_step_w                            src_stride_w * number of elements along W processed per workitem(in bytes)
  * @param[in] src_offset_first_element_in_bytes     The offset of the first element in the source tensor
  * @param[in] dst_ptr                               Pointer to the destination tensor. Supported data types: same as src_ptr
  * @param[in] dst_stride_x                          Stride of the destination tensor in X dimension (in bytes)
  * @param[in] dst_step_x                            dst_stride_x * number of elements along X processed per workitem(in bytes)
  * @param[in] dst_stride_y                          Stride of the destination tensor in Y dimension (in bytes)
  * @param[in] dst_step_y                            dst_stride_y * number of elements along Y processed per workitem(in bytes)
  * @param[in] dst_stride_z                          Stride of the destination tensor in Z dimension (in bytes)
  * @param[in] dst_step_z                            dst_stride_z * number of elements along Y processed per workitem(in bytes)
  * @param[in] dst_stride_w                          Stride of the destination tensor in W dimension (in bytes)
  * @param[in] dst_step_w                            dst_stride_w * number of elements along W processed per workitem(in bytes)
  * @param[in] dst_offset_first_element_in_bytes     The offset of the first element in the destination tensor
  * @param[in] weights_ptr                           Pointer to the weights tensor. Supported data types: F16/F32
  * @param[in] weights_stride_x                      Stride of the weights tensor in X dimension (in bytes)
  * @param[in] weights_step_x                        weights_stride_x * number of elements along X processed per workitem(in bytes)
  * @param[in] weights_stride_y                      Stride of the weights tensor in Y dimension (in bytes)
  * @param[in] weights_step_y                        weights_stride_y * number of elements along Y processed per workitem(in bytes)
  * @param[in] weights_stride_z                      Stride of the weights tensor in Z dimension (in bytes)
  * @param[in] weights_step_z                        weights_stride_z * number of elements along Y processed per workitem(in bytes)
  * @param[in] weights_offset_first_element_in_bytes The offset of the first element in the weights tensor
  * @param[in] max_offset                            Max offset for the input tensor
  * @param[in] biases_ptr                            (Optional) Pointer to the biases vector. Supported data types: same as src_ptr
  * @param[in] biases_stride_x                       (Optional) Stride of the biases vector in X dimension (in bytes)
  * @param[in] biases_step_x                         (Optional) biases_stride_x * number of elements along X processed per workitem(in bytes)
  * @param[in] biases_offset_first_element_in_bytes  (Optional) The offset of the first element in the biases vector
  */
 __kernel void depthwise_convolution_3x3_nhwc_stride1(
     TENSOR4D_DECLARATION(src),
     TENSOR4D_DECLARATION(dst),
     TENSOR3D_DECLARATION(weights)
 #if defined(HAS_BIAS)
     ,
     VECTOR_DECLARATION(biases)
 #endif /* defined(HAS_BIAS) */
 )
 {
     int x_offset = max((int)(get_global_id(0) * VEC_SIZE - (VEC_SIZE - PARTIAL_STORE_N0) % VEC_SIZE), 0) * sizeof(DATA_TYPE);
     int y        = get_global_id(1); // spatial coordinate x
 #if defined(DST_DEPTH)
     int z = get_global_id(2) % (int)DST_DEPTH; // spatial coordinate y
     int b = get_global_id(2) / (int)DST_DEPTH; // batch
 #else                                          // defined(DST_DEPTH)
     int             z        = get_global_id(2); // spatial coordinate y
 #endif                                         // defined(DST_DEPTH)

     __global uchar *weights_addr = weights_ptr + weights_offset_first_element_in_bytes + x_offset;

 #if defined(DST_DEPTH)
     __global uchar *src_addr = src_ptr + src_offset_first_element_in_bytes + x_offset + b * src_stride_w;
 #else  /* defined(DST_DEPTH) */
     __global uchar *src_addr = src_ptr + src_offset_first_element_in_bytes + x_offset;
 #endif /* defined(DST_DEPTH) */

     int4 src_coord_y = (int4)(y * NUM_ROWS_PROCESSED - CONV_PAD_LEFT) + V_OFFS4(int);
     int4 src_coord_z = (int4)(z * NUM_PLANES_PROCESSED - CONV_PAD_TOP) + V_OFFS4(int);

     int4 src_offset_y = clamp(src_coord_y, (int4)0, (int4)(SRC_DIM_1 - 1));
     int4 src_offset_z = clamp(src_coord_z, (int4)0, (int4)(SRC_DIM_2 - 1));

     // Use these vectors to check whether the unclamped load would have been out of bounds
     src_coord_y = (src_offset_y != src_coord_y);
     src_coord_z = (src_offset_z != src_coord_z);

     src_offset_y *= (int4)src_stride_y;
     src_offset_z *= (int4)src_stride_z;

     // We compute VEC_SIZEx2x2 [C,W,H] elements
     VEC_FLOAT acc0 = 0;
     VEC_FLOAT acc1 = 0;
     VEC_FLOAT acc2 = 0;
     VEC_FLOAT acc3 = 0;

     // Load weights
     VEC_FLOAT w0 = VLOAD(VEC_SIZE)(0, (__global DATA_TYPE *)(weights_addr + 0 * weights_stride_y + 0 * weights_stride_z));
     VEC_FLOAT w1 = VLOAD(VEC_SIZE)(0, (__global DATA_TYPE *)(weights_addr + 1 * weights_stride_y + 0 * weights_stride_z));
     VEC_FLOAT w2 = VLOAD(VEC_SIZE)(0, (__global DATA_TYPE *)(weights_addr + 2 * weights_stride_y + 0 * weights_stride_z));
     VEC_FLOAT w3 = VLOAD(VEC_SIZE)(0, (__global DATA_TYPE *)(weights_addr + 0 * weights_stride_y + 1 * weights_stride_z));
     VEC_FLOAT w4 = VLOAD(VEC_SIZE)(0, (__global DATA_TYPE *)(weights_addr + 1 * weights_stride_y + 1 * weights_stride_z));
     VEC_FLOAT w5 = VLOAD(VEC_SIZE)(0, (__global DATA_TYPE *)(weights_addr + 2 * weights_stride_y + 1 * weights_stride_z));
     VEC_FLOAT w6 = VLOAD(VEC_SIZE)(0, (__global DATA_TYPE *)(weights_addr + 0 * weights_stride_y + 2 * weights_stride_z));
     VEC_FLOAT w7 = VLOAD(VEC_SIZE)(0, (__global DATA_TYPE *)(weights_addr + 1 * weights_stride_y + 2 * weights_stride_z));
     VEC_FLOAT w8 = VLOAD(VEC_SIZE)(0, (__global DATA_TYPE *)(weights_addr + 2 * weights_stride_y + 2 * weights_stride_z));

     // Load input values
     // z == 0
     VEC_FLOAT values0 = VLOAD(VEC_SIZE)(0, (__global DATA_TYPE *)(src_addr + src_offset_z.s0 + src_offset_y.s0));
     VEC_FLOAT values1 = VLOAD(VEC_SIZE)(0, (__global DATA_TYPE *)(src_addr + src_offset_z.s0 + src_offset_y.s1));
     VEC_FLOAT values2 = VLOAD(VEC_SIZE)(0, (__global DATA_TYPE *)(src_addr + src_offset_z.s0 + src_offset_y.s2));
     VEC_FLOAT values3 = VLOAD(VEC_SIZE)(0, (__global DATA_TYPE *)(src_addr + src_offset_z.s0 + src_offset_y.s3));

     FILL_ZERO_OUT_OF_BOUND_4(DATA_TYPE, VEC_SIZE, values, src_coord_y | (int4)src_coord_z.s0);

     acc0 = fma(values0, w0, acc0);
     acc0 = fma(values1, w1, acc0);
     acc0 = fma(values2, w2, acc0);
     acc1 = fma(values1, w0, acc1);
     acc1 = fma(values2, w1, acc1);
     acc1 = fma(values3, w2, acc1);

     // z == 1
     values0 = VLOAD(VEC_SIZE)(0, (__global DATA_TYPE *)(src_addr + src_offset_z.s1 + src_offset_y.s0));
     values1 = VLOAD(VEC_SIZE)(0, (__global DATA_TYPE *)(src_addr + src_offset_z.s1 + src_offset_y.s1));
     values2 = VLOAD(VEC_SIZE)(0, (__global DATA_TYPE *)(src_addr + src_offset_z.s1 + src_offset_y.s2));
     values3 = VLOAD(VEC_SIZE)(0, (__global DATA_TYPE *)(src_addr + src_offset_z.s1 + src_offset_y.s3));

     FILL_ZERO_OUT_OF_BOUND_4(DATA_TYPE, VEC_SIZE, values, src_coord_y | (int4)src_coord_z.s1);

     acc0 = fma(values0, w3, acc0);
     acc0 = fma(values1, w4, acc0);
     acc0 = fma(values2, w5, acc0);
     acc1 = fma(values1, w3, acc1);
     acc1 = fma(values2, w4, acc1);
     acc1 = fma(values3, w5, acc1);

     acc2 = fma(values0, w0, acc2);
     acc2 = fma(values1, w1, acc2);
     acc2 = fma(values2, w2, acc2);
     acc3 = fma(values1, w0, acc3);
     acc3 = fma(values2, w1, acc3);
     acc3 = fma(values3, w2, acc3);

     // z == 2
     values0 = VLOAD(VEC_SIZE)(0, (__global DATA_TYPE *)(src_addr + src_offset_z.s2 + src_offset_y.s0));
     values1 = VLOAD(VEC_SIZE)(0, (__global DATA_TYPE *)(src_addr + src_offset_z.s2 + src_offset_y.s1));
     values2 = VLOAD(VEC_SIZE)(0, (__global DATA_TYPE *)(src_addr + src_offset_z.s2 + src_offset_y.s2));
     values3 = VLOAD(VEC_SIZE)(0, (__global DATA_TYPE *)(src_addr + src_offset_z.s2 + src_offset_y.s3));

     FILL_ZERO_OUT_OF_BOUND_4(DATA_TYPE, VEC_SIZE, values, src_coord_y | (int4)src_coord_z.s2);

     acc0 = fma(values0, w6, acc0);
     acc0 = fma(values1, w7, acc0);
     acc0 = fma(values2, w8, acc0);
     acc1 = fma(values1, w6, acc1);
     acc1 = fma(values2, w7, acc1);
     acc1 = fma(values3, w8, acc1);

     acc2 = fma(values0, w3, acc2);
     acc2 = fma(values1, w4, acc2);
     acc2 = fma(values2, w5, acc2);
     acc3 = fma(values1, w3, acc3);
     acc3 = fma(values2, w4, acc3);
     acc3 = fma(values3, w5, acc3);

     // z == 3
     values0 = VLOAD(VEC_SIZE)(0, (__global DATA_TYPE *)(src_addr + src_offset_z.s3 + src_offset_y.s0));
     values1 = VLOAD(VEC_SIZE)(0, (__global DATA_TYPE *)(src_addr + src_offset_z.s3 + src_offset_y.s1));
     values2 = VLOAD(VEC_SIZE)(0, (__global DATA_TYPE *)(src_addr + src_offset_z.s3 + src_offset_y.s2));
     values3 = VLOAD(VEC_SIZE)(0, (__global DATA_TYPE *)(src_addr + src_offset_z.s3 + src_offset_y.s3));

     FILL_ZERO_OUT_OF_BOUND_4(DATA_TYPE, VEC_SIZE, values, src_coord_y | (int4)src_coord_z.s3);

     acc2 = fma(values0, w6, acc2);
     acc2 = fma(values1, w7, acc2);
     acc2 = fma(values2, w8, acc2);
     acc3 = fma(values1, w6, acc3);
     acc3 = fma(values2, w7, acc3);
     acc3 = fma(values3, w8, acc3);

 #if defined(HAS_BIAS)
     __global uchar *biases_addr = biases_ptr + biases_offset_first_element_in_bytes + x_offset;

     VEC_FLOAT bias_values = VLOAD(VEC_SIZE)(0, (__global DATA_TYPE *)biases_addr);

     acc0 += bias_values;
     acc1 += bias_values;
     acc2 += bias_values;
     acc3 += bias_values;
 #endif // defined(HAS_BIAS)

     int2 dst_offset_y = min((int2)(y * NUM_ROWS_PROCESSED) + V_OFFS2(int), (int2)(DST_DIM_1 - 1)) * (int2)dst_stride_y;
     int  dst_coord_z  = z * NUM_PLANES_PROCESSED;

 #if defined(DST_DEPTH)
     __global uchar *dst_addr = dst_ptr + dst_offset_first_element_in_bytes + x_offset + dst_coord_z * dst_stride_z + b * dst_stride_w;
 #else  // defined(DST_DEPTH)
     __global uchar *dst_addr = dst_ptr + dst_offset_first_element_in_bytes + x_offset + dst_coord_z * dst_stride_z;
 #endif //  defined(DST_DEPTH)

     /* Store vectors in reverse order along the Y. The Y offsets are calculated so that they are forced to be in bound.
      * If only the first address is in bound, the Y offset of the second address will be brought back and there will be 2 writes in the same location for the same thread.
      * Since the last vector to be written is always the valid one for that location, it overwrites the wrong values.
      */
     values0 = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, acc1, A_VAL, B_VAL);
     STORE_VECTOR_SELECT(values, DATA_TYPE, dst_addr + dst_offset_y.s1, VEC_SIZE, PARTIAL_STORE_N0, PARTIAL_STORE_N0 != 0 && get_global_id(0) == 0)

     values0 = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, acc0, A_VAL, B_VAL);
     STORE_VECTOR_SELECT(values, DATA_TYPE, dst_addr + dst_offset_y.s0, VEC_SIZE, PARTIAL_STORE_N0, PARTIAL_STORE_N0 != 0 && get_global_id(0) == 0)

 #if((DST_DIM_2 % NUM_PLANES_PROCESSED) != 0)
     if((dst_coord_z + 1) < DST_DIM_2)
 #endif // ((DST_DIM_2 % NUM_PLANES_PROCESSED) != 0)
     {
         values0 = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, acc3, A_VAL, B_VAL);
         STORE_VECTOR_SELECT(values, DATA_TYPE, dst_addr + dst_stride_z + dst_offset_y.s1, VEC_SIZE, PARTIAL_STORE_N0, PARTIAL_STORE_N0 != 0 && get_global_id(0) == 0)

         values0 = ACTIVATION(ACTIVATION_TYPE, DATA_TYPE, VEC_SIZE, acc2, A_VAL, B_VAL);
         STORE_VECTOR_SELECT(values, DATA_TYPE, dst_addr + dst_stride_z + dst_offset_y.s0, VEC_SIZE, PARTIAL_STORE_N0, PARTIAL_STORE_N0 != 0 && get_global_id(0) == 0)
     }
 }

 #endif // defined(NUM_ROWS_PROCESSED) && defined(NUM_PLANES_PROCESSED)
 #endif // defined(VEC_SIZE) && defined(SRC_DIM_2) && defined(CONV_PAD_TOP) && defined(CONV_PAD_LEFT) && defined(DATA_TYPE)

 )"