| /* |
| * Copyright (C) 2013-2014 The Android Open Source Project |
| * |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| #define ENTRY(f) .text; .align 4; .globl f; .type f,#function; f: .fnstart |
| #define END(f) .fnend; .size f, .-f; |
| |
| #define BLEND_LIST(X) \ |
| X(0, CLEAR) \ |
| X(1, SRC) \ |
| X(2, DST) \ |
| X(3, SRC_OVER) \ |
| X(4, DST_OVER) \ |
| X(5, SRC_IN) \ |
| X(6, DST_IN) \ |
| X(7, SRC_OUT) \ |
| X(8, DST_OUT) \ |
| X(9, SRC_ATOP) \ |
| X(10, DST_ATOP) \ |
| X(11, XOR) \ |
| X(14, MULTIPLY) \ |
| X(21, DIFFERENCE) \ |
| X(34, ADD) \ |
| X(35, SUBTRACT) |
| |
| .eabi_attribute 25,1 @Tag_ABI_align8_preserved |
| .arm |
| |
| /* For every blend operation supported, define a macro with just the arithmetic |
| * component. The rest can be handled later on. |
| * |
| * At entry q0-q3 contain the RGBA data from the destination buffer, and q8-q11 |
| * contain the data from the source buffer. Both have already been split out |
| * into one colour component per register (if necessary). q3 and q11 contain |
| * the alpha components. |
| * |
| * At the same time as defining the assembly macro, define a corresponding |
| * preprocessor macro indicating any other requirements. |
| * zipped=0 -- The macro does not require the RGBA components to be |
| * separated. |
| * lddst=0 -- The macro does not require data from the destination buffer. |
| * ldsrc=0 -- The macro does not require data from the source buffer. |
| * nowrap=1 -- The macro requires no wrapper at all, and should simply be |
| * inserted without any surrounding load/store or loop code. |
| */ |
| |
| #define params_CLEAR zipped=0, lddst=0, ldsrc=0 |
| .macro blend_kernel_CLEAR |
| vmov.i8 q0, #0 |
| vmov.i8 q1, #0 |
| vmov.i8 q2, #0 |
| vmov.i8 q3, #0 |
| .endm |
| |
| #define params_SRC zipped=0, lddst=0 |
| .macro blend_kernel_SRC |
| vmov q0, q8 |
| vmov q1, q9 |
| vmov q2, q10 |
| vmov q3, q11 |
| .endm |
| |
| #define params_DST nowrap=1 |
| .macro blend_kernel_DST |
| /* nop */ |
| .endm |
| |
| #define params_SRC_OVER zipped=1 |
| .macro blend_kernel_SRC_OVER |
| vmvn q7, q11 |
| |
| vmull.u8 q12, d15, d1 |
| vmull.u8 q0, d14, d0 |
| vmull.u8 q13, d15, d3 |
| vmull.u8 q1, d14, d2 |
| vmull.u8 q14, d15, d5 |
| vmull.u8 q2, d14, d4 |
| vmull.u8 q15, d15, d7 |
| vmull.u8 q3, d14, d6 |
| |
| vrshrn.u16 d8, q0, #8 |
| vrshrn.u16 d9, q12, #8 |
| vrshrn.u16 d10, q1, #8 |
| vrshrn.u16 d11, q13, #8 |
| vrshrn.u16 d12, q2, #8 |
| vrshrn.u16 d13, q14, #8 |
| vrshrn.u16 d14, q3, #8 |
| vrshrn.u16 d15, q15, #8 |
| |
| vaddw.u8 q0, d8 |
| vaddw.u8 q12, d9 |
| vaddw.u8 q1, d10 |
| vaddw.u8 q13, d11 |
| vaddw.u8 q2, d12 |
| vaddw.u8 q14, d13 |
| vaddw.u8 q3, d14 |
| vaddw.u8 q15, d15 |
| |
| vrshrn.u16 d0, q0, #8 |
| vrshrn.u16 d1, q12, #8 |
| vrshrn.u16 d2, q1, #8 |
| vrshrn.u16 d3, q13, #8 |
| vrshrn.u16 d4, q2, #8 |
| vrshrn.u16 d5, q14, #8 |
| vrshrn.u16 d6, q3, #8 |
| vrshrn.u16 d7, q15, #8 |
| |
| vqadd.u8 q0, q8 |
| vqadd.u8 q1, q9 |
| vqadd.u8 q2, q10 |
| vqadd.u8 q3, q11 |
| .endm |
| |
| #define params_DST_OVER zipped=1 |
| .macro blend_kernel_DST_OVER |
| vmvn q7, q3 |
| |
| vmull.u8 q12, d15, d17 |
| vmull.u8 q8, d14, d16 |
| vmull.u8 q13, d15, d19 |
| vmull.u8 q9, d14, d18 |
| vmull.u8 q14, d15, d21 |
| vmull.u8 q10, d14, d20 |
| vmull.u8 q15, d15, d23 |
| vmull.u8 q11, d14, d22 |
| |
| vrshrn.u16 d8, q0, #8 |
| vrshrn.u16 d9, q12, #8 |
| vrshrn.u16 d10, q1, #8 |
| vrshrn.u16 d11, q13, #8 |
| vrshrn.u16 d12, q2, #8 |
| vrshrn.u16 d13, q14, #8 |
| vrshrn.u16 d14, q3, #8 |
| vrshrn.u16 d15, q15, #8 |
| |
| vaddw.u8 q8, d8 |
| vaddw.u8 q12, d9 |
| vaddw.u8 q9, d10 |
| vaddw.u8 q13, d11 |
| vaddw.u8 q10, d12 |
| vaddw.u8 q14, d13 |
| vaddw.u8 q11, d14 |
| vaddw.u8 q15, d15 |
| |
| vrshrn.u16 d16, q8, #8 |
| vrshrn.u16 d17, q12, #8 |
| vrshrn.u16 d18, q9, #8 |
| vrshrn.u16 d19, q13, #8 |
| vrshrn.u16 d20, q10, #8 |
| vrshrn.u16 d21, q14, #8 |
| vrshrn.u16 d22, q11, #8 |
| vrshrn.u16 d23, q15, #8 |
| |
| vqadd.u8 q0, q8 |
| vqadd.u8 q1, q9 |
| vqadd.u8 q2, q10 |
| vqadd.u8 q3, q11 |
| .endm |
| |
| #define params_SRC_IN zipped=1 |
| .macro blend_kernel_SRC_IN |
| vmull.u8 q12, d7, d17 |
| vmull.u8 q0, d6, d16 |
| vmull.u8 q13, d7, d19 |
| vmull.u8 q1, d6, d18 |
| vmull.u8 q14, d7, d21 |
| vmull.u8 q2, d6, d20 |
| vmull.u8 q15, d7, d23 |
| vmull.u8 q3, d6, d22 |
| |
| vrshrn.u16 d8, q0, #8 |
| vrshrn.u16 d9, q12, #8 |
| vrshrn.u16 d10, q1, #8 |
| vrshrn.u16 d11, q13, #8 |
| vrshrn.u16 d12, q2, #8 |
| vrshrn.u16 d13, q14, #8 |
| vrshrn.u16 d14, q3, #8 |
| vrshrn.u16 d15, q15, #8 |
| |
| vaddw.u8 q0, d8 |
| vaddw.u8 q12, d9 |
| vaddw.u8 q1, d10 |
| vaddw.u8 q13, d11 |
| vaddw.u8 q2, d12 |
| vaddw.u8 q14, d13 |
| vaddw.u8 q3, d14 |
| vaddw.u8 q15, d15 |
| |
| vrshrn.u16 d0, q0, #8 |
| vrshrn.u16 d1, q12, #8 |
| vrshrn.u16 d2, q1, #8 |
| vrshrn.u16 d3, q13, #8 |
| vrshrn.u16 d4, q2, #8 |
| vrshrn.u16 d5, q14, #8 |
| vrshrn.u16 d6, q3, #8 |
| vrshrn.u16 d7, q15, #8 |
| .endm |
| |
| #define params_DST_IN zipped=1 |
| .macro blend_kernel_DST_IN |
| vmull.u8 q12, d1, d23 |
| vmull.u8 q0, d0, d22 |
| vmull.u8 q13, d3, d23 |
| vmull.u8 q1, d2, d22 |
| vmull.u8 q14, d5, d23 |
| vmull.u8 q2, d4, d22 |
| vmull.u8 q15, d7, d23 |
| vmull.u8 q3, d6, d22 |
| |
| vrshrn.u16 d8, q0, #8 |
| vrshrn.u16 d9, q12, #8 |
| vrshrn.u16 d10, q1, #8 |
| vrshrn.u16 d11, q13, #8 |
| vrshrn.u16 d12, q2, #8 |
| vrshrn.u16 d13, q14, #8 |
| vrshrn.u16 d14, q3, #8 |
| vrshrn.u16 d15, q15, #8 |
| |
| vaddw.u8 q0, d8 |
| vaddw.u8 q12, d9 |
| vaddw.u8 q1, d10 |
| vaddw.u8 q13, d11 |
| vaddw.u8 q2, d12 |
| vaddw.u8 q14, d13 |
| vaddw.u8 q3, d14 |
| vaddw.u8 q15, d15 |
| |
| vrshrn.u16 d0, q0, #8 |
| vrshrn.u16 d1, q12, #8 |
| vrshrn.u16 d2, q1, #8 |
| vrshrn.u16 d3, q13, #8 |
| vrshrn.u16 d4, q2, #8 |
| vrshrn.u16 d5, q14, #8 |
| vrshrn.u16 d6, q3, #8 |
| vrshrn.u16 d7, q15, #8 |
| .endm |
| |
| #define params_SRC_OUT zipped=1 |
| .macro blend_kernel_SRC_OUT |
| vmvn q3, q3 |
| blend_kernel_SRC_IN |
| .endm |
| |
| |
| #define params_DST_OUT zipped=1 |
| .macro blend_kernel_DST_OUT |
| vmvn q11, q11 |
| blend_kernel_DST_IN |
| .endm |
| |
| #define params_SRC_ATOP zipped=1 |
| .macro blend_kernel_SRC_ATOP |
| vmvn q11, q11 |
| |
| vmull.u8 q12, d23, d1 |
| vmull.u8 q0, d22, d0 |
| vmull.u8 q13, d23, d3 |
| vmull.u8 q1, d22, d2 |
| vmull.u8 q14, d23, d5 |
| vmull.u8 q2, d22, d4 |
| |
| vmull.u8 q4, d7, d17 |
| vmull.u8 q8, d6, d16 |
| vmull.u8 q5, d7, d19 |
| vmull.u8 q9, d6, d18 |
| vmull.u8 q6, d7, d21 |
| vmull.u8 q10, d6, d20 |
| |
| vqadd.u16 q12, q4 |
| vqadd.u16 q0, q8 |
| vqadd.u16 q13, q5 |
| vqadd.u16 q1, q9 |
| vqadd.u16 q14, q6 |
| vqadd.u16 q2, q10 |
| |
| vrshr.u16 q8, q0, #8 |
| vrshr.u16 q4, q12, #8 |
| vrshr.u16 q9, q1, #8 |
| vrshr.u16 q5, q13, #8 |
| vrshr.u16 q10, q2, #8 |
| vrshr.u16 q6, q14, #8 |
| |
| vqadd.u16 q0, q8 |
| vqadd.u16 q12, q4 |
| vqadd.u16 q1, q9 |
| vqadd.u16 q13, q5 |
| vqadd.u16 q2, q10 |
| vqadd.u16 q14, q6 |
| |
| vqrshrn.u16 d0, q0, #8 |
| vqrshrn.u16 d1, q12, #8 |
| vqrshrn.u16 d2, q1, #8 |
| vqrshrn.u16 d3, q13, #8 |
| vqrshrn.u16 d4, q2, #8 |
| vqrshrn.u16 d5, q14, #8 |
| .endm |
| |
| #define params_DST_ATOP zipped=1 |
| .macro blend_kernel_DST_ATOP |
| vmvn q3, q3 |
| |
| vmull.u8 q12, d23, d1 |
| vmull.u8 q0, d22, d0 |
| vmull.u8 q13, d23, d3 |
| vmull.u8 q1, d22, d2 |
| vmull.u8 q14, d23, d5 |
| vmull.u8 q2, d22, d4 |
| |
| vmull.u8 q4, d7, d17 |
| vmull.u8 q8, d6, d16 |
| vmull.u8 q5, d7, d19 |
| vmull.u8 q9, d6, d18 |
| vmull.u8 q6, d7, d21 |
| vmull.u8 q10, d6, d20 |
| |
| vqadd.u16 q12, q4 |
| vqadd.u16 q0, q8 |
| vqadd.u16 q13, q5 |
| vqadd.u16 q1, q9 |
| vqadd.u16 q14, q6 |
| vqadd.u16 q2, q10 |
| |
| vrshr.u16 q8, q0, #8 |
| vrshr.u16 q4, q12, #8 |
| vrshr.u16 q9, q1, #8 |
| vrshr.u16 q5, q13, #8 |
| vrshr.u16 q10, q2, #8 |
| vrshr.u16 q6, q14, #8 |
| |
| vqadd.u16 q0, q8 |
| vqadd.u16 q12, q4 |
| vqadd.u16 q1, q9 |
| vqadd.u16 q13, q5 |
| vqadd.u16 q2, q10 |
| vqadd.u16 q14, q6 |
| |
| vqrshrn.u16 d0, q0, #8 |
| vqrshrn.u16 d1, q12, #8 |
| vqrshrn.u16 d2, q1, #8 |
| vqrshrn.u16 d3, q13, #8 |
| vqrshrn.u16 d4, q2, #8 |
| vqrshrn.u16 d5, q14, #8 |
| |
| vmov q3, q11 |
| .endm |
| |
| #define params_MULTIPLY zipped=0 |
| .macro blend_kernel_MULTIPLY |
| vmull.u8 q12, d1, d17 |
| vmull.u8 q0, d0, d16 |
| vmull.u8 q13, d3, d19 |
| vmull.u8 q1, d2, d18 |
| vmull.u8 q14, d5, d21 |
| vmull.u8 q2, d4, d20 |
| vmull.u8 q15, d7, d23 |
| vmull.u8 q3, d6, d22 |
| |
| vrshrn.u16 d8, q0, #8 |
| vrshrn.u16 d9, q12, #8 |
| vrshrn.u16 d10, q1, #8 |
| vrshrn.u16 d11, q13, #8 |
| vrshrn.u16 d12, q2, #8 |
| vrshrn.u16 d13, q14, #8 |
| vrshrn.u16 d14, q3, #8 |
| vrshrn.u16 d15, q15, #8 |
| |
| vaddw.u8 q0, d8 |
| vaddw.u8 q12, d9 |
| vaddw.u8 q1, d10 |
| vaddw.u8 q13, d11 |
| vaddw.u8 q2, d12 |
| vaddw.u8 q14, d13 |
| vaddw.u8 q3, d14 |
| vaddw.u8 q15, d15 |
| |
| vrshrn.u16 d0, q0, #8 |
| vrshrn.u16 d1, q12, #8 |
| vrshrn.u16 d2, q1, #8 |
| vrshrn.u16 d3, q13, #8 |
| vrshrn.u16 d4, q2, #8 |
| vrshrn.u16 d5, q14, #8 |
| vrshrn.u16 d6, q3, #8 |
| vrshrn.u16 d7, q15, #8 |
| .endm |
| |
| #define params_ADD zipped=0 |
| .macro blend_kernel_ADD |
| vqadd.u8 q0, q0, q8 |
| vqadd.u8 q1, q1, q9 |
| vqadd.u8 q2, q2, q10 |
| vqadd.u8 q3, q3, q11 |
| .endm |
| |
| #define params_SUBTRACT zipped=0 |
| .macro blend_kernel_SUBTRACT |
| vqsub.u8 q0, q0, q8 |
| vqsub.u8 q1, q1, q9 |
| vqsub.u8 q2, q2, q10 |
| vqsub.u8 q3, q3, q11 |
| .endm |
| |
| #define params_DIFFERENCE zipped=0 |
| .macro blend_kernel_DIFFERENCE |
| vabd.u8 q0, q0, q8 |
| vabd.u8 q1, q1, q9 |
| vabd.u8 q2, q2, q10 |
| vabd.u8 q3, q3, q11 |
| .endm |
| |
| #define params_XOR zipped=0 |
| .macro blend_kernel_XOR |
| veor q0, q0, q8 |
| veor q1, q1, q9 |
| veor q2, q2, q10 |
| veor q3, q3, q11 |
| .endm |
| |
| |
| /* Define the wrapper code which will load and store the data, iterate the |
| * correct number of times, and safely handle the remainder at the end of the |
| * loop. Various sections of assembly code are dropped or substituted for |
| * simpler operations if they're not needed. |
| */ |
| .macro wrap_line kernel, nowrap=0, zipped=1, lddst=1, ldsrc=1, pld=1 |
| .if \nowrap |
| \kernel |
| .else |
| vpush {d8-d15} |
| subs r2, #64 |
| b 2f |
| .align 4 |
| 1: |
| .if \lddst |
| .if \zipped |
| vld4.8 {d0,d2,d4,d6}, [r0]! |
| vld4.8 {d1,d3,d5,d7}, [r0]! |
| .else |
| vld1.8 {d0-d3}, [r0]! |
| vld1.8 {d4-d7}, [r0]! |
| .endif |
| sub r0, #64 |
| .endif |
| .if \ldsrc |
| .if \zipped |
| vld4.8 {d16,d18,d20,d22}, [r1]! |
| vld4.8 {d17,d19,d21,d23}, [r1]! |
| .else |
| vld1.8 {d16-d19}, [r1]! |
| vld1.8 {d20-d23}, [r1]! |
| .endif |
| .endif |
| .if \pld |
| .if \lddst ; pld [r0, #192] ; .endif |
| .if \ldsrc ; pld [r1, #192] ; .endif |
| .endif |
| |
| \kernel |
| |
| subs r2, #64 |
| .if \zipped |
| vst4.8 {d0,d2,d4,d6}, [r0]! |
| vst4.8 {d1,d3,d5,d7}, [r0]! |
| .else |
| vst1.8 {d0-d3}, [r0]! |
| vst1.8 {d4-d7}, [r0]! |
| .endif |
| |
| 2: bge 1b |
| adds r2, #64 |
| beq 2f |
| |
| /* To handle the tail portion of the data (something less than 64 |
| * bytes) load small power-of-two chunks into working registers. It |
| * doesn't matter where they end up in the register; the same process |
| * will store them back out using the same positions and the operations |
| * don't require data to interact with its neighbours. |
| */ |
| vmov.i8 q0, #0 |
| vmov.i8 q1, #0 |
| vmov.i8 q2, #0 |
| vmov.i8 q3, #0 |
| |
| vmov.i8 q8, #0 |
| vmov.i8 q9, #0 |
| vmov.i8 q10, #0 |
| vmov.i8 q11, #0 |
| |
| tst r2, #32 |
| beq 1f |
| .if \lddst ; vld1.64 {d4-d7}, [r0]! ; .endif |
| .if \ldsrc ; vld1.64 {d20-d23}, [r1]! ; .endif |
| 1: tst r2, #16 |
| beq 1f |
| .if \lddst ; vld1.64 {d2-d3}, [r0]! ; .endif |
| .if \ldsrc ; vld1.64 {d18-d19}, [r1]! ; .endif |
| 1: tst r2, #8 |
| beq 1f |
| .if \lddst ; vld1.64 {d1}, [r0]! ; .endif |
| .if \ldsrc ; vld1.64 {d17}, [r1]! ; .endif |
| 1: tst r2, #4 |
| beq 1f |
| .if \lddst ; vld1.32 {d0[1]}, [r0]! ; .endif |
| .if \ldsrc ; vld1.32 {d16[1]}, [r1]! ; .endif |
| 1: tst r2, #2 |
| beq 1f |
| .if \lddst ; vld1.16 {d0[1]}, [r0]! ; .endif |
| .if \ldsrc ; vld1.16 {d16[1]}, [r1]! ; .endif |
| 1: tst r2, #1 |
| beq 1f |
| .if \lddst ; vld1.8 {d0[1]}, [r0]! ; .endif |
| .if \ldsrc ; vld1.8 {d16[1]}, [r1]! ; .endif |
| 1: |
| .if \lddst ; sub r0, r2 ; .endif |
| |
| .if \zipped |
| /* One small impediment in the process above is that some of the load |
| * operations can't perform byte-wise structure deinterleaving at the |
| * same time as loading only part of a register. So the data is loaded |
| * linearly and unpacked manually at this point. |
| */ |
| vuzp.8 q0, q1 |
| vuzp.8 q2, q3 |
| vuzp.8 q0, q2 |
| vuzp.8 q1, q3 |
| |
| vuzp.8 q8, q9 |
| vuzp.8 q10, q11 |
| vuzp.8 q8, q10 |
| vuzp.8 q9, q11 |
| |
| \kernel |
| |
| vzip.8 q0, q2 |
| vzip.8 q1, q3 |
| vzip.8 q0, q1 |
| vzip.8 q2, q3 |
| .else |
| \kernel |
| .endif |
| |
| tst r2, #32 |
| beq 1f |
| vst1.64 {d4-d7}, [r0]! |
| 1: tst r2, #16 |
| beq 1f |
| vst1.64 {d2-d3}, [r0]! |
| 1: tst r2, #8 |
| beq 1f |
| vst1.64 {d1}, [r0]! |
| 1: tst r2, #4 |
| beq 1f |
| vst1.32 {d0[1]}, [r0]! |
| 1: tst r2, #2 |
| beq 1f |
| vst1.16 {d0[1]}, [r0]! |
| 1: tst r2, #1 |
| beq 2f |
| vst1.8 {d0[1]}, [r0]! |
| 2: vpop {d8-d15} |
| .endif |
| mov r0, #0 |
| bx lr |
| .endm |
| |
| |
| /* produce list of blend_line_XX() functions; each function uses the wrap_line |
| * macro, passing it the name of the operation macro it wants along with |
| * optional parameters to remove unnecessary operations. |
| */ |
| #define BLEND_X(d, n) ENTRY(blend_line_##n) ; wrap_line blend_kernel_##n, params_##n ; END(blend_line_##n) ; |
| BLEND_LIST(BLEND_X) |
| #undef BLEND_X |
| |
| |
| /* int rsdIntrinsicBlend_K( |
| * uchar4 *out, // r0 |
| * uchar4 const *in, // r1 |
| * int slot, // r2 |
| * size_t xstart, // r3 |
| * size_t xend); // [sp] |
| */ |
| ENTRY(rsdIntrinsicBlend_K) |
| adr ip, blend_functions |
| cmp r2, #(blend_functions_end - blend_functions) >> 2 |
| ldrlo ip, [ip, r2, LSL #2] |
| movhs ip, #0 |
| ldr r2, [sp] |
| add r0, r3, LSL #2 |
| add r1, r3, LSL #2 |
| sub r2, r3 |
| mov r2, r2, LSL #2 |
| cmp ip, #0 |
| addne ip, ip, pc |
| bxne ip |
| 1: mov r0, #-1 |
| bx lr |
| |
| blend_functions: |
| .set off,0 |
| #define BLEND_X(d, n) .rept d-off ; .word 0 ; .endr ; .word blend_line_##n-1b ; .set off, d+1 ; |
| BLEND_LIST(BLEND_X) |
| #undef BLEND_X |
| blend_functions_end: |
| |
| END(rsdIntrinsicBlend_K) |
