| ; |
| ; Copyright (c) 2010 The WebM project authors. All Rights Reserved. |
| ; |
| ; Use of this source code is governed by a BSD-style license |
| ; that can be found in the LICENSE file in the root of the source |
| ; tree. An additional intellectual property rights grant can be found |
| ; in the file PATENTS. All contributing project authors may |
| ; be found in the AUTHORS file in the root of the source tree. |
| ; |
| |
| |
| %include "vpx_ports/x86_abi_support.asm" |
| |
| ;void copy_mem16x16_sse2( |
| ; unsigned char *src, |
| ; int src_stride, |
| ; unsigned char *dst, |
| ; int dst_stride |
| ; ) |
| global sym(vp8_copy_mem16x16_sse2) PRIVATE |
| sym(vp8_copy_mem16x16_sse2): |
| push rbp |
| mov rbp, rsp |
| SHADOW_ARGS_TO_STACK 4 |
| push rsi |
| push rdi |
| ; end prolog |
| |
| mov rsi, arg(0) ;src; |
| movdqu xmm0, [rsi] |
| |
| movsxd rax, dword ptr arg(1) ;src_stride; |
| mov rdi, arg(2) ;dst; |
| |
| movdqu xmm1, [rsi+rax] |
| movdqu xmm2, [rsi+rax*2] |
| |
| movsxd rcx, dword ptr arg(3) ;dst_stride |
| lea rsi, [rsi+rax*2] |
| |
| movdqa [rdi], xmm0 |
| add rsi, rax |
| |
| movdqa [rdi+rcx], xmm1 |
| movdqa [rdi+rcx*2],xmm2 |
| |
| lea rdi, [rdi+rcx*2] |
| movdqu xmm3, [rsi] |
| |
| add rdi, rcx |
| movdqu xmm4, [rsi+rax] |
| |
| movdqu xmm5, [rsi+rax*2] |
| lea rsi, [rsi+rax*2] |
| |
| movdqa [rdi], xmm3 |
| add rsi, rax |
| |
| movdqa [rdi+rcx], xmm4 |
| movdqa [rdi+rcx*2],xmm5 |
| |
| lea rdi, [rdi+rcx*2] |
| movdqu xmm0, [rsi] |
| |
| add rdi, rcx |
| movdqu xmm1, [rsi+rax] |
| |
| movdqu xmm2, [rsi+rax*2] |
| lea rsi, [rsi+rax*2] |
| |
| movdqa [rdi], xmm0 |
| add rsi, rax |
| |
| movdqa [rdi+rcx], xmm1 |
| |
| movdqa [rdi+rcx*2], xmm2 |
| movdqu xmm3, [rsi] |
| |
| movdqu xmm4, [rsi+rax] |
| lea rdi, [rdi+rcx*2] |
| |
| add rdi, rcx |
| movdqu xmm5, [rsi+rax*2] |
| |
| lea rsi, [rsi+rax*2] |
| movdqa [rdi], xmm3 |
| |
| add rsi, rax |
| movdqa [rdi+rcx], xmm4 |
| |
| movdqa [rdi+rcx*2],xmm5 |
| movdqu xmm0, [rsi] |
| |
| lea rdi, [rdi+rcx*2] |
| movdqu xmm1, [rsi+rax] |
| |
| add rdi, rcx |
| movdqu xmm2, [rsi+rax*2] |
| |
| lea rsi, [rsi+rax*2] |
| movdqa [rdi], xmm0 |
| |
| movdqa [rdi+rcx], xmm1 |
| movdqa [rdi+rcx*2],xmm2 |
| |
| movdqu xmm3, [rsi+rax] |
| lea rdi, [rdi+rcx*2] |
| |
| movdqa [rdi+rcx], xmm3 |
| |
| ; begin epilog |
| pop rdi |
| pop rsi |
| UNSHADOW_ARGS |
| pop rbp |
| ret |
| |
| |
| ;void vp8_intra_pred_uv_dc_mmx2( |
| ; unsigned char *dst, |
| ; int dst_stride |
| ; unsigned char *above, |
| ; unsigned char *left, |
| ; int left_stride, |
| ; ) |
| global sym(vp8_intra_pred_uv_dc_mmx2) PRIVATE |
| sym(vp8_intra_pred_uv_dc_mmx2): |
| push rbp |
| mov rbp, rsp |
| SHADOW_ARGS_TO_STACK 5 |
| push rsi |
| push rdi |
| ; end prolog |
| |
| ; from top |
| mov rdi, arg(2) ;above; |
| mov rsi, arg(3) ;left; |
| movsxd rax, dword ptr arg(4) ;left_stride; |
| pxor mm0, mm0 |
| movq mm1, [rdi] |
| lea rdi, [rax*3] |
| psadbw mm1, mm0 |
| ; from left |
| movzx ecx, byte [rsi] |
| movzx edx, byte [rsi+rax*1] |
| add ecx, edx |
| movzx edx, byte [rsi+rax*2] |
| add ecx, edx |
| |
| movzx edx, byte [rsi+rdi] |
| lea rsi, [rsi+rax*4] |
| add ecx, edx |
| movzx edx, byte [rsi] |
| add ecx, edx |
| movzx edx, byte [rsi+rax] |
| add ecx, edx |
| movzx edx, byte [rsi+rax*2] |
| add ecx, edx |
| movzx edx, byte [rsi+rdi] |
| add ecx, edx |
| |
| ; add up |
| pextrw edx, mm1, 0x0 |
| lea edx, [edx+ecx+8] |
| sar edx, 4 |
| movd mm1, edx |
| movsxd rcx, dword ptr arg(1) ;dst_stride |
| pshufw mm1, mm1, 0x0 |
| mov rdi, arg(0) ;dst; |
| packuswb mm1, mm1 |
| |
| ; write out |
| lea rax, [rcx*3] |
| lea rdx, [rdi+rcx*4] |
| |
| movq [rdi ], mm1 |
| movq [rdi+rcx ], mm1 |
| movq [rdi+rcx*2], mm1 |
| movq [rdi+rax ], mm1 |
| movq [rdx ], mm1 |
| movq [rdx+rcx ], mm1 |
| movq [rdx+rcx*2], mm1 |
| movq [rdx+rax ], mm1 |
| |
| ; begin epilog |
| pop rdi |
| pop rsi |
| UNSHADOW_ARGS |
| pop rbp |
| ret |
| |
| ;void vp8_intra_pred_uv_dctop_mmx2( |
| ; unsigned char *dst, |
| ; int dst_stride |
| ; unsigned char *above, |
| ; unsigned char *left, |
| ; int left_stride, |
| ; ) |
| global sym(vp8_intra_pred_uv_dctop_mmx2) PRIVATE |
| sym(vp8_intra_pred_uv_dctop_mmx2): |
| push rbp |
| mov rbp, rsp |
| SHADOW_ARGS_TO_STACK 5 |
| GET_GOT rbx |
| push rsi |
| push rdi |
| ; end prolog |
| |
| ;arg(3), arg(4) not used |
| |
| ; from top |
| mov rsi, arg(2) ;above; |
| pxor mm0, mm0 |
| movq mm1, [rsi] |
| psadbw mm1, mm0 |
| |
| ; add up |
| paddw mm1, [GLOBAL(dc_4)] |
| psraw mm1, 3 |
| pshufw mm1, mm1, 0x0 |
| packuswb mm1, mm1 |
| |
| ; write out |
| mov rdi, arg(0) ;dst; |
| movsxd rcx, dword ptr arg(1) ;dst_stride |
| lea rax, [rcx*3] |
| |
| movq [rdi ], mm1 |
| movq [rdi+rcx ], mm1 |
| movq [rdi+rcx*2], mm1 |
| movq [rdi+rax ], mm1 |
| lea rdi, [rdi+rcx*4] |
| movq [rdi ], mm1 |
| movq [rdi+rcx ], mm1 |
| movq [rdi+rcx*2], mm1 |
| movq [rdi+rax ], mm1 |
| |
| ; begin epilog |
| pop rdi |
| pop rsi |
| RESTORE_GOT |
| UNSHADOW_ARGS |
| pop rbp |
| ret |
| |
| ;void vp8_intra_pred_uv_dcleft_mmx2( |
| ; unsigned char *dst, |
| ; int dst_stride |
| ; unsigned char *above, |
| ; unsigned char *left, |
| ; int left_stride, |
| ; ) |
| global sym(vp8_intra_pred_uv_dcleft_mmx2) PRIVATE |
| sym(vp8_intra_pred_uv_dcleft_mmx2): |
| push rbp |
| mov rbp, rsp |
| SHADOW_ARGS_TO_STACK 5 |
| push rsi |
| push rdi |
| ; end prolog |
| |
| ;arg(2) not used |
| |
| ; from left |
| mov rsi, arg(3) ;left; |
| movsxd rax, dword ptr arg(4) ;left_stride; |
| lea rdi, [rax*3] |
| movzx ecx, byte [rsi] |
| movzx edx, byte [rsi+rax] |
| add ecx, edx |
| movzx edx, byte [rsi+rax*2] |
| add ecx, edx |
| movzx edx, byte [rsi+rdi] |
| add ecx, edx |
| lea rsi, [rsi+rax*4] |
| movzx edx, byte [rsi] |
| add ecx, edx |
| movzx edx, byte [rsi+rax] |
| add ecx, edx |
| movzx edx, byte [rsi+rax*2] |
| add ecx, edx |
| movzx edx, byte [rsi+rdi] |
| lea edx, [ecx+edx+4] |
| |
| ; add up |
| shr edx, 3 |
| movd mm1, edx |
| pshufw mm1, mm1, 0x0 |
| packuswb mm1, mm1 |
| |
| ; write out |
| mov rdi, arg(0) ;dst; |
| movsxd rcx, dword ptr arg(1) ;dst_stride |
| lea rax, [rcx*3] |
| |
| movq [rdi ], mm1 |
| movq [rdi+rcx ], mm1 |
| movq [rdi+rcx*2], mm1 |
| movq [rdi+rax ], mm1 |
| lea rdi, [rdi+rcx*4] |
| movq [rdi ], mm1 |
| movq [rdi+rcx ], mm1 |
| movq [rdi+rcx*2], mm1 |
| movq [rdi+rax ], mm1 |
| |
| ; begin epilog |
| pop rdi |
| pop rsi |
| UNSHADOW_ARGS |
| pop rbp |
| ret |
| |
| ;void vp8_intra_pred_uv_dc128_mmx( |
| ; unsigned char *dst, |
| ; int dst_stride |
| ; unsigned char *above, |
| ; unsigned char *left, |
| ; int left_stride, |
| ; ) |
| global sym(vp8_intra_pred_uv_dc128_mmx) PRIVATE |
| sym(vp8_intra_pred_uv_dc128_mmx): |
| push rbp |
| mov rbp, rsp |
| SHADOW_ARGS_TO_STACK 5 |
| GET_GOT rbx |
| ; end prolog |
| |
| ;arg(2), arg(3), arg(4) not used |
| |
| ; write out |
| movq mm1, [GLOBAL(dc_128)] |
| mov rax, arg(0) ;dst; |
| movsxd rdx, dword ptr arg(1) ;dst_stride |
| lea rcx, [rdx*3] |
| |
| movq [rax ], mm1 |
| movq [rax+rdx ], mm1 |
| movq [rax+rdx*2], mm1 |
| movq [rax+rcx ], mm1 |
| lea rax, [rax+rdx*4] |
| movq [rax ], mm1 |
| movq [rax+rdx ], mm1 |
| movq [rax+rdx*2], mm1 |
| movq [rax+rcx ], mm1 |
| |
| ; begin epilog |
| RESTORE_GOT |
| UNSHADOW_ARGS |
| pop rbp |
| ret |
| |
| ;void vp8_intra_pred_uv_tm_sse2( |
| ; unsigned char *dst, |
| ; int dst_stride |
| ; unsigned char *above, |
| ; unsigned char *left, |
| ; int left_stride, |
| ; ) |
| %macro vp8_intra_pred_uv_tm 1 |
| global sym(vp8_intra_pred_uv_tm_%1) PRIVATE |
| sym(vp8_intra_pred_uv_tm_%1): |
| push rbp |
| mov rbp, rsp |
| SHADOW_ARGS_TO_STACK 5 |
| GET_GOT rbx |
| push rsi |
| push rdi |
| push rbx |
| ; end prolog |
| |
| ; read top row |
| mov edx, 4 |
| mov rsi, arg(2) ;above |
| movsxd rax, dword ptr arg(4) ;left_stride; |
| pxor xmm0, xmm0 |
| %ifidn %1, ssse3 |
| movdqa xmm2, [GLOBAL(dc_1024)] |
| %endif |
| movq xmm1, [rsi] |
| punpcklbw xmm1, xmm0 |
| |
| ; set up left ptrs ans subtract topleft |
| movd xmm3, [rsi-1] |
| mov rsi, arg(3) ;left; |
| %ifidn %1, sse2 |
| punpcklbw xmm3, xmm0 |
| pshuflw xmm3, xmm3, 0x0 |
| punpcklqdq xmm3, xmm3 |
| %else |
| pshufb xmm3, xmm2 |
| %endif |
| psubw xmm1, xmm3 |
| |
| ; set up dest ptrs |
| mov rdi, arg(0) ;dst; |
| movsxd rcx, dword ptr arg(1) ;dst_stride |
| |
| .vp8_intra_pred_uv_tm_%1_loop: |
| mov bl, [rsi] |
| movd xmm3, ebx |
| |
| mov bl, [rsi+rax] |
| movd xmm5, ebx |
| %ifidn %1, sse2 |
| punpcklbw xmm3, xmm0 |
| punpcklbw xmm5, xmm0 |
| pshuflw xmm3, xmm3, 0x0 |
| pshuflw xmm5, xmm5, 0x0 |
| punpcklqdq xmm3, xmm3 |
| punpcklqdq xmm5, xmm5 |
| %else |
| pshufb xmm3, xmm2 |
| pshufb xmm5, xmm2 |
| %endif |
| paddw xmm3, xmm1 |
| paddw xmm5, xmm1 |
| packuswb xmm3, xmm5 |
| movq [rdi ], xmm3 |
| movhps[rdi+rcx], xmm3 |
| lea rsi, [rsi+rax*2] |
| lea rdi, [rdi+rcx*2] |
| dec edx |
| jnz .vp8_intra_pred_uv_tm_%1_loop |
| |
| ; begin epilog |
| pop rbx |
| pop rdi |
| pop rsi |
| RESTORE_GOT |
| UNSHADOW_ARGS |
| pop rbp |
| ret |
| %endmacro |
| |
| vp8_intra_pred_uv_tm sse2 |
| vp8_intra_pred_uv_tm ssse3 |
| |
| ;void vp8_intra_pred_uv_ve_mmx( |
| ; unsigned char *dst, |
| ; int dst_stride |
| ; unsigned char *above, |
| ; unsigned char *left, |
| ; int left_stride, |
| ; ) |
| global sym(vp8_intra_pred_uv_ve_mmx) PRIVATE |
| sym(vp8_intra_pred_uv_ve_mmx): |
| push rbp |
| mov rbp, rsp |
| SHADOW_ARGS_TO_STACK 5 |
| ; end prolog |
| |
| ; arg(3), arg(4) not used |
| |
| ; read from top |
| mov rax, arg(2) ;src; |
| |
| movq mm1, [rax] |
| |
| ; write out |
| mov rax, arg(0) ;dst; |
| movsxd rdx, dword ptr arg(1) ;dst_stride |
| lea rcx, [rdx*3] |
| |
| movq [rax ], mm1 |
| movq [rax+rdx ], mm1 |
| movq [rax+rdx*2], mm1 |
| movq [rax+rcx ], mm1 |
| lea rax, [rax+rdx*4] |
| movq [rax ], mm1 |
| movq [rax+rdx ], mm1 |
| movq [rax+rdx*2], mm1 |
| movq [rax+rcx ], mm1 |
| |
| ; begin epilog |
| UNSHADOW_ARGS |
| pop rbp |
| ret |
| |
| ;void vp8_intra_pred_uv_ho_mmx2( |
| ; unsigned char *dst, |
| ; int dst_stride |
| ; unsigned char *above, |
| ; unsigned char *left, |
| ; int left_stride |
| ; ) |
| %macro vp8_intra_pred_uv_ho 1 |
| global sym(vp8_intra_pred_uv_ho_%1) PRIVATE |
| sym(vp8_intra_pred_uv_ho_%1): |
| push rbp |
| mov rbp, rsp |
| SHADOW_ARGS_TO_STACK 5 |
| push rsi |
| push rdi |
| push rbx |
| %ifidn %1, ssse3 |
| GET_GOT rbx |
| %endif |
| ; end prolog |
| |
| ;arg(2) not used |
| |
| ; read from left and write out |
| %ifidn %1, mmx2 |
| mov edx, 4 |
| %endif |
| mov rsi, arg(3) ;left |
| movsxd rax, dword ptr arg(4) ;left_stride; |
| mov rdi, arg(0) ;dst; |
| movsxd rcx, dword ptr arg(1) ;dst_stride |
| %ifidn %1, ssse3 |
| lea rdx, [rcx*3] |
| movdqa xmm2, [GLOBAL(dc_00001111)] |
| %endif |
| |
| %ifidn %1, mmx2 |
| .vp8_intra_pred_uv_ho_%1_loop: |
| mov bl, [rsi] |
| movd mm0, ebx |
| |
| mov bl, [rsi+rax] |
| movd mm1, ebx |
| |
| punpcklbw mm0, mm0 |
| punpcklbw mm1, mm1 |
| pshufw mm0, mm0, 0x0 |
| pshufw mm1, mm1, 0x0 |
| movq [rdi ], mm0 |
| movq [rdi+rcx], mm1 |
| lea rsi, [rsi+rax*2] |
| lea rdi, [rdi+rcx*2] |
| dec edx |
| jnz .vp8_intra_pred_uv_ho_%1_loop |
| %else |
| mov bl, [rsi] |
| movd xmm0, ebx |
| |
| mov bl, [rsi+rax] |
| movd xmm3, ebx |
| |
| mov bl, [rsi+rax*2] |
| movd xmm1, ebx |
| |
| lea rbx, [rax*3] |
| mov bl, [rsi+rbx] |
| movd xmm4, ebx |
| |
| punpcklbw xmm0, xmm3 |
| punpcklbw xmm1, xmm4 |
| pshufb xmm0, xmm2 |
| pshufb xmm1, xmm2 |
| movq [rdi ], xmm0 |
| movhps [rdi+rcx], xmm0 |
| movq [rdi+rcx*2], xmm1 |
| movhps [rdi+rdx], xmm1 |
| lea rsi, [rsi+rax*4] |
| lea rdi, [rdi+rcx*4] |
| |
| mov bl, [rsi] |
| movd xmm0, ebx |
| |
| mov bl, [rsi+rax] |
| movd xmm3, ebx |
| |
| mov bl, [rsi+rax*2] |
| movd xmm1, ebx |
| |
| lea rbx, [rax*3] |
| mov bl, [rsi+rbx] |
| movd xmm4, ebx |
| |
| punpcklbw xmm0, xmm3 |
| punpcklbw xmm1, xmm4 |
| pshufb xmm0, xmm2 |
| pshufb xmm1, xmm2 |
| movq [rdi ], xmm0 |
| movhps [rdi+rcx], xmm0 |
| movq [rdi+rcx*2], xmm1 |
| movhps [rdi+rdx], xmm1 |
| %endif |
| |
| ; begin epilog |
| %ifidn %1, ssse3 |
| RESTORE_GOT |
| %endif |
| pop rbx |
| pop rdi |
| pop rsi |
| UNSHADOW_ARGS |
| pop rbp |
| ret |
| %endmacro |
| |
| vp8_intra_pred_uv_ho mmx2 |
| vp8_intra_pred_uv_ho ssse3 |
| |
| ;void vp8_intra_pred_y_dc_sse2( |
| ; unsigned char *dst, |
| ; int dst_stride |
| ; unsigned char *above, |
| ; unsigned char *left, |
| ; int left_stride |
| ; ) |
| global sym(vp8_intra_pred_y_dc_sse2) PRIVATE |
| sym(vp8_intra_pred_y_dc_sse2): |
| push rbp |
| mov rbp, rsp |
| SHADOW_ARGS_TO_STACK 5 |
| push rsi |
| push rdi |
| ; end prolog |
| |
| ; from top |
| mov rdi, arg(2) ;above |
| mov rsi, arg(3) ;left |
| movsxd rax, dword ptr arg(4) ;left_stride; |
| |
| pxor xmm0, xmm0 |
| movdqa xmm1, [rdi] |
| psadbw xmm1, xmm0 |
| movq xmm2, xmm1 |
| punpckhqdq xmm1, xmm1 |
| paddw xmm1, xmm2 |
| |
| ; from left |
| lea rdi, [rax*3] |
| |
| movzx ecx, byte [rsi] |
| movzx edx, byte [rsi+rax] |
| add ecx, edx |
| movzx edx, byte [rsi+rax*2] |
| add ecx, edx |
| movzx edx, byte [rsi+rdi] |
| add ecx, edx |
| lea rsi, [rsi+rax*4] |
| |
| movzx edx, byte [rsi] |
| add ecx, edx |
| movzx edx, byte [rsi+rax] |
| add ecx, edx |
| movzx edx, byte [rsi+rax*2] |
| add ecx, edx |
| movzx edx, byte [rsi+rdi] |
| add ecx, edx |
| lea rsi, [rsi+rax*4] |
| |
| movzx edx, byte [rsi] |
| add ecx, edx |
| movzx edx, byte [rsi+rax] |
| add ecx, edx |
| movzx edx, byte [rsi+rax*2] |
| add ecx, edx |
| movzx edx, byte [rsi+rdi] |
| add ecx, edx |
| lea rsi, [rsi+rax*4] |
| |
| movzx edx, byte [rsi] |
| add ecx, edx |
| movzx edx, byte [rsi+rax] |
| add ecx, edx |
| movzx edx, byte [rsi+rax*2] |
| add ecx, edx |
| movzx edx, byte [rsi+rdi] |
| add ecx, edx |
| |
| ; add up |
| pextrw edx, xmm1, 0x0 |
| lea edx, [edx+ecx+16] |
| sar edx, 5 |
| movd xmm1, edx |
| ; FIXME use pshufb for ssse3 version |
| pshuflw xmm1, xmm1, 0x0 |
| punpcklqdq xmm1, xmm1 |
| packuswb xmm1, xmm1 |
| |
| ; write out |
| mov rsi, 2 |
| mov rdi, arg(0) ;dst; |
| movsxd rcx, dword ptr arg(1) ;dst_stride |
| lea rax, [rcx*3] |
| |
| .label |
| movdqa [rdi ], xmm1 |
| movdqa [rdi+rcx ], xmm1 |
| movdqa [rdi+rcx*2], xmm1 |
| movdqa [rdi+rax ], xmm1 |
| lea rdi, [rdi+rcx*4] |
| movdqa [rdi ], xmm1 |
| movdqa [rdi+rcx ], xmm1 |
| movdqa [rdi+rcx*2], xmm1 |
| movdqa [rdi+rax ], xmm1 |
| lea rdi, [rdi+rcx*4] |
| dec rsi |
| jnz .label |
| |
| ; begin epilog |
| pop rdi |
| pop rsi |
| UNSHADOW_ARGS |
| pop rbp |
| ret |
| |
| ;void vp8_intra_pred_y_dctop_sse2( |
| ; unsigned char *dst, |
| ; int dst_stride |
| ; unsigned char *above, |
| ; unsigned char *left, |
| ; int left_stride |
| ; ) |
| global sym(vp8_intra_pred_y_dctop_sse2) PRIVATE |
| sym(vp8_intra_pred_y_dctop_sse2): |
| push rbp |
| mov rbp, rsp |
| SHADOW_ARGS_TO_STACK 5 |
| push rsi |
| GET_GOT rbx |
| ; end prolog |
| |
| ;arg(3), arg(4) not used |
| |
| ; from top |
| mov rcx, arg(2) ;above; |
| pxor xmm0, xmm0 |
| movdqa xmm1, [rcx] |
| psadbw xmm1, xmm0 |
| movdqa xmm2, xmm1 |
| punpckhqdq xmm1, xmm1 |
| paddw xmm1, xmm2 |
| |
| ; add up |
| paddw xmm1, [GLOBAL(dc_8)] |
| psraw xmm1, 4 |
| ; FIXME use pshufb for ssse3 version |
| pshuflw xmm1, xmm1, 0x0 |
| punpcklqdq xmm1, xmm1 |
| packuswb xmm1, xmm1 |
| |
| ; write out |
| mov rsi, 2 |
| mov rdx, arg(0) ;dst; |
| movsxd rcx, dword ptr arg(1) ;dst_stride |
| lea rax, [rcx*3] |
| |
| .label |
| movdqa [rdx ], xmm1 |
| movdqa [rdx+rcx ], xmm1 |
| movdqa [rdx+rcx*2], xmm1 |
| movdqa [rdx+rax ], xmm1 |
| lea rdx, [rdx+rcx*4] |
| movdqa [rdx ], xmm1 |
| movdqa [rdx+rcx ], xmm1 |
| movdqa [rdx+rcx*2], xmm1 |
| movdqa [rdx+rax ], xmm1 |
| lea rdx, [rdx+rcx*4] |
| dec rsi |
| jnz .label |
| |
| ; begin epilog |
| RESTORE_GOT |
| pop rsi |
| UNSHADOW_ARGS |
| pop rbp |
| ret |
| |
| ;void vp8_intra_pred_y_dcleft_sse2( |
| ; unsigned char *dst, |
| ; int dst_stride |
| ; unsigned char *above, |
| ; unsigned char *left, |
| ; int left_stride |
| ; ) |
| global sym(vp8_intra_pred_y_dcleft_sse2) PRIVATE |
| sym(vp8_intra_pred_y_dcleft_sse2): |
| push rbp |
| mov rbp, rsp |
| SHADOW_ARGS_TO_STACK 5 |
| push rsi |
| push rdi |
| ; end prolog |
| |
| ;arg(2) not used |
| |
| ; from left |
| mov rsi, arg(3) ;left; |
| movsxd rax, dword ptr arg(4) ;left_stride; |
| |
| lea rdi, [rax*3] |
| movzx ecx, byte [rsi] |
| movzx edx, byte [rsi+rax] |
| add ecx, edx |
| movzx edx, byte [rsi+rax*2] |
| add ecx, edx |
| movzx edx, byte [rsi+rdi] |
| add ecx, edx |
| lea rsi, [rsi+rax*4] |
| movzx edx, byte [rsi] |
| add ecx, edx |
| movzx edx, byte [rsi+rax] |
| add ecx, edx |
| movzx edx, byte [rsi+rax*2] |
| add ecx, edx |
| movzx edx, byte [rsi+rdi] |
| add ecx, edx |
| lea rsi, [rsi+rax*4] |
| movzx edx, byte [rsi] |
| add ecx, edx |
| movzx edx, byte [rsi+rax] |
| add ecx, edx |
| movzx edx, byte [rsi+rax*2] |
| add ecx, edx |
| movzx edx, byte [rsi+rdi] |
| add ecx, edx |
| lea rsi, [rsi+rax*4] |
| movzx edx, byte [rsi] |
| add ecx, edx |
| movzx edx, byte [rsi+rax] |
| add ecx, edx |
| movzx edx, byte [rsi+rax*2] |
| add ecx, edx |
| movzx edx, byte [rsi+rdi] |
| lea edx, [ecx+edx+8] |
| |
| ; add up |
| shr edx, 4 |
| movd xmm1, edx |
| ; FIXME use pshufb for ssse3 version |
| pshuflw xmm1, xmm1, 0x0 |
| punpcklqdq xmm1, xmm1 |
| packuswb xmm1, xmm1 |
| |
| ; write out |
| mov rsi, 2 |
| mov rdi, arg(0) ;dst; |
| movsxd rcx, dword ptr arg(1) ;dst_stride |
| lea rax, [rcx*3] |
| |
| .label |
| movdqa [rdi ], xmm1 |
| movdqa [rdi+rcx ], xmm1 |
| movdqa [rdi+rcx*2], xmm1 |
| movdqa [rdi+rax ], xmm1 |
| lea rdi, [rdi+rcx*4] |
| movdqa [rdi ], xmm1 |
| movdqa [rdi+rcx ], xmm1 |
| movdqa [rdi+rcx*2], xmm1 |
| movdqa [rdi+rax ], xmm1 |
| lea rdi, [rdi+rcx*4] |
| dec rsi |
| jnz .label |
| |
| ; begin epilog |
| pop rdi |
| pop rsi |
| UNSHADOW_ARGS |
| pop rbp |
| ret |
| |
| ;void vp8_intra_pred_y_dc128_sse2( |
| ; unsigned char *dst, |
| ; int dst_stride |
| ; unsigned char *above, |
| ; unsigned char *left, |
| ; int left_stride |
| ; ) |
| global sym(vp8_intra_pred_y_dc128_sse2) PRIVATE |
| sym(vp8_intra_pred_y_dc128_sse2): |
| push rbp |
| mov rbp, rsp |
| SHADOW_ARGS_TO_STACK 5 |
| push rsi |
| GET_GOT rbx |
| ; end prolog |
| |
| ;arg(2), arg(3), arg(4) not used |
| |
| ; write out |
| mov rsi, 2 |
| movdqa xmm1, [GLOBAL(dc_128)] |
| mov rax, arg(0) ;dst; |
| movsxd rdx, dword ptr arg(1) ;dst_stride |
| lea rcx, [rdx*3] |
| |
| .label |
| movdqa [rax ], xmm1 |
| movdqa [rax+rdx ], xmm1 |
| movdqa [rax+rdx*2], xmm1 |
| movdqa [rax+rcx ], xmm1 |
| lea rax, [rax+rdx*4] |
| movdqa [rax ], xmm1 |
| movdqa [rax+rdx ], xmm1 |
| movdqa [rax+rdx*2], xmm1 |
| movdqa [rax+rcx ], xmm1 |
| lea rax, [rax+rdx*4] |
| dec rsi |
| jnz .label |
| |
| ; begin epilog |
| RESTORE_GOT |
| pop rsi |
| UNSHADOW_ARGS |
| pop rbp |
| ret |
| |
| ;void vp8_intra_pred_y_tm_sse2( |
| ; unsigned char *dst, |
| ; int dst_stride |
| ; unsigned char *above, |
| ; unsigned char *left, |
| ; int left_stride |
| ; ) |
| %macro vp8_intra_pred_y_tm 1 |
| global sym(vp8_intra_pred_y_tm_%1) PRIVATE |
| sym(vp8_intra_pred_y_tm_%1): |
| push rbp |
| mov rbp, rsp |
| SHADOW_ARGS_TO_STACK 5 |
| SAVE_XMM 7 |
| push rsi |
| push rdi |
| push rbx |
| GET_GOT rbx |
| ; end prolog |
| |
| ; read top row |
| mov edx, 8 |
| mov rsi, arg(2) ;above |
| movsxd rax, dword ptr arg(4) ;left_stride; |
| pxor xmm0, xmm0 |
| %ifidn %1, ssse3 |
| movdqa xmm3, [GLOBAL(dc_1024)] |
| %endif |
| movdqa xmm1, [rsi] |
| movdqa xmm2, xmm1 |
| punpcklbw xmm1, xmm0 |
| punpckhbw xmm2, xmm0 |
| |
| ; set up left ptrs ans subtract topleft |
| movd xmm4, [rsi-1] |
| mov rsi, arg(3) ;left |
| %ifidn %1, sse2 |
| punpcklbw xmm4, xmm0 |
| pshuflw xmm4, xmm4, 0x0 |
| punpcklqdq xmm4, xmm4 |
| %else |
| pshufb xmm4, xmm3 |
| %endif |
| psubw xmm1, xmm4 |
| psubw xmm2, xmm4 |
| |
| ; set up dest ptrs |
| mov rdi, arg(0) ;dst; |
| movsxd rcx, dword ptr arg(1) ;dst_stride |
| vp8_intra_pred_y_tm_%1_loop: |
| mov bl, [rsi] |
| movd xmm4, ebx |
| |
| mov bl, [rsi+rax] |
| movd xmm5, ebx |
| %ifidn %1, sse2 |
| punpcklbw xmm4, xmm0 |
| punpcklbw xmm5, xmm0 |
| pshuflw xmm4, xmm4, 0x0 |
| pshuflw xmm5, xmm5, 0x0 |
| punpcklqdq xmm4, xmm4 |
| punpcklqdq xmm5, xmm5 |
| %else |
| pshufb xmm4, xmm3 |
| pshufb xmm5, xmm3 |
| %endif |
| movdqa xmm6, xmm4 |
| movdqa xmm7, xmm5 |
| paddw xmm4, xmm1 |
| paddw xmm6, xmm2 |
| paddw xmm5, xmm1 |
| paddw xmm7, xmm2 |
| packuswb xmm4, xmm6 |
| packuswb xmm5, xmm7 |
| movdqa [rdi ], xmm4 |
| movdqa [rdi+rcx], xmm5 |
| lea rsi, [rsi+rax*2] |
| lea rdi, [rdi+rcx*2] |
| dec edx |
| jnz vp8_intra_pred_y_tm_%1_loop |
| |
| ; begin epilog |
| RESTORE_GOT |
| pop rbx |
| pop rdi |
| pop rsi |
| RESTORE_XMM |
| UNSHADOW_ARGS |
| pop rbp |
| ret |
| %endmacro |
| |
| vp8_intra_pred_y_tm sse2 |
| vp8_intra_pred_y_tm ssse3 |
| |
| ;void vp8_intra_pred_y_ve_sse2( |
| ; unsigned char *dst, |
| ; int dst_stride |
| ; unsigned char *above, |
| ; unsigned char *left, |
| ; int left_stride |
| ; ) |
| global sym(vp8_intra_pred_y_ve_sse2) PRIVATE |
| sym(vp8_intra_pred_y_ve_sse2): |
| push rbp |
| mov rbp, rsp |
| SHADOW_ARGS_TO_STACK 5 |
| push rsi |
| ; end prolog |
| |
| ;arg(3), arg(4) not used |
| |
| mov rax, arg(2) ;above; |
| mov rsi, 2 |
| movsxd rdx, dword ptr arg(1) ;dst_stride |
| |
| ; read from top |
| movdqa xmm1, [rax] |
| |
| ; write out |
| mov rax, arg(0) ;dst; |
| lea rcx, [rdx*3] |
| |
| .label |
| movdqa [rax ], xmm1 |
| movdqa [rax+rdx ], xmm1 |
| movdqa [rax+rdx*2], xmm1 |
| movdqa [rax+rcx ], xmm1 |
| lea rax, [rax+rdx*4] |
| movdqa [rax ], xmm1 |
| movdqa [rax+rdx ], xmm1 |
| movdqa [rax+rdx*2], xmm1 |
| movdqa [rax+rcx ], xmm1 |
| lea rax, [rax+rdx*4] |
| dec rsi |
| jnz .label |
| |
| ; begin epilog |
| pop rsi |
| UNSHADOW_ARGS |
| pop rbp |
| ret |
| |
| ;void vp8_intra_pred_y_ho_sse2( |
| ; unsigned char *dst, |
| ; int dst_stride |
| ; unsigned char *above, |
| ; unsigned char *left, |
| ; int left_stride, |
| ; ) |
| global sym(vp8_intra_pred_y_ho_sse2) PRIVATE |
| sym(vp8_intra_pred_y_ho_sse2): |
| push rbp |
| mov rbp, rsp |
| SHADOW_ARGS_TO_STACK 5 |
| push rsi |
| push rdi |
| push rbx |
| ; end prolog |
| |
| ;arg(2) not used |
| |
| ; read from left and write out |
| mov edx, 8 |
| mov rsi, arg(3) ;left; |
| movsxd rax, dword ptr arg(4) ;left_stride; |
| mov rdi, arg(0) ;dst; |
| movsxd rcx, dword ptr arg(1) ;dst_stride |
| |
| vp8_intra_pred_y_ho_sse2_loop: |
| mov bl, [rsi] |
| movd xmm0, ebx |
| mov bl, [rsi+rax] |
| movd xmm1, ebx |
| |
| ; FIXME use pshufb for ssse3 version |
| punpcklbw xmm0, xmm0 |
| punpcklbw xmm1, xmm1 |
| pshuflw xmm0, xmm0, 0x0 |
| pshuflw xmm1, xmm1, 0x0 |
| punpcklqdq xmm0, xmm0 |
| punpcklqdq xmm1, xmm1 |
| movdqa [rdi ], xmm0 |
| movdqa [rdi+rcx], xmm1 |
| lea rsi, [rsi+rax*2] |
| lea rdi, [rdi+rcx*2] |
| dec edx |
| jnz vp8_intra_pred_y_ho_sse2_loop |
| |
| ; begin epilog |
| pop rbx |
| pop rdi |
| pop rsi |
| UNSHADOW_ARGS |
| pop rbp |
| ret |
| |
| SECTION_RODATA |
| align 16 |
| dc_128: |
| times 16 db 128 |
| dc_4: |
| times 4 dw 4 |
| align 16 |
| dc_8: |
| times 8 dw 8 |
| align 16 |
| dc_1024: |
| times 8 dw 0x400 |
| align 16 |
| dc_00001111: |
| times 8 db 0 |
| times 8 db 1 |
