| /************************************************************************** |
| * |
| * Copyright 2009 VMware, Inc. |
| * All Rights Reserved. |
| * |
| * Permission is hereby granted, free of charge, to any person obtaining a |
| * copy of this software and associated documentation files (the |
| * "Software"), to deal in the Software without restriction, including |
| * without limitation the rights to use, copy, modify, merge, publish, |
| * distribute, sub license, and/or sell copies of the Software, and to |
| * permit persons to whom the Software is furnished to do so, subject to |
| * the following conditions: |
| * |
| * The above copyright notice and this permission notice (including the |
| * next paragraph) shall be included in all copies or substantial portions |
| * of the Software. |
| * |
| * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS |
| * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
| * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. |
| * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR |
| * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, |
| * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE |
| * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. |
| * |
| **************************************************************************/ |
| |
| /** |
| * @file |
| * Helper functions for swizzling/shuffling. |
| * |
| * @author Jose Fonseca <jfonseca@vmware.com> |
| */ |
| |
| |
| #include "util/u_debug.h" |
| |
| #include "lp_bld_type.h" |
| #include "lp_bld_const.h" |
| #include "lp_bld_init.h" |
| #include "lp_bld_logic.h" |
| #include "lp_bld_swizzle.h" |
| #include "lp_bld_pack.h" |
| |
| |
| LLVMValueRef |
| lp_build_broadcast(struct gallivm_state *gallivm, |
| LLVMTypeRef vec_type, |
| LLVMValueRef scalar) |
| { |
| LLVMValueRef res; |
| |
| if (LLVMGetTypeKind(vec_type) != LLVMVectorTypeKind) { |
| /* scalar */ |
| assert(vec_type == LLVMTypeOf(scalar)); |
| res = scalar; |
| } else { |
| LLVMBuilderRef builder = gallivm->builder; |
| const unsigned length = LLVMGetVectorSize(vec_type); |
| LLVMValueRef undef = LLVMGetUndef(vec_type); |
| LLVMTypeRef i32_type = LLVMInt32TypeInContext(gallivm->context); |
| |
| assert(LLVMGetElementType(vec_type) == LLVMTypeOf(scalar)); |
| |
| if (HAVE_LLVM >= 0x207) { |
| /* The shuffle vector is always made of int32 elements */ |
| LLVMTypeRef i32_vec_type = LLVMVectorType(i32_type, length); |
| res = LLVMBuildInsertElement(builder, undef, scalar, LLVMConstNull(i32_type), ""); |
| res = LLVMBuildShuffleVector(builder, res, undef, LLVMConstNull(i32_vec_type), ""); |
| } else { |
| /* XXX: The above path provokes a bug in LLVM 2.6 */ |
| unsigned i; |
| res = undef; |
| for(i = 0; i < length; ++i) { |
| LLVMValueRef index = lp_build_const_int32(gallivm, i); |
| res = LLVMBuildInsertElement(builder, res, scalar, index, ""); |
| } |
| } |
| } |
| |
| return res; |
| } |
| |
| |
| /** |
| * Broadcast |
| */ |
| LLVMValueRef |
| lp_build_broadcast_scalar(struct lp_build_context *bld, |
| LLVMValueRef scalar) |
| { |
| assert(lp_check_elem_type(bld->type, LLVMTypeOf(scalar))); |
| |
| return lp_build_broadcast(bld->gallivm, bld->vec_type, scalar); |
| } |
| |
| |
| /** |
| * Combined extract and broadcast (mere shuffle in most cases) |
| */ |
| LLVMValueRef |
| lp_build_extract_broadcast(struct gallivm_state *gallivm, |
| struct lp_type src_type, |
| struct lp_type dst_type, |
| LLVMValueRef vector, |
| LLVMValueRef index) |
| { |
| LLVMTypeRef i32t = LLVMInt32TypeInContext(gallivm->context); |
| LLVMValueRef res; |
| |
| assert(src_type.floating == dst_type.floating); |
| assert(src_type.width == dst_type.width); |
| |
| assert(lp_check_value(src_type, vector)); |
| assert(LLVMTypeOf(index) == i32t); |
| |
| if (src_type.length == 1) { |
| if (dst_type.length == 1) { |
| /* |
| * Trivial scalar -> scalar. |
| */ |
| |
| res = vector; |
| } |
| else { |
| /* |
| * Broadcast scalar -> vector. |
| */ |
| |
| res = lp_build_broadcast(gallivm, |
| lp_build_vec_type(gallivm, dst_type), |
| vector); |
| } |
| } |
| else { |
| if (dst_type.length > 1) { |
| /* |
| * shuffle - result can be of different length. |
| */ |
| |
| LLVMValueRef shuffle; |
| shuffle = lp_build_broadcast(gallivm, |
| LLVMVectorType(i32t, dst_type.length), |
| index); |
| res = LLVMBuildShuffleVector(gallivm->builder, vector, |
| LLVMGetUndef(lp_build_vec_type(gallivm, src_type)), |
| shuffle, ""); |
| } |
| else { |
| /* |
| * Trivial extract scalar from vector. |
| */ |
| res = LLVMBuildExtractElement(gallivm->builder, vector, index, ""); |
| } |
| } |
| |
| return res; |
| } |
| |
| |
| /** |
| * Swizzle one channel into all other three channels. |
| */ |
| LLVMValueRef |
| lp_build_swizzle_scalar_aos(struct lp_build_context *bld, |
| LLVMValueRef a, |
| unsigned channel) |
| { |
| LLVMBuilderRef builder = bld->gallivm->builder; |
| const struct lp_type type = bld->type; |
| const unsigned n = type.length; |
| unsigned i, j; |
| |
| if(a == bld->undef || a == bld->zero || a == bld->one) |
| return a; |
| |
| /* XXX: SSE3 has PSHUFB which should be better than bitmasks, but forcing |
| * using shuffles here actually causes worst results. More investigation is |
| * needed. */ |
| if (type.width >= 16) { |
| /* |
| * Shuffle. |
| */ |
| LLVMTypeRef elem_type = LLVMInt32TypeInContext(bld->gallivm->context); |
| LLVMValueRef shuffles[LP_MAX_VECTOR_LENGTH]; |
| |
| for(j = 0; j < n; j += 4) |
| for(i = 0; i < 4; ++i) |
| shuffles[j + i] = LLVMConstInt(elem_type, j + channel, 0); |
| |
| return LLVMBuildShuffleVector(builder, a, bld->undef, LLVMConstVector(shuffles, n), ""); |
| } |
| else { |
| /* |
| * Bit mask and recursive shifts |
| * |
| * XYZW XYZW .... XYZW <= input |
| * 0Y00 0Y00 .... 0Y00 |
| * YY00 YY00 .... YY00 |
| * YYYY YYYY .... YYYY <= output |
| */ |
| struct lp_type type4; |
| const char shifts[4][2] = { |
| { 1, 2}, |
| {-1, 2}, |
| { 1, -2}, |
| {-1, -2} |
| }; |
| unsigned i; |
| |
| a = LLVMBuildAnd(builder, a, |
| lp_build_const_mask_aos(bld->gallivm, |
| type, 1 << channel), ""); |
| |
| /* |
| * Build a type where each element is an integer that cover the four |
| * channels. |
| */ |
| |
| type4 = type; |
| type4.floating = FALSE; |
| type4.width *= 4; |
| type4.length /= 4; |
| |
| a = LLVMBuildBitCast(builder, a, lp_build_vec_type(bld->gallivm, type4), ""); |
| |
| for(i = 0; i < 2; ++i) { |
| LLVMValueRef tmp = NULL; |
| int shift = shifts[channel][i]; |
| |
| #ifdef PIPE_ARCH_LITTLE_ENDIAN |
| shift = -shift; |
| #endif |
| |
| if(shift > 0) |
| tmp = LLVMBuildLShr(builder, a, lp_build_const_int_vec(bld->gallivm, type4, shift*type.width), ""); |
| if(shift < 0) |
| tmp = LLVMBuildShl(builder, a, lp_build_const_int_vec(bld->gallivm, type4, -shift*type.width), ""); |
| |
| assert(tmp); |
| if(tmp) |
| a = LLVMBuildOr(builder, a, tmp, ""); |
| } |
| |
| return LLVMBuildBitCast(builder, a, lp_build_vec_type(bld->gallivm, type), ""); |
| } |
| } |
| |
| |
| LLVMValueRef |
| lp_build_swizzle_aos(struct lp_build_context *bld, |
| LLVMValueRef a, |
| const unsigned char swizzles[4]) |
| { |
| LLVMBuilderRef builder = bld->gallivm->builder; |
| const struct lp_type type = bld->type; |
| const unsigned n = type.length; |
| unsigned i, j; |
| |
| if (swizzles[0] == PIPE_SWIZZLE_RED && |
| swizzles[1] == PIPE_SWIZZLE_GREEN && |
| swizzles[2] == PIPE_SWIZZLE_BLUE && |
| swizzles[3] == PIPE_SWIZZLE_ALPHA) { |
| return a; |
| } |
| |
| if (swizzles[0] == swizzles[1] && |
| swizzles[1] == swizzles[2] && |
| swizzles[2] == swizzles[3]) { |
| switch (swizzles[0]) { |
| case PIPE_SWIZZLE_RED: |
| case PIPE_SWIZZLE_GREEN: |
| case PIPE_SWIZZLE_BLUE: |
| case PIPE_SWIZZLE_ALPHA: |
| return lp_build_swizzle_scalar_aos(bld, a, swizzles[0]); |
| case PIPE_SWIZZLE_ZERO: |
| return bld->zero; |
| case PIPE_SWIZZLE_ONE: |
| return bld->one; |
| case LP_BLD_SWIZZLE_DONTCARE: |
| return bld->undef; |
| default: |
| assert(0); |
| return bld->undef; |
| } |
| } |
| |
| if (type.width >= 16) { |
| /* |
| * Shuffle. |
| */ |
| LLVMValueRef undef = LLVMGetUndef(lp_build_elem_type(bld->gallivm, type)); |
| LLVMTypeRef i32t = LLVMInt32TypeInContext(bld->gallivm->context); |
| LLVMValueRef shuffles[LP_MAX_VECTOR_LENGTH]; |
| LLVMValueRef aux[LP_MAX_VECTOR_LENGTH]; |
| |
| memset(aux, 0, sizeof aux); |
| |
| for(j = 0; j < n; j += 4) { |
| for(i = 0; i < 4; ++i) { |
| unsigned shuffle; |
| switch (swizzles[i]) { |
| default: |
| assert(0); |
| /* fall through */ |
| case PIPE_SWIZZLE_RED: |
| case PIPE_SWIZZLE_GREEN: |
| case PIPE_SWIZZLE_BLUE: |
| case PIPE_SWIZZLE_ALPHA: |
| shuffle = j + swizzles[i]; |
| shuffles[j + i] = LLVMConstInt(i32t, shuffle, 0); |
| break; |
| case PIPE_SWIZZLE_ZERO: |
| shuffle = type.length + 0; |
| shuffles[j + i] = LLVMConstInt(i32t, shuffle, 0); |
| if (!aux[0]) { |
| aux[0] = lp_build_const_elem(bld->gallivm, type, 0.0); |
| } |
| break; |
| case PIPE_SWIZZLE_ONE: |
| shuffle = type.length + 1; |
| shuffles[j + i] = LLVMConstInt(i32t, shuffle, 0); |
| if (!aux[1]) { |
| aux[1] = lp_build_const_elem(bld->gallivm, type, 1.0); |
| } |
| break; |
| case LP_BLD_SWIZZLE_DONTCARE: |
| shuffles[j + i] = LLVMGetUndef(i32t); |
| break; |
| } |
| } |
| } |
| |
| for (i = 0; i < n; ++i) { |
| if (!aux[i]) { |
| aux[i] = undef; |
| } |
| } |
| |
| return LLVMBuildShuffleVector(builder, a, |
| LLVMConstVector(aux, n), |
| LLVMConstVector(shuffles, n), ""); |
| } else { |
| /* |
| * Bit mask and shifts. |
| * |
| * For example, this will convert BGRA to RGBA by doing |
| * |
| * rgba = (bgra & 0x00ff0000) >> 16 |
| * | (bgra & 0xff00ff00) |
| * | (bgra & 0x000000ff) << 16 |
| * |
| * This is necessary not only for faster cause, but because X86 backend |
| * will refuse shuffles of <4 x i8> vectors |
| */ |
| LLVMValueRef res; |
| struct lp_type type4; |
| unsigned cond = 0; |
| unsigned chan; |
| int shift; |
| |
| /* |
| * Start with a mixture of 1 and 0. |
| */ |
| for (chan = 0; chan < 4; ++chan) { |
| if (swizzles[chan] == PIPE_SWIZZLE_ONE) { |
| cond |= 1 << chan; |
| } |
| } |
| res = lp_build_select_aos(bld, cond, bld->one, bld->zero); |
| |
| /* |
| * Build a type where each element is an integer that cover the four |
| * channels. |
| */ |
| type4 = type; |
| type4.floating = FALSE; |
| type4.width *= 4; |
| type4.length /= 4; |
| |
| a = LLVMBuildBitCast(builder, a, lp_build_vec_type(bld->gallivm, type4), ""); |
| res = LLVMBuildBitCast(builder, res, lp_build_vec_type(bld->gallivm, type4), ""); |
| |
| /* |
| * Mask and shift the channels, trying to group as many channels in the |
| * same shift as possible |
| */ |
| for (shift = -3; shift <= 3; ++shift) { |
| unsigned long long mask = 0; |
| |
| assert(type4.width <= sizeof(mask)*8); |
| |
| for (chan = 0; chan < 4; ++chan) { |
| /* FIXME: big endian */ |
| if (swizzles[chan] < 4 && |
| chan - swizzles[chan] == shift) { |
| mask |= ((1ULL << type.width) - 1) << (swizzles[chan] * type.width); |
| } |
| } |
| |
| if (mask) { |
| LLVMValueRef masked; |
| LLVMValueRef shifted; |
| |
| if (0) |
| debug_printf("shift = %i, mask = 0x%08llx\n", shift, mask); |
| |
| masked = LLVMBuildAnd(builder, a, |
| lp_build_const_int_vec(bld->gallivm, type4, mask), ""); |
| if (shift > 0) { |
| shifted = LLVMBuildShl(builder, masked, |
| lp_build_const_int_vec(bld->gallivm, type4, shift*type.width), ""); |
| } else if (shift < 0) { |
| shifted = LLVMBuildLShr(builder, masked, |
| lp_build_const_int_vec(bld->gallivm, type4, -shift*type.width), ""); |
| } else { |
| shifted = masked; |
| } |
| |
| res = LLVMBuildOr(builder, res, shifted, ""); |
| } |
| } |
| |
| return LLVMBuildBitCast(builder, res, |
| lp_build_vec_type(bld->gallivm, type), ""); |
| } |
| } |
| |
| |
| /** |
| * Extended swizzle of a single channel of a SoA vector. |
| * |
| * @param bld building context |
| * @param unswizzled array with the 4 unswizzled values |
| * @param swizzle one of the PIPE_SWIZZLE_* |
| * |
| * @return the swizzled value. |
| */ |
| LLVMValueRef |
| lp_build_swizzle_soa_channel(struct lp_build_context *bld, |
| const LLVMValueRef *unswizzled, |
| unsigned swizzle) |
| { |
| switch (swizzle) { |
| case PIPE_SWIZZLE_RED: |
| case PIPE_SWIZZLE_GREEN: |
| case PIPE_SWIZZLE_BLUE: |
| case PIPE_SWIZZLE_ALPHA: |
| return unswizzled[swizzle]; |
| case PIPE_SWIZZLE_ZERO: |
| return bld->zero; |
| case PIPE_SWIZZLE_ONE: |
| return bld->one; |
| default: |
| assert(0); |
| return bld->undef; |
| } |
| } |
| |
| |
| /** |
| * Extended swizzle of a SoA vector. |
| * |
| * @param bld building context |
| * @param unswizzled array with the 4 unswizzled values |
| * @param swizzles array of PIPE_SWIZZLE_* |
| * @param swizzled output swizzled values |
| */ |
| void |
| lp_build_swizzle_soa(struct lp_build_context *bld, |
| const LLVMValueRef *unswizzled, |
| const unsigned char swizzles[4], |
| LLVMValueRef *swizzled) |
| { |
| unsigned chan; |
| |
| for (chan = 0; chan < 4; ++chan) { |
| swizzled[chan] = lp_build_swizzle_soa_channel(bld, unswizzled, |
| swizzles[chan]); |
| } |
| } |
| |
| |
| /** |
| * Do an extended swizzle of a SoA vector inplace. |
| * |
| * @param bld building context |
| * @param values intput/output array with the 4 values |
| * @param swizzles array of PIPE_SWIZZLE_* |
| */ |
| void |
| lp_build_swizzle_soa_inplace(struct lp_build_context *bld, |
| LLVMValueRef *values, |
| const unsigned char swizzles[4]) |
| { |
| LLVMValueRef unswizzled[4]; |
| unsigned chan; |
| |
| for (chan = 0; chan < 4; ++chan) { |
| unswizzled[chan] = values[chan]; |
| } |
| |
| lp_build_swizzle_soa(bld, unswizzled, swizzles, values); |
| } |
| |
| |
| /** |
| * Transpose from AOS <-> SOA |
| * |
| * @param single_type_lp type of pixels |
| * @param src the 4 * n pixel input |
| * @param dst the 4 * n pixel output |
| */ |
| void |
| lp_build_transpose_aos(struct gallivm_state *gallivm, |
| struct lp_type single_type_lp, |
| const LLVMValueRef src[4], |
| LLVMValueRef dst[4]) |
| { |
| struct lp_type double_type_lp = single_type_lp; |
| LLVMTypeRef single_type; |
| LLVMTypeRef double_type; |
| LLVMValueRef t0, t1, t2, t3; |
| |
| double_type_lp.length >>= 1; |
| double_type_lp.width <<= 1; |
| |
| double_type = lp_build_vec_type(gallivm, double_type_lp); |
| single_type = lp_build_vec_type(gallivm, single_type_lp); |
| |
| /* Interleave x, y, z, w -> xy and zw */ |
| t0 = lp_build_interleave2_half(gallivm, single_type_lp, src[0], src[1], 0); |
| t1 = lp_build_interleave2_half(gallivm, single_type_lp, src[2], src[3], 0); |
| t2 = lp_build_interleave2_half(gallivm, single_type_lp, src[0], src[1], 1); |
| t3 = lp_build_interleave2_half(gallivm, single_type_lp, src[2], src[3], 1); |
| |
| /* Cast to double width type for second interleave */ |
| t0 = LLVMBuildBitCast(gallivm->builder, t0, double_type, "t0"); |
| t1 = LLVMBuildBitCast(gallivm->builder, t1, double_type, "t1"); |
| t2 = LLVMBuildBitCast(gallivm->builder, t2, double_type, "t2"); |
| t3 = LLVMBuildBitCast(gallivm->builder, t3, double_type, "t3"); |
| |
| /* Interleave xy, zw -> xyzw */ |
| dst[0] = lp_build_interleave2_half(gallivm, double_type_lp, t0, t1, 0); |
| dst[1] = lp_build_interleave2_half(gallivm, double_type_lp, t0, t1, 1); |
| dst[2] = lp_build_interleave2_half(gallivm, double_type_lp, t2, t3, 0); |
| dst[3] = lp_build_interleave2_half(gallivm, double_type_lp, t2, t3, 1); |
| |
| /* Cast back to original single width type */ |
| dst[0] = LLVMBuildBitCast(gallivm->builder, dst[0], single_type, "dst0"); |
| dst[1] = LLVMBuildBitCast(gallivm->builder, dst[1], single_type, "dst1"); |
| dst[2] = LLVMBuildBitCast(gallivm->builder, dst[2], single_type, "dst2"); |
| dst[3] = LLVMBuildBitCast(gallivm->builder, dst[3], single_type, "dst3"); |
| } |
| |
| |
| /** |
| * Pack first element of aos values, |
| * pad out to destination size. |
| * i.e. x1 _ _ _ x2 _ _ _ will become x1 x2 _ _ |
| */ |
| LLVMValueRef |
| lp_build_pack_aos_scalars(struct gallivm_state *gallivm, |
| struct lp_type src_type, |
| struct lp_type dst_type, |
| const LLVMValueRef src) |
| { |
| LLVMTypeRef i32t = LLVMInt32TypeInContext(gallivm->context); |
| LLVMValueRef undef = LLVMGetUndef(i32t); |
| LLVMValueRef shuffles[LP_MAX_VECTOR_LENGTH]; |
| unsigned num_src = src_type.length / 4; |
| unsigned num_dst = dst_type.length; |
| unsigned i; |
| |
| assert(num_src <= num_dst); |
| |
| for (i = 0; i < num_src; i++) { |
| shuffles[i] = LLVMConstInt(i32t, i * 4, 0); |
| } |
| for (i = num_src; i < num_dst; i++) { |
| shuffles[i] = undef; |
| } |
| |
| if (num_dst == 1) { |
| return LLVMBuildExtractElement(gallivm->builder, src, shuffles[0], ""); |
| } |
| else { |
| return LLVMBuildShuffleVector(gallivm->builder, src, src, |
| LLVMConstVector(shuffles, num_dst), ""); |
| } |
| } |
| |
| |
| /** |
| * Unpack and broadcast packed aos values consisting of only the |
| * first value, i.e. x1 x2 _ _ will become x1 x1 x1 x1 x2 x2 x2 x2 |
| */ |
| LLVMValueRef |
| lp_build_unpack_broadcast_aos_scalars(struct gallivm_state *gallivm, |
| struct lp_type src_type, |
| struct lp_type dst_type, |
| const LLVMValueRef src) |
| { |
| LLVMTypeRef i32t = LLVMInt32TypeInContext(gallivm->context); |
| LLVMValueRef shuffles[LP_MAX_VECTOR_LENGTH]; |
| unsigned num_dst = dst_type.length; |
| unsigned num_src = dst_type.length / 4; |
| unsigned i; |
| |
| assert(num_dst / 4 <= src_type.length); |
| |
| for (i = 0; i < num_src; i++) { |
| shuffles[i*4] = LLVMConstInt(i32t, i, 0); |
| shuffles[i*4+1] = LLVMConstInt(i32t, i, 0); |
| shuffles[i*4+2] = LLVMConstInt(i32t, i, 0); |
| shuffles[i*4+3] = LLVMConstInt(i32t, i, 0); |
| } |
| |
| if (num_src == 1) { |
| return lp_build_extract_broadcast(gallivm, src_type, dst_type, |
| src, shuffles[0]); |
| } |
| else { |
| return LLVMBuildShuffleVector(gallivm->builder, src, src, |
| LLVMConstVector(shuffles, num_dst), ""); |
| } |
| } |
| |
