src/gallium/auxiliary/gallivm/lp_bld_ir_common.c - platform/external/mesa3d - Git at Google

 /**************************************************************************
  *
  * Copyright 2009 VMware, Inc.
  * Copyright 2007-2008 VMware, Inc.
  * All Rights Reserved.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the
  * "Software"), to deal in the Software without restriction, including
  * without limitation the rights to use, copy, modify, merge, publish,
  * distribute, sub license, and/or sell copies of the Software, and to
  * permit persons to whom the Software is furnished to do so, subject to
  * the following conditions:
  *
  * The above copyright notice and this permission notice (including the
  * next paragraph) shall be included in all copies or substantial portions
  * of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  *
  **************************************************************************/

 #include "util/u_memory.h"
 #include "lp_bld_type.h"
 #include "lp_bld_init.h"
 #include "lp_bld_flow.h"
 #include "lp_bld_ir_common.h"
 #include "lp_bld_logic.h"

 /*
  * Return the context for the current function.
  * (always 'main', if shader doesn't do any function calls)
  */
 static inline struct function_ctx *
 func_ctx(struct lp_exec_mask *mask)
 {
    assert(mask->function_stack_size > 0);
    assert(mask->function_stack_size <= LP_MAX_NUM_FUNCS);
    return &mask->function_stack[mask->function_stack_size - 1];
 }

 /*
  * Returns true if we're in a loop.
  * It's global, meaning that it returns true even if there's
  * no loop inside the current function, but we were inside
  * a loop inside another function, from which this one was called.
  */
 static inline boolean
 mask_has_loop(struct lp_exec_mask *mask)
 {
    int i;
    for (i = mask->function_stack_size - 1; i >= 0; --i) {
       const struct function_ctx *ctx = &mask->function_stack[i];
       if (ctx->loop_stack_size > 0)
          return TRUE;
    }
    return FALSE;
 }

 /*
  * Returns true if we're inside a switch statement.
  * It's global, meaning that it returns true even if there's
  * no switch in the current function, but we were inside
  * a switch inside another function, from which this one was called.
  */
 static inline boolean
 mask_has_switch(struct lp_exec_mask *mask)
 {
    int i;
    for (i = mask->function_stack_size - 1; i >= 0; --i) {
       const struct function_ctx *ctx = &mask->function_stack[i];
       if (ctx->switch_stack_size > 0)
          return TRUE;
    }
    return FALSE;
 }

 /*
  * Returns true if we're inside a conditional.
  * It's global, meaning that it returns true even if there's
  * no conditional in the current function, but we were inside
  * a conditional inside another function, from which this one was called.
  */
 static inline boolean
 mask_has_cond(struct lp_exec_mask *mask)
 {
    int i;
    for (i = mask->function_stack_size - 1; i >= 0; --i) {
       const struct function_ctx *ctx = &mask->function_stack[i];
       if (ctx->cond_stack_size > 0)
          return TRUE;
    }
    return FALSE;
 }

 void lp_exec_mask_update(struct lp_exec_mask *mask)
 {
    LLVMBuilderRef builder = mask->bld->gallivm->builder;
    boolean has_loop_mask = mask_has_loop(mask);
    boolean has_cond_mask = mask_has_cond(mask);
    boolean has_switch_mask = mask_has_switch(mask);
    boolean has_ret_mask = mask->function_stack_size > 1 ||
          mask->ret_in_main;

    if (has_loop_mask) {
       /*for loops we need to update the entire mask at runtime */
       LLVMValueRef tmp;
       assert(mask->break_mask);
       tmp = LLVMBuildAnd(builder,
                          mask->cont_mask,
                          mask->break_mask,
                          "maskcb");
       mask->exec_mask = LLVMBuildAnd(builder,
                                      mask->cond_mask,
                                      tmp,
                                      "maskfull");
    } else
       mask->exec_mask = mask->cond_mask;

    if (has_switch_mask) {
       mask->exec_mask = LLVMBuildAnd(builder,
                                      mask->exec_mask,
                                      mask->switch_mask,
                                      "switchmask");
    }

    if (has_ret_mask) {
       mask->exec_mask = LLVMBuildAnd(builder,
                                      mask->exec_mask,
                                      mask->ret_mask,
                                      "callmask");
    }

    mask->has_mask = (has_cond_mask ||
                      has_loop_mask ||
                      has_switch_mask ||
                      has_ret_mask);
 }

 /*
  * Initialize a function context at the specified index.
  */
 void
 lp_exec_mask_function_init(struct lp_exec_mask *mask, int function_idx)
 {
    LLVMTypeRef int_type = LLVMInt32TypeInContext(mask->bld->gallivm->context);
    LLVMBuilderRef builder = mask->bld->gallivm->builder;
    struct function_ctx *ctx =  &mask->function_stack[function_idx];

    ctx->cond_stack_size = 0;
    ctx->loop_stack_size = 0;
    ctx->bgnloop_stack_size = 0;
    ctx->switch_stack_size = 0;

    if (function_idx == 0) {
       ctx->ret_mask = mask->ret_mask;
    }

    ctx->loop_limiter = lp_build_alloca(mask->bld->gallivm,
                                        int_type, "looplimiter");
    LLVMBuildStore(
       builder,
       LLVMConstInt(int_type, LP_MAX_TGSI_LOOP_ITERATIONS, false),
       ctx->loop_limiter);
 }

 void lp_exec_mask_init(struct lp_exec_mask *mask, struct lp_build_context *bld)
 {
    mask->bld = bld;
    mask->has_mask = FALSE;
    mask->ret_in_main = FALSE;
    /* For the main function */
    mask->function_stack_size = 1;

    mask->int_vec_type = lp_build_int_vec_type(bld->gallivm, mask->bld->type);
    mask->exec_mask = mask->ret_mask = mask->break_mask = mask->cont_mask =
          mask->cond_mask = mask->switch_mask =
          LLVMConstAllOnes(mask->int_vec_type);

    mask->function_stack = CALLOC(LP_MAX_NUM_FUNCS,
                                  sizeof(mask->function_stack[0]));
    lp_exec_mask_function_init(mask, 0);
 }

 void
 lp_exec_mask_fini(struct lp_exec_mask *mask)
 {
    FREE(mask->function_stack);
 }

 /* stores val into an address pointed to by dst_ptr.
  * mask->exec_mask is used to figure out which bits of val
  * should be stored into the address
  * (0 means don't store this bit, 1 means do store).
  */
 void lp_exec_mask_store(struct lp_exec_mask *mask,
                         struct lp_build_context *bld_store,
                         LLVMValueRef val,
                         LLVMValueRef dst_ptr)
 {
    LLVMBuilderRef builder = mask->bld->gallivm->builder;
    LLVMValueRef exec_mask = mask->has_mask ? mask->exec_mask : NULL;

    assert(lp_check_value(bld_store->type, val));
    assert(LLVMGetTypeKind(LLVMTypeOf(dst_ptr)) == LLVMPointerTypeKind);
    assert(LLVMGetElementType(LLVMTypeOf(dst_ptr)) == LLVMTypeOf(val) ||
           LLVMGetTypeKind(LLVMGetElementType(LLVMTypeOf(dst_ptr))) == LLVMArrayTypeKind);

    if (exec_mask) {
       LLVMValueRef res, dst;

       dst = LLVMBuildLoad(builder, dst_ptr, "");
       res = lp_build_select(bld_store, exec_mask, val, dst);
       LLVMBuildStore(builder, res, dst_ptr);
    } else
       LLVMBuildStore(builder, val, dst_ptr);
 }

 void lp_exec_bgnloop_post_phi(struct lp_exec_mask *mask)
 {
    LLVMBuilderRef builder = mask->bld->gallivm->builder;
    struct function_ctx *ctx = func_ctx(mask);

    if (ctx->loop_stack_size != ctx->bgnloop_stack_size) {
       mask->break_mask = LLVMBuildLoad(builder, ctx->break_var, "");
       lp_exec_mask_update(mask);
       ctx->bgnloop_stack_size = ctx->loop_stack_size;
    }
 }

 void lp_exec_bgnloop(struct lp_exec_mask *mask, bool load)
 {
    LLVMBuilderRef builder = mask->bld->gallivm->builder;
    struct function_ctx *ctx = func_ctx(mask);

    if (ctx->loop_stack_size >= LP_MAX_TGSI_NESTING) {
       ++ctx->loop_stack_size;
       return;
    }

    ctx->break_type_stack[ctx->loop_stack_size + ctx->switch_stack_size] =
       ctx->break_type;
    ctx->break_type = LP_EXEC_MASK_BREAK_TYPE_LOOP;

    ctx->loop_stack[ctx->loop_stack_size].loop_block = ctx->loop_block;
    ctx->loop_stack[ctx->loop_stack_size].cont_mask = mask->cont_mask;
    ctx->loop_stack[ctx->loop_stack_size].break_mask = mask->break_mask;
    ctx->loop_stack[ctx->loop_stack_size].break_var = ctx->break_var;
    ++ctx->loop_stack_size;

    ctx->break_var = lp_build_alloca(mask->bld->gallivm, mask->int_vec_type, "");
    LLVMBuildStore(builder, mask->break_mask, ctx->break_var);

    ctx->loop_block = lp_build_insert_new_block(mask->bld->gallivm, "bgnloop");

    LLVMBuildBr(builder, ctx->loop_block);
    LLVMPositionBuilderAtEnd(builder, ctx->loop_block);

    if (load) {
       lp_exec_bgnloop_post_phi(mask);
    }
 }

 void lp_exec_endloop(struct gallivm_state *gallivm,
                      struct lp_exec_mask *mask)
 {
    LLVMBuilderRef builder = mask->bld->gallivm->builder;
    struct function_ctx *ctx = func_ctx(mask);
    LLVMBasicBlockRef endloop;
    LLVMTypeRef int_type = LLVMInt32TypeInContext(mask->bld->gallivm->context);
    LLVMTypeRef reg_type = LLVMIntTypeInContext(gallivm->context,
                                                mask->bld->type.width *
                                                mask->bld->type.length);
    LLVMValueRef i1cond, i2cond, icond, limiter;

    assert(mask->break_mask);

    assert(ctx->loop_stack_size);
    if (ctx->loop_stack_size > LP_MAX_TGSI_NESTING) {
       --ctx->loop_stack_size;
       --ctx->bgnloop_stack_size;
       return;
    }

    /*
     * Restore the cont_mask, but don't pop
     */
    mask->cont_mask = ctx->loop_stack[ctx->loop_stack_size - 1].cont_mask;
    lp_exec_mask_update(mask);

    /*
     * Unlike the continue mask, the break_mask must be preserved across loop
     * iterations
     */
    LLVMBuildStore(builder, mask->break_mask, ctx->break_var);

    /* Decrement the loop limiter */
    limiter = LLVMBuildLoad(builder, ctx->loop_limiter, "");

    limiter = LLVMBuildSub(
       builder,
       limiter,
       LLVMConstInt(int_type, 1, false),
       "");

    LLVMBuildStore(builder, limiter, ctx->loop_limiter);

    /* i1cond = (mask != 0) */
    i1cond = LLVMBuildICmp(
       builder,
       LLVMIntNE,
       LLVMBuildBitCast(builder, mask->exec_mask, reg_type, ""),
       LLVMConstNull(reg_type), "i1cond");

    /* i2cond = (looplimiter > 0) */
    i2cond = LLVMBuildICmp(
       builder,
       LLVMIntSGT,
       limiter,
       LLVMConstNull(int_type), "i2cond");

    /* if( i1cond && i2cond ) */
    icond = LLVMBuildAnd(builder, i1cond, i2cond, "");

    endloop = lp_build_insert_new_block(mask->bld->gallivm, "endloop");

    LLVMBuildCondBr(builder,
                    icond, ctx->loop_block, endloop);

    LLVMPositionBuilderAtEnd(builder, endloop);

    assert(ctx->loop_stack_size);
    --ctx->loop_stack_size;
    --ctx->bgnloop_stack_size;
    mask->cont_mask = ctx->loop_stack[ctx->loop_stack_size].cont_mask;
    mask->break_mask = ctx->loop_stack[ctx->loop_stack_size].break_mask;
    ctx->loop_block = ctx->loop_stack[ctx->loop_stack_size].loop_block;
    ctx->break_var = ctx->loop_stack[ctx->loop_stack_size].break_var;
    ctx->break_type = ctx->break_type_stack[ctx->loop_stack_size +
          ctx->switch_stack_size];

    lp_exec_mask_update(mask);
 }

 void lp_exec_mask_cond_push(struct lp_exec_mask *mask,
                             LLVMValueRef val)
 {
    LLVMBuilderRef builder = mask->bld->gallivm->builder;
    struct function_ctx *ctx = func_ctx(mask);

    if (ctx->cond_stack_size >= LP_MAX_TGSI_NESTING) {
       ctx->cond_stack_size++;
       return;
    }
    if (ctx->cond_stack_size == 0 && mask->function_stack_size == 1) {
       assert(mask->cond_mask == LLVMConstAllOnes(mask->int_vec_type));
    }
    ctx->cond_stack[ctx->cond_stack_size++] = mask->cond_mask;
    assert(LLVMTypeOf(val) == mask->int_vec_type);
    mask->cond_mask = LLVMBuildAnd(builder,
                                   mask->cond_mask,
                                   val,
                                   "");
    lp_exec_mask_update(mask);
 }

 void lp_exec_mask_cond_invert(struct lp_exec_mask *mask)
 {
    LLVMBuilderRef builder = mask->bld->gallivm->builder;
    struct function_ctx *ctx = func_ctx(mask);
    LLVMValueRef prev_mask;
    LLVMValueRef inv_mask;

    assert(ctx->cond_stack_size);
    if (ctx->cond_stack_size >= LP_MAX_TGSI_NESTING)
       return;
    prev_mask = ctx->cond_stack[ctx->cond_stack_size - 1];
    if (ctx->cond_stack_size == 1 && mask->function_stack_size == 1) {
       assert(prev_mask == LLVMConstAllOnes(mask->int_vec_type));
    }

    inv_mask = LLVMBuildNot(builder, mask->cond_mask, "");

    mask->cond_mask = LLVMBuildAnd(builder,
                                   inv_mask,
                                   prev_mask, "");
    lp_exec_mask_update(mask);
 }

 void lp_exec_mask_cond_pop(struct lp_exec_mask *mask)
 {
    struct function_ctx *ctx = func_ctx(mask);
    assert(ctx->cond_stack_size);
    --ctx->cond_stack_size;
    if (ctx->cond_stack_size >= LP_MAX_TGSI_NESTING)
       return;
    mask->cond_mask = ctx->cond_stack[ctx->cond_stack_size];
    lp_exec_mask_update(mask);
 }


 void lp_exec_continue(struct lp_exec_mask *mask)
 {
    LLVMBuilderRef builder = mask->bld->gallivm->builder;
    LLVMValueRef exec_mask = LLVMBuildNot(builder,
                                          mask->exec_mask,
                                          "");

    mask->cont_mask = LLVMBuildAnd(builder,
                                   mask->cont_mask,
                                   exec_mask, "");

    lp_exec_mask_update(mask);
 }

 void lp_exec_break(struct lp_exec_mask *mask, int *pc,
                    bool break_always)
 {
    LLVMBuilderRef builder = mask->bld->gallivm->builder;
    struct function_ctx *ctx = func_ctx(mask);

    if (ctx->break_type == LP_EXEC_MASK_BREAK_TYPE_LOOP) {
       LLVMValueRef exec_mask = LLVMBuildNot(builder,
                                             mask->exec_mask,
                                             "break");

       mask->break_mask = LLVMBuildAnd(builder,
                                       mask->break_mask,
                                       exec_mask, "break_full");
    }
    else {
       if (ctx->switch_in_default) {
          /*
           * stop default execution but only if this is an unconditional switch.
           * (The condition here is not perfect since dead code after break is
           * allowed but should be sufficient since false negatives are just
           * unoptimized - so we don't have to pre-evaluate that).
           */
          if(break_always && ctx->switch_pc) {
             if (pc)
                *pc = ctx->switch_pc;
             return;
          }
       }

       if (break_always) {
          mask->switch_mask = LLVMConstNull(mask->bld->int_vec_type);
       }
       else {
          LLVMValueRef exec_mask = LLVMBuildNot(builder,
                                                mask->exec_mask,
                                                "break");
          mask->switch_mask = LLVMBuildAnd(builder,
                                           mask->switch_mask,
                                           exec_mask, "break_switch");
       }
    }

    lp_exec_mask_update(mask);
 }
	/**************************************************************************
	*
	* Copyright 2009 VMware, Inc.
	* Copyright 2007-2008 VMware, Inc.
	* All Rights Reserved.
	*
	* Permission is hereby granted, free of charge, to any person obtaining a
	* copy of this software and associated documentation files (the
	* "Software"), to deal in the Software without restriction, including
	* without limitation the rights to use, copy, modify, merge, publish,
	* distribute, sub license, and/or sell copies of the Software, and to
	* permit persons to whom the Software is furnished to do so, subject to
	* the following conditions:
	*
	* The above copyright notice and this permission notice (including the
	* next paragraph) shall be included in all copies or substantial portions
	* of the Software.
	*
	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
	* OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
	* IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
	* ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
	* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
	* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
	*
	**************************************************************************/

	#include "util/u_memory.h"
	#include "lp_bld_type.h"
	#include "lp_bld_init.h"
	#include "lp_bld_flow.h"
	#include "lp_bld_ir_common.h"
	#include "lp_bld_logic.h"

	/*
	* Return the context for the current function.
	* (always 'main', if shader doesn't do any function calls)
	*/
	static inline struct function_ctx *
	func_ctx(struct lp_exec_mask *mask)
	{
	assert(mask->function_stack_size > 0);
	assert(mask->function_stack_size <= LP_MAX_NUM_FUNCS);
	return &mask->function_stack[mask->function_stack_size - 1];
	}

	/*
	* Returns true if we're in a loop.
	* It's global, meaning that it returns true even if there's
	* no loop inside the current function, but we were inside
	* a loop inside another function, from which this one was called.
	*/
	static inline boolean
	mask_has_loop(struct lp_exec_mask *mask)
	{
	int i;
	for (i = mask->function_stack_size - 1; i >= 0; --i) {
	const struct function_ctx *ctx = &mask->function_stack[i];
	if (ctx->loop_stack_size > 0)
	return TRUE;
	}
	return FALSE;
	}

	/*
	* Returns true if we're inside a switch statement.
	* It's global, meaning that it returns true even if there's
	* no switch in the current function, but we were inside
	* a switch inside another function, from which this one was called.
	*/
	static inline boolean
	mask_has_switch(struct lp_exec_mask *mask)
	{
	int i;
	for (i = mask->function_stack_size - 1; i >= 0; --i) {
	const struct function_ctx *ctx = &mask->function_stack[i];
	if (ctx->switch_stack_size > 0)
	return TRUE;
	}
	return FALSE;
	}

	/*
	* Returns true if we're inside a conditional.
	* It's global, meaning that it returns true even if there's
	* no conditional in the current function, but we were inside
	* a conditional inside another function, from which this one was called.
	*/
	static inline boolean
	mask_has_cond(struct lp_exec_mask *mask)
	{
	int i;
	for (i = mask->function_stack_size - 1; i >= 0; --i) {
	const struct function_ctx *ctx = &mask->function_stack[i];
	if (ctx->cond_stack_size > 0)
	return TRUE;
	}
	return FALSE;
	}

	void lp_exec_mask_update(struct lp_exec_mask *mask)
	{
	LLVMBuilderRef builder = mask->bld->gallivm->builder;
	boolean has_loop_mask = mask_has_loop(mask);
	boolean has_cond_mask = mask_has_cond(mask);
	boolean has_switch_mask = mask_has_switch(mask);
	boolean has_ret_mask = mask->function_stack_size > 1 \|\|
	mask->ret_in_main;

	if (has_loop_mask) {
	/for loops we need to update the entire mask at runtime /
	LLVMValueRef tmp;
	assert(mask->break_mask);
	tmp = LLVMBuildAnd(builder,
	mask->cont_mask,
	mask->break_mask,
	"maskcb");
	mask->exec_mask = LLVMBuildAnd(builder,
	mask->cond_mask,
	tmp,
	"maskfull");
	} else
	mask->exec_mask = mask->cond_mask;

	if (has_switch_mask) {
	mask->exec_mask = LLVMBuildAnd(builder,
	mask->exec_mask,
	mask->switch_mask,
	"switchmask");
	}

	if (has_ret_mask) {
	mask->exec_mask = LLVMBuildAnd(builder,
	mask->exec_mask,
	mask->ret_mask,
	"callmask");
	}

	mask->has_mask = (has_cond_mask \|\|
	has_loop_mask \|\|
	has_switch_mask \|\|
	has_ret_mask);
	}

	/*
	* Initialize a function context at the specified index.
	*/
	void
	lp_exec_mask_function_init(struct lp_exec_mask *mask, int function_idx)
	{
	LLVMTypeRef int_type = LLVMInt32TypeInContext(mask->bld->gallivm->context);
	LLVMBuilderRef builder = mask->bld->gallivm->builder;
	struct function_ctx *ctx = &mask->function_stack[function_idx];

	ctx->cond_stack_size = 0;
	ctx->loop_stack_size = 0;
	ctx->bgnloop_stack_size = 0;
	ctx->switch_stack_size = 0;

	if (function_idx == 0) {
	ctx->ret_mask = mask->ret_mask;
	}

	ctx->loop_limiter = lp_build_alloca(mask->bld->gallivm,
	int_type, "looplimiter");
	LLVMBuildStore(
	builder,
	LLVMConstInt(int_type, LP_MAX_TGSI_LOOP_ITERATIONS, false),
	ctx->loop_limiter);
	}

	void lp_exec_mask_init(struct lp_exec_mask mask, struct lp_build_context bld)
	{
	mask->bld = bld;
	mask->has_mask = FALSE;
	mask->ret_in_main = FALSE;
	/* For the main function */
	mask->function_stack_size = 1;

	mask->int_vec_type = lp_build_int_vec_type(bld->gallivm, mask->bld->type);
	mask->exec_mask = mask->ret_mask = mask->break_mask = mask->cont_mask =
	mask->cond_mask = mask->switch_mask =
	LLVMConstAllOnes(mask->int_vec_type);

	mask->function_stack = CALLOC(LP_MAX_NUM_FUNCS,
	sizeof(mask->function_stack[0]));
	lp_exec_mask_function_init(mask, 0);
	}

	void
	lp_exec_mask_fini(struct lp_exec_mask *mask)
	{
	FREE(mask->function_stack);
	}

	/* stores val into an address pointed to by dst_ptr.
	* mask->exec_mask is used to figure out which bits of val
	* should be stored into the address
	* (0 means don't store this bit, 1 means do store).
	*/
	void lp_exec_mask_store(struct lp_exec_mask *mask,
	struct lp_build_context *bld_store,
	LLVMValueRef val,
	LLVMValueRef dst_ptr)
	{
	LLVMBuilderRef builder = mask->bld->gallivm->builder;
	LLVMValueRef exec_mask = mask->has_mask ? mask->exec_mask : NULL;

	assert(lp_check_value(bld_store->type, val));
	assert(LLVMGetTypeKind(LLVMTypeOf(dst_ptr)) == LLVMPointerTypeKind);
	assert(LLVMGetElementType(LLVMTypeOf(dst_ptr)) == LLVMTypeOf(val) \|\|
	LLVMGetTypeKind(LLVMGetElementType(LLVMTypeOf(dst_ptr))) == LLVMArrayTypeKind);

	if (exec_mask) {
	LLVMValueRef res, dst;

	dst = LLVMBuildLoad(builder, dst_ptr, "");
	res = lp_build_select(bld_store, exec_mask, val, dst);
	LLVMBuildStore(builder, res, dst_ptr);
	} else
	LLVMBuildStore(builder, val, dst_ptr);
	}

	void lp_exec_bgnloop_post_phi(struct lp_exec_mask *mask)
	{
	LLVMBuilderRef builder = mask->bld->gallivm->builder;
	struct function_ctx *ctx = func_ctx(mask);

	if (ctx->loop_stack_size != ctx->bgnloop_stack_size) {
	mask->break_mask = LLVMBuildLoad(builder, ctx->break_var, "");
	lp_exec_mask_update(mask);
	ctx->bgnloop_stack_size = ctx->loop_stack_size;
	}
	}

	void lp_exec_bgnloop(struct lp_exec_mask *mask, bool load)
	{
	LLVMBuilderRef builder = mask->bld->gallivm->builder;
	struct function_ctx *ctx = func_ctx(mask);

	if (ctx->loop_stack_size >= LP_MAX_TGSI_NESTING) {
	++ctx->loop_stack_size;
	return;
	}

	ctx->break_type_stack[ctx->loop_stack_size + ctx->switch_stack_size] =
	ctx->break_type;
	ctx->break_type = LP_EXEC_MASK_BREAK_TYPE_LOOP;

	ctx->loop_stack[ctx->loop_stack_size].loop_block = ctx->loop_block;
	ctx->loop_stack[ctx->loop_stack_size].cont_mask = mask->cont_mask;
	ctx->loop_stack[ctx->loop_stack_size].break_mask = mask->break_mask;
	ctx->loop_stack[ctx->loop_stack_size].break_var = ctx->break_var;
	++ctx->loop_stack_size;

	ctx->break_var = lp_build_alloca(mask->bld->gallivm, mask->int_vec_type, "");
	LLVMBuildStore(builder, mask->break_mask, ctx->break_var);

	ctx->loop_block = lp_build_insert_new_block(mask->bld->gallivm, "bgnloop");

	LLVMBuildBr(builder, ctx->loop_block);
	LLVMPositionBuilderAtEnd(builder, ctx->loop_block);

	if (load) {
	lp_exec_bgnloop_post_phi(mask);
	}
	}

	void lp_exec_endloop(struct gallivm_state *gallivm,
	struct lp_exec_mask *mask)
	{
	LLVMBuilderRef builder = mask->bld->gallivm->builder;
	struct function_ctx *ctx = func_ctx(mask);
	LLVMBasicBlockRef endloop;
	LLVMTypeRef int_type = LLVMInt32TypeInContext(mask->bld->gallivm->context);
	LLVMTypeRef reg_type = LLVMIntTypeInContext(gallivm->context,
	mask->bld->type.width *
	mask->bld->type.length);
	LLVMValueRef i1cond, i2cond, icond, limiter;

	assert(mask->break_mask);

	assert(ctx->loop_stack_size);
	if (ctx->loop_stack_size > LP_MAX_TGSI_NESTING) {
	--ctx->loop_stack_size;
	--ctx->bgnloop_stack_size;
	return;
	}

	/*
	* Restore the cont_mask, but don't pop
	*/
	mask->cont_mask = ctx->loop_stack[ctx->loop_stack_size - 1].cont_mask;
	lp_exec_mask_update(mask);

	/*
	* Unlike the continue mask, the break_mask must be preserved across loop
	* iterations
	*/
	LLVMBuildStore(builder, mask->break_mask, ctx->break_var);

	/* Decrement the loop limiter */
	limiter = LLVMBuildLoad(builder, ctx->loop_limiter, "");

	limiter = LLVMBuildSub(
	builder,
	limiter,
	LLVMConstInt(int_type, 1, false),
	"");

	LLVMBuildStore(builder, limiter, ctx->loop_limiter);

	/* i1cond = (mask != 0) */
	i1cond = LLVMBuildICmp(
	builder,
	LLVMIntNE,
	LLVMBuildBitCast(builder, mask->exec_mask, reg_type, ""),
	LLVMConstNull(reg_type), "i1cond");

	/* i2cond = (looplimiter > 0) */
	i2cond = LLVMBuildICmp(
	builder,
	LLVMIntSGT,
	limiter,
	LLVMConstNull(int_type), "i2cond");

	/* if( i1cond && i2cond ) */
	icond = LLVMBuildAnd(builder, i1cond, i2cond, "");

	endloop = lp_build_insert_new_block(mask->bld->gallivm, "endloop");

	LLVMBuildCondBr(builder,
	icond, ctx->loop_block, endloop);

	LLVMPositionBuilderAtEnd(builder, endloop);

	assert(ctx->loop_stack_size);
	--ctx->loop_stack_size;
	--ctx->bgnloop_stack_size;
	mask->cont_mask = ctx->loop_stack[ctx->loop_stack_size].cont_mask;
	mask->break_mask = ctx->loop_stack[ctx->loop_stack_size].break_mask;
	ctx->loop_block = ctx->loop_stack[ctx->loop_stack_size].loop_block;
	ctx->break_var = ctx->loop_stack[ctx->loop_stack_size].break_var;
	ctx->break_type = ctx->break_type_stack[ctx->loop_stack_size +
	ctx->switch_stack_size];

	lp_exec_mask_update(mask);
	}

	void lp_exec_mask_cond_push(struct lp_exec_mask *mask,
	LLVMValueRef val)
	{
	LLVMBuilderRef builder = mask->bld->gallivm->builder;
	struct function_ctx *ctx = func_ctx(mask);

	if (ctx->cond_stack_size >= LP_MAX_TGSI_NESTING) {
	ctx->cond_stack_size++;
	return;
	}
	if (ctx->cond_stack_size == 0 && mask->function_stack_size == 1) {
	assert(mask->cond_mask == LLVMConstAllOnes(mask->int_vec_type));
	}
	ctx->cond_stack[ctx->cond_stack_size++] = mask->cond_mask;
	assert(LLVMTypeOf(val) == mask->int_vec_type);
	mask->cond_mask = LLVMBuildAnd(builder,
	mask->cond_mask,
	val,
	"");
	lp_exec_mask_update(mask);
	}

	void lp_exec_mask_cond_invert(struct lp_exec_mask *mask)
	{
	LLVMBuilderRef builder = mask->bld->gallivm->builder;
	struct function_ctx *ctx = func_ctx(mask);
	LLVMValueRef prev_mask;
	LLVMValueRef inv_mask;

	assert(ctx->cond_stack_size);
	if (ctx->cond_stack_size >= LP_MAX_TGSI_NESTING)
	return;
	prev_mask = ctx->cond_stack[ctx->cond_stack_size - 1];
	if (ctx->cond_stack_size == 1 && mask->function_stack_size == 1) {
	assert(prev_mask == LLVMConstAllOnes(mask->int_vec_type));
	}

	inv_mask = LLVMBuildNot(builder, mask->cond_mask, "");

	mask->cond_mask = LLVMBuildAnd(builder,
	inv_mask,
	prev_mask, "");
	lp_exec_mask_update(mask);
	}

	void lp_exec_mask_cond_pop(struct lp_exec_mask *mask)
	{
	struct function_ctx *ctx = func_ctx(mask);
	assert(ctx->cond_stack_size);
	--ctx->cond_stack_size;
	if (ctx->cond_stack_size >= LP_MAX_TGSI_NESTING)
	return;
	mask->cond_mask = ctx->cond_stack[ctx->cond_stack_size];
	lp_exec_mask_update(mask);
	}


	void lp_exec_continue(struct lp_exec_mask *mask)
	{
	LLVMBuilderRef builder = mask->bld->gallivm->builder;
	LLVMValueRef exec_mask = LLVMBuildNot(builder,
	mask->exec_mask,
	"");

	mask->cont_mask = LLVMBuildAnd(builder,
	mask->cont_mask,
	exec_mask, "");

	lp_exec_mask_update(mask);
	}

	void lp_exec_break(struct lp_exec_mask mask, int pc,
	bool break_always)
	{
	LLVMBuilderRef builder = mask->bld->gallivm->builder;
	struct function_ctx *ctx = func_ctx(mask);

	if (ctx->break_type == LP_EXEC_MASK_BREAK_TYPE_LOOP) {
	LLVMValueRef exec_mask = LLVMBuildNot(builder,
	mask->exec_mask,
	"break");

	mask->break_mask = LLVMBuildAnd(builder,
	mask->break_mask,
	exec_mask, "break_full");
	}
	else {
	if (ctx->switch_in_default) {
	/*
	* stop default execution but only if this is an unconditional switch.
	* (The condition here is not perfect since dead code after break is
	* allowed but should be sufficient since false negatives are just
	* unoptimized - so we don't have to pre-evaluate that).
	*/
	if(break_always && ctx->switch_pc) {
	if (pc)
	*pc = ctx->switch_pc;
	return;
	}
	}

	if (break_always) {
	mask->switch_mask = LLVMConstNull(mask->bld->int_vec_type);
	}
	else {
	LLVMValueRef exec_mask = LLVMBuildNot(builder,
	mask->exec_mask,
	"break");
	mask->switch_mask = LLVMBuildAnd(builder,
	mask->switch_mask,
	exec_mask, "break_switch");
	}
	}

	lp_exec_mask_update(mask);
	}