/**************************************************************************
 *
 * Copyright 2019 Red Hat.
 * All Rights Reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 **************************************************************************/

#include "lp_bld_nir.h"
#include "lp_bld_arit.h"
#include "lp_bld_bitarit.h"
#include "lp_bld_const.h"
#include "lp_bld_gather.h"
#include "lp_bld_logic.h"
#include "lp_bld_quad.h"
#include "lp_bld_flow.h"
#include "lp_bld_struct.h"
#include "lp_bld_debug.h"
#include "lp_bld_printf.h"
#include "nir_deref.h"

static void visit_cf_list(struct lp_build_nir_context *bld_base,
                          struct exec_list *list);

static LLVMValueRef cast_type(struct lp_build_nir_context *bld_base, LLVMValueRef val,
                              nir_alu_type alu_type, unsigned bit_size)
{
   LLVMBuilderRef builder = bld_base->base.gallivm->builder;
   switch (alu_type) {
   case nir_type_float:
      switch (bit_size) {
      case 32:
         return LLVMBuildBitCast(builder, val, bld_base->base.vec_type, "");
      case 64:
         return LLVMBuildBitCast(builder, val, bld_base->dbl_bld.vec_type, "");
      default:
         assert(0);
         break;
      }
      break;
   case nir_type_int:
      switch (bit_size) {
      case 8:
         return LLVMBuildBitCast(builder, val, bld_base->int8_bld.vec_type, "");
      case 16:
         return LLVMBuildBitCast(builder, val, bld_base->int16_bld.vec_type, "");
      case 32:
         return LLVMBuildBitCast(builder, val, bld_base->int_bld.vec_type, "");
      case 64:
         return LLVMBuildBitCast(builder, val, bld_base->int64_bld.vec_type, "");
      default:
         assert(0);
         break;
      }
      break;
   case nir_type_uint:
      switch (bit_size) {
      case 8:
         return LLVMBuildBitCast(builder, val, bld_base->uint8_bld.vec_type, "");
      case 16:
         return LLVMBuildBitCast(builder, val, bld_base->uint16_bld.vec_type, "");
      case 32:
         return LLVMBuildBitCast(builder, val, bld_base->uint_bld.vec_type, "");
      case 64:
         return LLVMBuildBitCast(builder, val, bld_base->uint64_bld.vec_type, "");
      default:
         assert(0);
         break;
      }
      break;
   case nir_type_uint32:
      return LLVMBuildBitCast(builder, val, bld_base->uint_bld.vec_type, "");
   default:
      return val;
   }
   return NULL;
}


static struct lp_build_context *get_flt_bld(struct lp_build_nir_context *bld_base,
                                            unsigned op_bit_size)
{
   if (op_bit_size == 64)
      return &bld_base->dbl_bld;
   else
      return &bld_base->base;
}

static unsigned glsl_sampler_to_pipe(int sampler_dim, bool is_array)
{
   unsigned pipe_target = PIPE_BUFFER;
   switch (sampler_dim) {
   case GLSL_SAMPLER_DIM_1D:
      pipe_target = is_array ? PIPE_TEXTURE_1D_ARRAY : PIPE_TEXTURE_1D;
      break;
   case GLSL_SAMPLER_DIM_2D:
      pipe_target = is_array ? PIPE_TEXTURE_2D_ARRAY : PIPE_TEXTURE_2D;
      break;
   case GLSL_SAMPLER_DIM_3D:
      pipe_target = PIPE_TEXTURE_3D;
      break;
   case GLSL_SAMPLER_DIM_MS:
      pipe_target = is_array ? PIPE_TEXTURE_2D_ARRAY : PIPE_TEXTURE_2D;
      break;
   case GLSL_SAMPLER_DIM_CUBE:
      pipe_target = is_array ? PIPE_TEXTURE_CUBE_ARRAY : PIPE_TEXTURE_CUBE;
      break;
   case GLSL_SAMPLER_DIM_RECT:
      pipe_target = PIPE_TEXTURE_RECT;
      break;
   case GLSL_SAMPLER_DIM_BUF:
      pipe_target = PIPE_BUFFER;
      break;
   default:
      break;
   }
   return pipe_target;
}

static LLVMValueRef get_ssa_src(struct lp_build_nir_context *bld_base, nir_ssa_def *ssa)
{
   return bld_base->ssa_defs[ssa->index];
}

static LLVMValueRef get_src(struct lp_build_nir_context *bld_base, nir_src src);

static LLVMValueRef get_reg_src(struct lp_build_nir_context *bld_base, nir_reg_src src)
{
   struct hash_entry *entry = _mesa_hash_table_search(bld_base->regs, src.reg);
   LLVMValueRef reg_storage = (LLVMValueRef)entry->data;
   struct lp_build_context *reg_bld = get_int_bld(bld_base, true, src.reg->bit_size);
   LLVMValueRef indir_src = NULL;
   if (src.indirect)
      indir_src = get_src(bld_base, *src.indirect);
   return bld_base->load_reg(bld_base, reg_bld, &src, indir_src, reg_storage);
}

static LLVMValueRef get_src(struct lp_build_nir_context *bld_base, nir_src src)
{
   if (src.is_ssa)
      return get_ssa_src(bld_base, src.ssa);
   else
      return get_reg_src(bld_base, src.reg);
}

static void assign_ssa(struct lp_build_nir_context *bld_base, int idx, LLVMValueRef ptr)
{
   bld_base->ssa_defs[idx] = ptr;
}

static void assign_ssa_dest(struct lp_build_nir_context *bld_base, const nir_ssa_def *ssa,
                            LLVMValueRef vals[NIR_MAX_VEC_COMPONENTS])
{
   assign_ssa(bld_base, ssa->index, ssa->num_components == 1 ? vals[0] : lp_nir_array_build_gather_values(bld_base->base.gallivm->builder, vals, ssa->num_components));
}

static void assign_reg(struct lp_build_nir_context *bld_base, const nir_reg_dest *reg,
                       unsigned write_mask,
                       LLVMValueRef vals[NIR_MAX_VEC_COMPONENTS])
{
   struct hash_entry *entry = _mesa_hash_table_search(bld_base->regs, reg->reg);
   LLVMValueRef reg_storage = (LLVMValueRef)entry->data;
   struct lp_build_context *reg_bld = get_int_bld(bld_base, true, reg->reg->bit_size);
   LLVMValueRef indir_src = NULL;
   if (reg->indirect)
      indir_src = get_src(bld_base, *reg->indirect);
   bld_base->store_reg(bld_base, reg_bld, reg, write_mask ? write_mask : 0xf, indir_src, reg_storage, vals);
}

static void assign_dest(struct lp_build_nir_context *bld_base, const nir_dest *dest, LLVMValueRef vals[NIR_MAX_VEC_COMPONENTS])
{
   if (dest->is_ssa)
      assign_ssa_dest(bld_base, &dest->ssa, vals);
   else
      assign_reg(bld_base, &dest->reg, 0, vals);
}

static void assign_alu_dest(struct lp_build_nir_context *bld_base, const nir_alu_dest *dest, LLVMValueRef vals[NIR_MAX_VEC_COMPONENTS])
{
   if (dest->dest.is_ssa)
      assign_ssa_dest(bld_base, &dest->dest.ssa, vals);
   else
      assign_reg(bld_base, &dest->dest.reg, dest->write_mask, vals);
}

static LLVMValueRef int_to_bool32(struct lp_build_nir_context *bld_base,
                                uint32_t src_bit_size,
                                bool is_unsigned,
                                LLVMValueRef val)
{
   LLVMBuilderRef builder = bld_base->base.gallivm->builder;
   struct lp_build_context *int_bld = get_int_bld(bld_base, is_unsigned, src_bit_size);
   LLVMValueRef result = lp_build_compare(bld_base->base.gallivm, int_bld->type, PIPE_FUNC_NOTEQUAL, val, int_bld->zero);
   if (src_bit_size == 64)
      result = LLVMBuildTrunc(builder, result, bld_base->int_bld.vec_type, "");
   return result;
}

static LLVMValueRef flt_to_bool32(struct lp_build_nir_context *bld_base,
                                  uint32_t src_bit_size,
                                  LLVMValueRef val)
{
   LLVMBuilderRef builder = bld_base->base.gallivm->builder;
   struct lp_build_context *flt_bld = get_flt_bld(bld_base, src_bit_size);
   LLVMValueRef result = lp_build_cmp(flt_bld, PIPE_FUNC_NOTEQUAL, val, flt_bld->zero);
   if (src_bit_size == 64)
      result = LLVMBuildTrunc(builder, result, bld_base->int_bld.vec_type, "");
   return result;
}

static LLVMValueRef fcmp32(struct lp_build_nir_context *bld_base,
                           enum pipe_compare_func compare,
                           uint32_t src_bit_size,
                           LLVMValueRef src[NIR_MAX_VEC_COMPONENTS])
{
   LLVMBuilderRef builder = bld_base->base.gallivm->builder;
   struct lp_build_context *flt_bld = get_flt_bld(bld_base, src_bit_size);
   LLVMValueRef result;

   if (compare != PIPE_FUNC_NOTEQUAL)
      result = lp_build_cmp_ordered(flt_bld, compare, src[0], src[1]);
   else
      result = lp_build_cmp(flt_bld, compare, src[0], src[1]);
   if (src_bit_size == 64)
      result = LLVMBuildTrunc(builder, result, bld_base->int_bld.vec_type, "");
   return result;
}

static LLVMValueRef icmp32(struct lp_build_nir_context *bld_base,
                           enum pipe_compare_func compare,
                           bool is_unsigned,
                           uint32_t src_bit_size,
                           LLVMValueRef src[NIR_MAX_VEC_COMPONENTS])
{
   LLVMBuilderRef builder = bld_base->base.gallivm->builder;
   struct lp_build_context *i_bld = get_int_bld(bld_base, is_unsigned, src_bit_size);
   LLVMValueRef result = lp_build_cmp(i_bld, compare, src[0], src[1]);
   if (src_bit_size < 32)
      result = LLVMBuildSExt(builder, result, bld_base->int_bld.vec_type, "");
   else if (src_bit_size == 64)
      result = LLVMBuildTrunc(builder, result, bld_base->int_bld.vec_type, "");
   return result;
}

static LLVMValueRef get_alu_src(struct lp_build_nir_context *bld_base,
                                nir_alu_src src,
                                unsigned num_components)
{
   LLVMBuilderRef builder = bld_base->base.gallivm->builder;
   struct gallivm_state *gallivm = bld_base->base.gallivm;
   LLVMValueRef value = get_src(bld_base, src.src);
   bool need_swizzle = false;

   assert(value);
   unsigned src_components = nir_src_num_components(src.src);
   for (unsigned i = 0; i < num_components; ++i) {
      assert(src.swizzle[i] < src_components);
      if (src.swizzle[i] != i)
         need_swizzle = true;
   }

   if (need_swizzle || num_components != src_components) {
      if (src_components > 1 && num_components == 1) {
         value = LLVMBuildExtractValue(gallivm->builder, value,
                                       src.swizzle[0], "");
      } else if (src_components == 1 && num_components > 1) {
         LLVMValueRef values[] = {value, value, value, value, value, value, value, value, value, value, value, value, value, value, value, value};
         value = lp_nir_array_build_gather_values(builder, values, num_components);
      } else {
         LLVMValueRef arr = LLVMGetUndef(LLVMArrayType(LLVMTypeOf(LLVMBuildExtractValue(builder, value, 0, "")), num_components));
         for (unsigned i = 0; i < num_components; i++)
            arr = LLVMBuildInsertValue(builder, arr, LLVMBuildExtractValue(builder, value, src.swizzle[i], ""), i, "");
         value = arr;
      }
   }
   assert(!src.negate);
   assert(!src.abs);
   return value;
}

static LLVMValueRef emit_b2f(struct lp_build_nir_context *bld_base,
                             LLVMValueRef src0,
                             unsigned bitsize)
{
   LLVMBuilderRef builder = bld_base->base.gallivm->builder;
   LLVMValueRef result = LLVMBuildAnd(builder, cast_type(bld_base, src0, nir_type_int, 32),
                                      LLVMBuildBitCast(builder, lp_build_const_vec(bld_base->base.gallivm, bld_base->base.type,
                                                                                   1.0), bld_base->int_bld.vec_type, ""),
                                      "");
   result = LLVMBuildBitCast(builder, result, bld_base->base.vec_type, "");
   switch (bitsize) {
   case 32:
      break;
   case 64:
      result = LLVMBuildFPExt(builder, result, bld_base->dbl_bld.vec_type, "");
      break;
   default:
      unreachable("unsupported bit size.");
   }
   return result;
}

static LLVMValueRef emit_b2i(struct lp_build_nir_context *bld_base,
                             LLVMValueRef src0,
                             unsigned bitsize)
{
   LLVMBuilderRef builder = bld_base->base.gallivm->builder;
   LLVMValueRef result = LLVMBuildAnd(builder, cast_type(bld_base, src0, nir_type_int, 32),
                                      lp_build_const_int_vec(bld_base->base.gallivm, bld_base->base.type, 1), "");
   switch (bitsize) {
   case 32:
      return result;
   case 64:
      return LLVMBuildZExt(builder, result, bld_base->int64_bld.vec_type, "");
   default:
      unreachable("unsupported bit size.");
   }
}

static LLVMValueRef emit_b32csel(struct lp_build_nir_context *bld_base,
                               unsigned src_bit_size[NIR_MAX_VEC_COMPONENTS],
                               LLVMValueRef src[NIR_MAX_VEC_COMPONENTS])
{
   LLVMValueRef sel = cast_type(bld_base, src[0], nir_type_int, 32);
   LLVMValueRef v = lp_build_compare(bld_base->base.gallivm, bld_base->int_bld.type, PIPE_FUNC_NOTEQUAL, sel, bld_base->int_bld.zero);
   struct lp_build_context *bld = get_int_bld(bld_base, false, src_bit_size[1]);
   return lp_build_select(bld, v, src[1], src[2]);
}

static LLVMValueRef split_64bit(struct lp_build_nir_context *bld_base,
                                LLVMValueRef src,
                                bool hi)
{
   struct gallivm_state *gallivm = bld_base->base.gallivm;
   LLVMValueRef shuffles[LP_MAX_VECTOR_WIDTH/32];
   LLVMValueRef shuffles2[LP_MAX_VECTOR_WIDTH/32];
   int len = bld_base->base.type.length * 2;
   for (unsigned i = 0; i < bld_base->base.type.length; i++) {
#if UTIL_ARCH_LITTLE_ENDIAN
      shuffles[i] = lp_build_const_int32(gallivm, i * 2);
      shuffles2[i] = lp_build_const_int32(gallivm, (i * 2) + 1);
#else
      shuffles[i] = lp_build_const_int32(gallivm, (i * 2) + 1);
      shuffles2[i] = lp_build_const_int32(gallivm, (i * 2));
#endif
   }

   src = LLVMBuildBitCast(gallivm->builder, src, LLVMVectorType(LLVMInt32TypeInContext(gallivm->context), len), "");
   return LLVMBuildShuffleVector(gallivm->builder, src,
                                 LLVMGetUndef(LLVMTypeOf(src)),
                                 LLVMConstVector(hi ? shuffles2 : shuffles,
                                                 bld_base->base.type.length),
                                 "");
}

static LLVMValueRef
merge_64bit(struct lp_build_nir_context *bld_base,
            LLVMValueRef input,
            LLVMValueRef input2)
{
   struct gallivm_state *gallivm = bld_base->base.gallivm;
   LLVMBuilderRef builder = gallivm->builder;
   int i;
   LLVMValueRef shuffles[2 * (LP_MAX_VECTOR_WIDTH/32)];
   int len = bld_base->base.type.length * 2;
   assert(len <= (2 * (LP_MAX_VECTOR_WIDTH/32)));

   for (i = 0; i < bld_base->base.type.length * 2; i+=2) {
#if UTIL_ARCH_LITTLE_ENDIAN
      shuffles[i] = lp_build_const_int32(gallivm, i / 2);
      shuffles[i + 1] = lp_build_const_int32(gallivm, i / 2 + bld_base->base.type.length);
#else
      shuffles[i] = lp_build_const_int32(gallivm, i / 2 + bld_base->base.type.length);
      shuffles[i + 1] = lp_build_const_int32(gallivm, i / 2);
#endif
   }
   return LLVMBuildShuffleVector(builder, input, input2, LLVMConstVector(shuffles, len), "");
}

static LLVMValueRef
do_int_divide(struct lp_build_nir_context *bld_base,
              bool is_unsigned, unsigned src_bit_size,
              LLVMValueRef src, LLVMValueRef src2)
{
   struct gallivm_state *gallivm = bld_base->base.gallivm;
   LLVMBuilderRef builder = gallivm->builder;
   struct lp_build_context *int_bld = get_int_bld(bld_base, is_unsigned, src_bit_size);
   struct lp_build_context *mask_bld = get_int_bld(bld_base, true, src_bit_size);
   LLVMValueRef div_mask = lp_build_cmp(mask_bld, PIPE_FUNC_EQUAL, src2,
                                        mask_bld->zero);

   if (!is_unsigned) {
      /* INT_MIN (0x80000000) / -1 (0xffffffff) causes sigfpe, seen with blender. */
      div_mask = LLVMBuildAnd(builder, div_mask, lp_build_const_int_vec(gallivm, int_bld->type, 0x7fffffff), "");
   }
   LLVMValueRef divisor = LLVMBuildOr(builder,
                                      div_mask,
                                      src2, "");
   LLVMValueRef result = lp_build_div(int_bld, src, divisor);

   if (!is_unsigned) {
      LLVMValueRef not_div_mask = LLVMBuildNot(builder, div_mask, "");
      return LLVMBuildAnd(builder, not_div_mask, result, "");
   } else
      /* udiv by zero is guaranteed to return 0xffffffff at least with d3d10
       * may as well do same for idiv */
      return LLVMBuildOr(builder, div_mask, result, "");
}

static LLVMValueRef
do_int_mod(struct lp_build_nir_context *bld_base,
           bool is_unsigned, unsigned src_bit_size,
           LLVMValueRef src, LLVMValueRef src2)
{
   struct gallivm_state *gallivm = bld_base->base.gallivm;
   LLVMBuilderRef builder = gallivm->builder;
   struct lp_build_context *int_bld = get_int_bld(bld_base, is_unsigned, src_bit_size);
   LLVMValueRef div_mask = lp_build_cmp(int_bld, PIPE_FUNC_EQUAL, src2,
                                        int_bld->zero);
   LLVMValueRef divisor = LLVMBuildOr(builder,
                                      div_mask,
                                      src2, "");
   LLVMValueRef result = lp_build_mod(int_bld, src, divisor);
   return LLVMBuildOr(builder, div_mask, result, "");
}

static LLVMValueRef do_alu_action(struct lp_build_nir_context *bld_base,
                                  nir_op op, unsigned src_bit_size[NIR_MAX_VEC_COMPONENTS], LLVMValueRef src[NIR_MAX_VEC_COMPONENTS])
{
   struct gallivm_state *gallivm = bld_base->base.gallivm;
   LLVMBuilderRef builder = gallivm->builder;
   LLVMValueRef result;
   switch (op) {
   case nir_op_b2f32:
      result = emit_b2f(bld_base, src[0], 32);
      break;
   case nir_op_b2f64:
      result = emit_b2f(bld_base, src[0], 64);
      break;
   case nir_op_b2i32:
      result = emit_b2i(bld_base, src[0], 32);
      break;
   case nir_op_b2i64:
      result = emit_b2i(bld_base, src[0], 64);
      break;
   case nir_op_b32csel:
      result = emit_b32csel(bld_base, src_bit_size, src);
      break;
   case nir_op_bit_count:
      result = lp_build_popcount(get_int_bld(bld_base, false, src_bit_size[0]), src[0]);
      break;
   case nir_op_bitfield_select:
      result = lp_build_xor(&bld_base->uint_bld, src[2], lp_build_and(&bld_base->uint_bld, src[0], lp_build_xor(&bld_base->uint_bld, src[1], src[2])));
      break;
   case nir_op_bitfield_reverse:
      result = lp_build_bitfield_reverse(get_int_bld(bld_base, false, src_bit_size[0]), src[0]);
      break;
   case nir_op_f2b32:
      result = flt_to_bool32(bld_base, src_bit_size[0], src[0]);
      break;
   case nir_op_f2f32:
      result = LLVMBuildFPTrunc(builder, src[0],
                                bld_base->base.vec_type, "");
      break;
   case nir_op_f2f64:
      result = LLVMBuildFPExt(builder, src[0],
                              bld_base->dbl_bld.vec_type, "");
      break;
   case nir_op_f2i32:
      result = LLVMBuildFPToSI(builder, src[0], bld_base->base.int_vec_type, "");
      break;
   case nir_op_f2u32:
      result = LLVMBuildFPToUI(builder,
                               src[0],
                               bld_base->base.int_vec_type, "");
      break;
   case nir_op_f2i64:
      result = LLVMBuildFPToSI(builder,
                               src[0],
                               bld_base->int64_bld.vec_type, "");
      break;
   case nir_op_f2u64:
      result = LLVMBuildFPToUI(builder,
                               src[0],
                               bld_base->uint64_bld.vec_type, "");
      break;
   case nir_op_fabs:
      result = lp_build_abs(get_flt_bld(bld_base, src_bit_size[0]), src[0]);
      break;
   case nir_op_fadd:
      result = lp_build_add(get_flt_bld(bld_base, src_bit_size[0]),
                            src[0], src[1]);
      break;
   case nir_op_fceil:
      result = lp_build_ceil(get_flt_bld(bld_base, src_bit_size[0]), src[0]);
      break;
   case nir_op_fcos:
      result = lp_build_cos(&bld_base->base, src[0]);
      break;
   case nir_op_fddx:
   case nir_op_fddx_coarse:
   case nir_op_fddx_fine:
      result = lp_build_ddx(&bld_base->base, src[0]);
      break;
   case nir_op_fddy:
   case nir_op_fddy_coarse:
   case nir_op_fddy_fine:
      result = lp_build_ddy(&bld_base->base, src[0]);
      break;
   case nir_op_fdiv:
      result = lp_build_div(get_flt_bld(bld_base, src_bit_size[0]),
                            src[0], src[1]);
      break;
   case nir_op_feq32:
      result = fcmp32(bld_base, PIPE_FUNC_EQUAL, src_bit_size[0], src);
      break;
   case nir_op_fexp2:
      result = lp_build_exp2(&bld_base->base, src[0]);
      break;
   case nir_op_ffloor:
      result = lp_build_floor(get_flt_bld(bld_base, src_bit_size[0]), src[0]);
      break;
   case nir_op_ffma:
      result = lp_build_fmuladd(builder, src[0], src[1], src[2]);
      break;
   case nir_op_ffract: {
      struct lp_build_context *flt_bld = get_flt_bld(bld_base, src_bit_size[0]);
      LLVMValueRef tmp = lp_build_floor(flt_bld, src[0]);
      result = lp_build_sub(flt_bld, src[0], tmp);
      break;
   }
   case nir_op_fge32:
      result = fcmp32(bld_base, PIPE_FUNC_GEQUAL, src_bit_size[0], src);
      break;
   case nir_op_find_lsb:
      result = lp_build_cttz(get_int_bld(bld_base, false, src_bit_size[0]), src[0]);
      break;
   case nir_op_flog2:
      result = lp_build_log2_safe(&bld_base->base, src[0]);
      break;
   case nir_op_flt32:
      result = fcmp32(bld_base, PIPE_FUNC_LESS, src_bit_size[0], src);
      break;
   case nir_op_fmin:
      result = lp_build_min(get_flt_bld(bld_base, src_bit_size[0]), src[0], src[1]);
      break;
   case nir_op_fmod: {
      struct lp_build_context *flt_bld = get_flt_bld(bld_base, src_bit_size[0]);
      result = lp_build_div(flt_bld, src[0], src[1]);
      result = lp_build_floor(flt_bld, result);
      result = lp_build_mul(flt_bld, src[1], result);
      result = lp_build_sub(flt_bld, src[0], result);
      break;
   }
   case nir_op_fmul:
      result = lp_build_mul(get_flt_bld(bld_base, src_bit_size[0]),
                            src[0], src[1]);
      break;
   case nir_op_fmax:
      result = lp_build_max(get_flt_bld(bld_base, src_bit_size[0]), src[0], src[1]);
      break;
   case nir_op_fne32:
      result = fcmp32(bld_base, PIPE_FUNC_NOTEQUAL, src_bit_size[0], src);
      break;
   case nir_op_fneg:
      result = lp_build_negate(get_flt_bld(bld_base, src_bit_size[0]), src[0]);
      break;
   case nir_op_fpow:
      result = lp_build_pow(&bld_base->base, src[0], src[1]);
      break;
   case nir_op_frcp:
      result = lp_build_rcp(get_flt_bld(bld_base, src_bit_size[0]), src[0]);
      break;
   case nir_op_fround_even:
      result = lp_build_round(get_flt_bld(bld_base, src_bit_size[0]), src[0]);
      break;
   case nir_op_frsq:
      result = lp_build_rsqrt(get_flt_bld(bld_base, src_bit_size[0]), src[0]);
      break;
   case nir_op_fsat:
      result = lp_build_clamp_zero_one_nanzero(get_flt_bld(bld_base, src_bit_size[0]), src[0]);
      break;
   case nir_op_fsign:
      result = lp_build_sgn(get_flt_bld(bld_base, src_bit_size[0]), src[0]);
      break;
   case nir_op_fsin:
      result = lp_build_sin(&bld_base->base, src[0]);
      break;
   case nir_op_fsqrt:
      result = lp_build_sqrt(get_flt_bld(bld_base, src_bit_size[0]), src[0]);
      break;
   case nir_op_ftrunc:
      result = lp_build_trunc(get_flt_bld(bld_base, src_bit_size[0]), src[0]);
      break;
   case nir_op_i2b32:
      result = int_to_bool32(bld_base, src_bit_size[0], false, src[0]);
      break;
   case nir_op_i2f32:
      result = lp_build_int_to_float(&bld_base->base, src[0]);
      break;
   case nir_op_i2f64:
      result = lp_build_int_to_float(&bld_base->dbl_bld, src[0]);
      break;
   case nir_op_i2i8:
      result = LLVMBuildTrunc(builder, src[0], bld_base->int8_bld.vec_type, "");
      break;
   case nir_op_i2i16:
      if (src_bit_size[0] < 16)
         result = LLVMBuildSExt(builder, src[0], bld_base->int16_bld.vec_type, "");
      else
         result = LLVMBuildTrunc(builder, src[0], bld_base->int16_bld.vec_type, "");
      break;
   case nir_op_i2i32:
      if (src_bit_size[0] < 32)
         result = LLVMBuildSExt(builder, src[0], bld_base->int_bld.vec_type, "");
      else
         result = LLVMBuildTrunc(builder, src[0], bld_base->int_bld.vec_type, "");
      break;
   case nir_op_i2i64:
      result = LLVMBuildSExt(builder, src[0], bld_base->int64_bld.vec_type, "");
      break;
   case nir_op_iabs:
      result = lp_build_abs(get_int_bld(bld_base, false, src_bit_size[0]), src[0]);
      break;
   case nir_op_iadd:
      result = lp_build_add(get_int_bld(bld_base, false, src_bit_size[0]),
                            src[0], src[1]);
      break;
   case nir_op_iand:
      result = lp_build_and(get_int_bld(bld_base, false, src_bit_size[0]),
                            src[0], src[1]);
      break;
   case nir_op_idiv:
      result = do_int_divide(bld_base, false, src_bit_size[0], src[0], src[1]);
      break;
   case nir_op_ieq32:
      result = icmp32(bld_base, PIPE_FUNC_EQUAL, false, src_bit_size[0], src);
      break;
   case nir_op_ige32:
      result = icmp32(bld_base, PIPE_FUNC_GEQUAL, false, src_bit_size[0], src);
      break;
   case nir_op_ilt32:
      result = icmp32(bld_base, PIPE_FUNC_LESS, false, src_bit_size[0], src);
      break;
   case nir_op_imax:
      result = lp_build_max(get_int_bld(bld_base, false, src_bit_size[0]), src[0], src[1]);
      break;
   case nir_op_imin:
      result = lp_build_min(get_int_bld(bld_base, false, src_bit_size[0]), src[0], src[1]);
      break;
   case nir_op_imul:
   case nir_op_imul24:
      result = lp_build_mul(get_int_bld(bld_base, false, src_bit_size[0]),
                            src[0], src[1]);
      break;
   case nir_op_imul_high: {
      LLVMValueRef hi_bits;
      lp_build_mul_32_lohi(&bld_base->int_bld, src[0], src[1], &hi_bits);
      result = hi_bits;
      break;
   }
   case nir_op_ine32:
      result = icmp32(bld_base, PIPE_FUNC_NOTEQUAL, false, src_bit_size[0], src);
      break;
   case nir_op_ineg:
      result = lp_build_negate(get_int_bld(bld_base, false, src_bit_size[0]), src[0]);
      break;
   case nir_op_inot:
      result = lp_build_not(get_int_bld(bld_base, false, src_bit_size[0]), src[0]);
      break;
   case nir_op_ior:
      result = lp_build_or(get_int_bld(bld_base, false, src_bit_size[0]),
                           src[0], src[1]);
      break;
   case nir_op_irem:
      result = do_int_mod(bld_base, false, src_bit_size[0], src[0], src[1]);
      break;
   case nir_op_ishl: {
      struct lp_build_context *uint_bld = get_int_bld(bld_base, true, src_bit_size[0]);
      struct lp_build_context *int_bld = get_int_bld(bld_base, false, src_bit_size[0]);
      if (src_bit_size[0] == 64)
         src[1] = LLVMBuildZExt(builder, src[1], uint_bld->vec_type, "");
      if (src_bit_size[0] < 32)
         src[1] = LLVMBuildTrunc(builder, src[1], uint_bld->vec_type, "");
      src[1] = lp_build_and(uint_bld, src[1], lp_build_const_int_vec(gallivm, uint_bld->type, (src_bit_size[0] - 1)));
      result = lp_build_shl(int_bld, src[0], src[1]);
      break;
   }
   case nir_op_ishr: {
      struct lp_build_context *uint_bld = get_int_bld(bld_base, true, src_bit_size[0]);
      struct lp_build_context *int_bld = get_int_bld(bld_base, false, src_bit_size[0]);
      if (src_bit_size[0] == 64)
         src[1] = LLVMBuildZExt(builder, src[1], uint_bld->vec_type, "");
      if (src_bit_size[0] < 32)
         src[1] = LLVMBuildTrunc(builder, src[1], uint_bld->vec_type, "");
      src[1] = lp_build_and(uint_bld, src[1], lp_build_const_int_vec(gallivm, uint_bld->type, (src_bit_size[0] - 1)));
      result = lp_build_shr(int_bld, src[0], src[1]);
      break;
   }
   case nir_op_isign:
      result = lp_build_sgn(get_int_bld(bld_base, false, src_bit_size[0]), src[0]);
      break;
   case nir_op_isub:
      result = lp_build_sub(get_int_bld(bld_base, false, src_bit_size[0]),
                            src[0], src[1]);
      break;
   case nir_op_ixor:
      result = lp_build_xor(get_int_bld(bld_base, false, src_bit_size[0]),
                            src[0], src[1]);
      break;
   case nir_op_mov:
      result = src[0];
      break;
   case nir_op_unpack_64_2x32_split_x:
      result = split_64bit(bld_base, src[0], false);
      break;
   case nir_op_unpack_64_2x32_split_y:
      result = split_64bit(bld_base, src[0], true);
      break;

   case nir_op_pack_64_2x32_split: {
      LLVMValueRef tmp = merge_64bit(bld_base, src[0], src[1]);
      result = LLVMBuildBitCast(builder, tmp, bld_base->dbl_bld.vec_type, "");
      break;
   }
   case nir_op_u2f32:
      result = LLVMBuildUIToFP(builder, src[0], bld_base->base.vec_type, "");
      break;
   case nir_op_u2f64:
      result = LLVMBuildUIToFP(builder, src[0], bld_base->dbl_bld.vec_type, "");
      break;
   case nir_op_u2u8:
      result = LLVMBuildTrunc(builder, src[0], bld_base->uint8_bld.vec_type, "");
      break;
   case nir_op_u2u16:
      if (src_bit_size[0] < 16)
         result = LLVMBuildZExt(builder, src[0], bld_base->uint16_bld.vec_type, "");
      else
         result = LLVMBuildTrunc(builder, src[0], bld_base->uint16_bld.vec_type, "");
      break;
   case nir_op_u2u32:
      if (src_bit_size[0] < 32)
         result = LLVMBuildZExt(builder, src[0], bld_base->uint_bld.vec_type, "");
      else
         result = LLVMBuildTrunc(builder, src[0], bld_base->uint_bld.vec_type, "");
      break;
   case nir_op_u2u64:
      result = LLVMBuildZExt(builder, src[0], bld_base->uint64_bld.vec_type, "");
      break;
   case nir_op_udiv:
      result = do_int_divide(bld_base, true, src_bit_size[0], src[0], src[1]);
      break;
   case nir_op_ufind_msb: {
      struct lp_build_context *uint_bld = get_int_bld(bld_base, true, src_bit_size[0]);
      result = lp_build_ctlz(uint_bld, src[0]);
      result = lp_build_sub(uint_bld, lp_build_const_int_vec(gallivm, uint_bld->type, src_bit_size[0] - 1), result);
      break;
   }
   case nir_op_uge32:
      result = icmp32(bld_base, PIPE_FUNC_GEQUAL, true, src_bit_size[0], src);
      break;
   case nir_op_ult32:
      result = icmp32(bld_base, PIPE_FUNC_LESS, true, src_bit_size[0], src);
      break;
   case nir_op_umax:
      result = lp_build_max(get_int_bld(bld_base, true, src_bit_size[0]), src[0], src[1]);
      break;
   case nir_op_umin:
      result = lp_build_min(get_int_bld(bld_base, true, src_bit_size[0]), src[0], src[1]);
      break;
   case nir_op_umod:
      result = do_int_mod(bld_base, true, src_bit_size[0], src[0], src[1]);
      break;
   case nir_op_umul_high: {
      LLVMValueRef hi_bits;
      lp_build_mul_32_lohi(&bld_base->uint_bld, src[0], src[1], &hi_bits);
      result = hi_bits;
      break;
   }
   case nir_op_ushr: {
      struct lp_build_context *uint_bld = get_int_bld(bld_base, true, src_bit_size[0]);
      if (src_bit_size[0] == 64)
         src[1] = LLVMBuildZExt(builder, src[1], uint_bld->vec_type, "");
      if (src_bit_size[0] < 32)
         src[1] = LLVMBuildTrunc(builder, src[1], uint_bld->vec_type, "");
      src[1] = lp_build_and(uint_bld, src[1], lp_build_const_int_vec(gallivm, uint_bld->type, (src_bit_size[0] - 1)));
      result = lp_build_shr(uint_bld, src[0], src[1]);
      break;
   }
   default:
      assert(0);
      break;
   }
   return result;
}

static void visit_alu(struct lp_build_nir_context *bld_base, const nir_alu_instr *instr)
{
   struct gallivm_state *gallivm = bld_base->base.gallivm;
   LLVMValueRef src[NIR_MAX_VEC_COMPONENTS];
   unsigned src_bit_size[NIR_MAX_VEC_COMPONENTS];
   unsigned num_components = nir_dest_num_components(instr->dest.dest);
   unsigned src_components;
   switch (instr->op) {
   case nir_op_vec2:
   case nir_op_vec3:
   case nir_op_vec4:
   case nir_op_vec8:
   case nir_op_vec16:
      src_components = 1;
      break;
   case nir_op_pack_half_2x16:
      src_components = 2;
      break;
   case nir_op_unpack_half_2x16:
      src_components = 1;
      break;
   case nir_op_cube_face_coord:
   case nir_op_cube_face_index:
      src_components = 3;
      break;
   default:
      src_components = num_components;
      break;
   }
   for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++) {
      src[i] = get_alu_src(bld_base, instr->src[i], src_components);
      src_bit_size[i] = nir_src_bit_size(instr->src[i].src);
   }

   LLVMValueRef result[NIR_MAX_VEC_COMPONENTS];
   if (instr->op == nir_op_vec4 || instr->op == nir_op_vec3 || instr->op == nir_op_vec2 || instr->op == nir_op_vec8 || instr->op == nir_op_vec16) {
      for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++) {
         result[i] = cast_type(bld_base, src[i], nir_op_infos[instr->op].input_types[i], src_bit_size[i]);
      }
   } else {
      for (unsigned c = 0; c < num_components; c++) {
         LLVMValueRef src_chan[NIR_MAX_VEC_COMPONENTS];

         for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++) {
            if (num_components > 1) {
               src_chan[i] = LLVMBuildExtractValue(gallivm->builder,
                                                     src[i], c, "");
            } else
               src_chan[i] = src[i];
            src_chan[i] = cast_type(bld_base, src_chan[i], nir_op_infos[instr->op].input_types[i], src_bit_size[i]);
         }
         result[c] = do_alu_action(bld_base, instr->op, src_bit_size, src_chan);
         result[c] = cast_type(bld_base, result[c], nir_op_infos[instr->op].output_type, nir_dest_bit_size(instr->dest.dest));
      }
   }
   assign_alu_dest(bld_base, &instr->dest, result);
 }

static void visit_load_const(struct lp_build_nir_context *bld_base,
                             const nir_load_const_instr *instr)
{
   LLVMValueRef result[NIR_MAX_VEC_COMPONENTS];
   struct lp_build_context *int_bld = get_int_bld(bld_base, true, instr->def.bit_size);
   for (unsigned i = 0; i < instr->def.num_components; i++)
      result[i] = lp_build_const_int_vec(bld_base->base.gallivm, int_bld->type, instr->def.bit_size == 32 ? instr->value[i].u32 : instr->value[i].u64);
   assign_ssa_dest(bld_base, &instr->def, result);
}

static void
get_deref_offset(struct lp_build_nir_context *bld_base, nir_deref_instr *instr,
                 bool vs_in, unsigned *vertex_index_out,
                 LLVMValueRef *vertex_index_ref,
                 unsigned *const_out, LLVMValueRef *indir_out)
{
   LLVMBuilderRef builder = bld_base->base.gallivm->builder;
   nir_variable *var = nir_deref_instr_get_variable(instr);
   nir_deref_path path;
   unsigned idx_lvl = 1;

   nir_deref_path_init(&path, instr, NULL);

   if (vertex_index_out != NULL || vertex_index_ref != NULL) {
      if (vertex_index_ref) {
         *vertex_index_ref = get_src(bld_base, path.path[idx_lvl]->arr.index);
         if (vertex_index_out)
            *vertex_index_out = 0;
      } else {
         *vertex_index_out = nir_src_as_uint(path.path[idx_lvl]->arr.index);
      }
      ++idx_lvl;
   }

   uint32_t const_offset = 0;
   LLVMValueRef offset = NULL;

   if (var->data.compact) {
      assert(instr->deref_type == nir_deref_type_array);
      const_offset = nir_src_as_uint(instr->arr.index);
      goto out;
   }

   for (; path.path[idx_lvl]; ++idx_lvl) {
      const struct glsl_type *parent_type = path.path[idx_lvl - 1]->type;
      if (path.path[idx_lvl]->deref_type == nir_deref_type_struct) {
         unsigned index = path.path[idx_lvl]->strct.index;

         for (unsigned i = 0; i < index; i++) {
            const struct glsl_type *ft = glsl_get_struct_field(parent_type, i);
            const_offset += glsl_count_attribute_slots(ft, vs_in);
         }
      } else if(path.path[idx_lvl]->deref_type == nir_deref_type_array) {
         unsigned size = glsl_count_attribute_slots(path.path[idx_lvl]->type, vs_in);
         if (nir_src_is_const(path.path[idx_lvl]->arr.index)) {
           const_offset += nir_src_comp_as_int(path.path[idx_lvl]->arr.index, 0) * size;
         } else {
           LLVMValueRef idx_src = get_src(bld_base, path.path[idx_lvl]->arr.index);
           idx_src = cast_type(bld_base, idx_src, nir_type_uint, 32);
           LLVMValueRef array_off = lp_build_mul(&bld_base->uint_bld, lp_build_const_int_vec(bld_base->base.gallivm, bld_base->base.type, size),
                                               idx_src);
           if (offset)
             offset = lp_build_add(&bld_base->uint_bld, offset, array_off);
           else
             offset = array_off;
         }
      } else
         unreachable("Uhandled deref type in get_deref_instr_offset");
   }

out:
   nir_deref_path_finish(&path);

   if (const_offset && offset)
      offset = LLVMBuildAdd(builder, offset,
                            lp_build_const_int_vec(bld_base->base.gallivm, bld_base->uint_bld.type, const_offset),
                            "");
   *const_out = const_offset;
   *indir_out = offset;
}

static void visit_load_var(struct lp_build_nir_context *bld_base,
                           nir_intrinsic_instr *instr,
                           LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
{
   nir_deref_instr *deref = nir_instr_as_deref(instr->src[0].ssa->parent_instr);
   nir_variable *var = nir_deref_instr_get_variable(deref);
   nir_variable_mode mode = deref->mode;
   unsigned const_index;
   LLVMValueRef indir_index;
   LLVMValueRef indir_vertex_index = NULL;
   unsigned vertex_index = 0;
   unsigned nc = nir_dest_num_components(instr->dest);
   unsigned bit_size = nir_dest_bit_size(instr->dest);
   if (var) {
      bool vs_in = bld_base->shader->info.stage == MESA_SHADER_VERTEX &&
         var->data.mode == nir_var_shader_in;
      bool gs_in = bld_base->shader->info.stage == MESA_SHADER_GEOMETRY &&
         var->data.mode == nir_var_shader_in;
      bool tcs_in = bld_base->shader->info.stage == MESA_SHADER_TESS_CTRL &&
         var->data.mode == nir_var_shader_in;
      bool tcs_out = bld_base->shader->info.stage == MESA_SHADER_TESS_CTRL &&
         var->data.mode == nir_var_shader_out && !var->data.patch;
      bool tes_in = bld_base->shader->info.stage == MESA_SHADER_TESS_EVAL &&
         var->data.mode == nir_var_shader_in && !var->data.patch;

      mode = var->data.mode;

      get_deref_offset(bld_base, deref, vs_in, gs_in ? &vertex_index : NULL, (tcs_in || tcs_out || tes_in) ? &indir_vertex_index : NULL,
                       &const_index, &indir_index);
   }
   bld_base->load_var(bld_base, mode, nc, bit_size, var, vertex_index, indir_vertex_index, const_index, indir_index, result);
}

static void
visit_store_var(struct lp_build_nir_context *bld_base,
                nir_intrinsic_instr *instr)
{
   nir_deref_instr *deref = nir_instr_as_deref(instr->src[0].ssa->parent_instr);
   nir_variable *var = nir_deref_instr_get_variable(deref);
   nir_variable_mode mode = deref->mode;
   int writemask = instr->const_index[0];
   unsigned bit_size = nir_src_bit_size(instr->src[1]);
   LLVMValueRef src = get_src(bld_base, instr->src[1]);
   unsigned const_index = 0;
   LLVMValueRef indir_index, indir_vertex_index = NULL;
   if (var) {
      bool tcs_out = bld_base->shader->info.stage == MESA_SHADER_TESS_CTRL &&
         var->data.mode == nir_var_shader_out && !var->data.patch;
      get_deref_offset(bld_base, deref, false, NULL, tcs_out ? &indir_vertex_index : NULL,
                       &const_index, &indir_index);
   }
   bld_base->store_var(bld_base, mode, instr->num_components, bit_size, var, writemask, indir_vertex_index, const_index, indir_index, src);
}

static void visit_load_ubo(struct lp_build_nir_context *bld_base,
                           nir_intrinsic_instr *instr,
                           LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
{
   struct gallivm_state *gallivm = bld_base->base.gallivm;
   LLVMBuilderRef builder = gallivm->builder;
   LLVMValueRef idx = get_src(bld_base, instr->src[0]);
   LLVMValueRef offset = get_src(bld_base, instr->src[1]);

   bool offset_is_uniform = nir_src_is_dynamically_uniform(instr->src[1]);
   idx = LLVMBuildExtractElement(builder, idx, lp_build_const_int32(gallivm, 0), "");
   bld_base->load_ubo(bld_base, nir_dest_num_components(instr->dest), nir_dest_bit_size(instr->dest),
                      offset_is_uniform, idx, offset, result);
}


static void visit_load_ssbo(struct lp_build_nir_context *bld_base,
                           nir_intrinsic_instr *instr,
                           LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
{
   LLVMValueRef idx = get_src(bld_base, instr->src[0]);
   LLVMValueRef offset = get_src(bld_base, instr->src[1]);
   bld_base->load_mem(bld_base, nir_dest_num_components(instr->dest), nir_dest_bit_size(instr->dest),
                       idx, offset, result);
}

static void visit_store_ssbo(struct lp_build_nir_context *bld_base,
                             nir_intrinsic_instr *instr)
{
   LLVMValueRef val = get_src(bld_base, instr->src[0]);
   LLVMValueRef idx = get_src(bld_base, instr->src[1]);
   LLVMValueRef offset = get_src(bld_base, instr->src[2]);
   int writemask = instr->const_index[0];
   int nc = nir_src_num_components(instr->src[0]);
   int bitsize = nir_src_bit_size(instr->src[0]);
   bld_base->store_mem(bld_base, writemask, nc, bitsize, idx, offset, val);
}

static void visit_get_buffer_size(struct lp_build_nir_context *bld_base,
                                  nir_intrinsic_instr *instr,
                                  LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
{
   LLVMValueRef idx = get_src(bld_base, instr->src[0]);
   result[0] = bld_base->get_buffer_size(bld_base, idx);
}

static void visit_ssbo_atomic(struct lp_build_nir_context *bld_base,
                              nir_intrinsic_instr *instr,
                              LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
{
   LLVMValueRef idx = get_src(bld_base, instr->src[0]);
   LLVMValueRef offset = get_src(bld_base, instr->src[1]);
   LLVMValueRef val = get_src(bld_base, instr->src[2]);
   LLVMValueRef val2 = NULL;
   if (instr->intrinsic == nir_intrinsic_ssbo_atomic_comp_swap)
      val2 = get_src(bld_base, instr->src[3]);

   bld_base->atomic_mem(bld_base, instr->intrinsic, idx, offset, val, val2, &result[0]);

}

static void visit_load_image(struct lp_build_nir_context *bld_base,
                             nir_intrinsic_instr *instr,
                             LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
{
   struct gallivm_state *gallivm = bld_base->base.gallivm;
   LLVMBuilderRef builder = gallivm->builder;
   nir_deref_instr *deref = nir_instr_as_deref(instr->src[0].ssa->parent_instr);
   nir_variable *var = nir_deref_instr_get_variable(deref);
   LLVMValueRef coord_val = get_src(bld_base, instr->src[1]);
   LLVMValueRef coords[5];
   struct lp_img_params params;
   const struct glsl_type *type = glsl_without_array(var->type);

   memset(&params, 0, sizeof(params));
   params.target = glsl_sampler_to_pipe(glsl_get_sampler_dim(type), glsl_sampler_type_is_array(type));
   for (unsigned i = 0; i < 4; i++)
      coords[i] = LLVMBuildExtractValue(builder, coord_val, i, "");
   if (params.target == PIPE_TEXTURE_1D_ARRAY)
      coords[2] = coords[1];

   params.coords = coords;
   params.outdata = result;
   params.img_op = LP_IMG_LOAD;
   if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_MS)
      params.ms_index = get_src(bld_base, instr->src[2]);
   params.image_index = var->data.binding;
   bld_base->image_op(bld_base, &params);
}

static void visit_store_image(struct lp_build_nir_context *bld_base,
                              nir_intrinsic_instr *instr)
{
   struct gallivm_state *gallivm = bld_base->base.gallivm;
   LLVMBuilderRef builder = gallivm->builder;
   nir_deref_instr *deref = nir_instr_as_deref(instr->src[0].ssa->parent_instr);
   nir_variable *var = nir_deref_instr_get_variable(deref);
   LLVMValueRef coord_val = get_src(bld_base, instr->src[1]);
   LLVMValueRef in_val = get_src(bld_base, instr->src[3]);
   LLVMValueRef coords[5];
   struct lp_img_params params;
   const struct glsl_type *type = glsl_without_array(var->type);

   memset(&params, 0, sizeof(params));
   params.target = glsl_sampler_to_pipe(glsl_get_sampler_dim(type), glsl_sampler_type_is_array(type));
   for (unsigned i = 0; i < 4; i++)
      coords[i] = LLVMBuildExtractValue(builder, coord_val, i, "");
   if (params.target == PIPE_TEXTURE_1D_ARRAY)
      coords[2] = coords[1];
   params.coords = coords;

   for (unsigned i = 0; i < 4; i++) {
      params.indata[i] = LLVMBuildExtractValue(builder, in_val, i, "");
      params.indata[i] = LLVMBuildBitCast(builder, params.indata[i], bld_base->base.vec_type, "");
   }
   if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_MS)
      params.ms_index = get_src(bld_base, instr->src[2]);
   params.img_op = LP_IMG_STORE;
   params.image_index = var->data.binding;

   if (params.target == PIPE_TEXTURE_1D_ARRAY)
      coords[2] = coords[1];
   bld_base->image_op(bld_base, &params);
}

static void visit_atomic_image(struct lp_build_nir_context *bld_base,
                               nir_intrinsic_instr *instr,
                               LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
{
   struct gallivm_state *gallivm = bld_base->base.gallivm;
   LLVMBuilderRef builder = gallivm->builder;
   nir_deref_instr *deref = nir_instr_as_deref(instr->src[0].ssa->parent_instr);
   nir_variable *var = nir_deref_instr_get_variable(deref);
   struct lp_img_params params;
   LLVMValueRef coord_val = get_src(bld_base, instr->src[1]);
   LLVMValueRef in_val = get_src(bld_base, instr->src[3]);
   LLVMValueRef coords[5];
   const struct glsl_type *type = glsl_without_array(var->type);

   memset(&params, 0, sizeof(params));

   switch (instr->intrinsic) {
   case nir_intrinsic_image_deref_atomic_add:
      params.op = LLVMAtomicRMWBinOpAdd;
      break;
   case nir_intrinsic_image_deref_atomic_exchange:
      params.op = LLVMAtomicRMWBinOpXchg;
      break;
   case nir_intrinsic_image_deref_atomic_and:
      params.op = LLVMAtomicRMWBinOpAnd;
      break;
   case nir_intrinsic_image_deref_atomic_or:
      params.op = LLVMAtomicRMWBinOpOr;
      break;
   case nir_intrinsic_image_deref_atomic_xor:
      params.op = LLVMAtomicRMWBinOpXor;
      break;
   case nir_intrinsic_image_deref_atomic_umin:
      params.op = LLVMAtomicRMWBinOpUMin;
      break;
   case nir_intrinsic_image_deref_atomic_umax:
      params.op = LLVMAtomicRMWBinOpUMax;
      break;
   case nir_intrinsic_image_deref_atomic_imin:
      params.op = LLVMAtomicRMWBinOpMin;
      break;
   case nir_intrinsic_image_deref_atomic_imax:
      params.op = LLVMAtomicRMWBinOpMax;
      break;
   default:
      break;
   }

   params.target = glsl_sampler_to_pipe(glsl_get_sampler_dim(type), glsl_sampler_type_is_array(type));
   for (unsigned i = 0; i < 4; i++)
      coords[i] = LLVMBuildExtractValue(builder, coord_val, i, "");
   if (params.target == PIPE_TEXTURE_1D_ARRAY)
      coords[2] = coords[1];
   params.coords = coords;
   if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_MS)
      params.ms_index = get_src(bld_base, instr->src[2]);
   if (instr->intrinsic == nir_intrinsic_image_deref_atomic_comp_swap) {
      LLVMValueRef cas_val = get_src(bld_base, instr->src[4]);
      params.indata[0] = in_val;
      params.indata2[0] = cas_val;
   } else
      params.indata[0] = in_val;

   params.outdata = result;
   params.img_op = (instr->intrinsic == nir_intrinsic_image_deref_atomic_comp_swap) ? LP_IMG_ATOMIC_CAS : LP_IMG_ATOMIC;
   params.image_index = var->data.binding;

   bld_base->image_op(bld_base, &params);
}


static void visit_image_size(struct lp_build_nir_context *bld_base,
                             nir_intrinsic_instr *instr,
                             LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
{
   nir_deref_instr *deref = nir_instr_as_deref(instr->src[0].ssa->parent_instr);
   nir_variable *var = nir_deref_instr_get_variable(deref);
   struct lp_sampler_size_query_params params = { 0 };
   params.texture_unit = var->data.binding;
   params.target = glsl_sampler_to_pipe(glsl_get_sampler_dim(var->type), glsl_sampler_type_is_array(var->type));
   params.sizes_out = result;

   bld_base->image_size(bld_base, &params);
}

static void visit_image_samples(struct lp_build_nir_context *bld_base,
                                nir_intrinsic_instr *instr,
                                LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
{
   nir_deref_instr *deref = nir_instr_as_deref(instr->src[0].ssa->parent_instr);
   nir_variable *var = nir_deref_instr_get_variable(deref);
   struct lp_sampler_size_query_params params = { 0 };
   params.texture_unit = var->data.binding;
   params.target = glsl_sampler_to_pipe(glsl_get_sampler_dim(var->type), glsl_sampler_type_is_array(var->type));
   params.sizes_out = result;
   params.samples_only = true;

   bld_base->image_size(bld_base, &params);
}

static void visit_shared_load(struct lp_build_nir_context *bld_base,
                                nir_intrinsic_instr *instr,
                                LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
{
   LLVMValueRef offset = get_src(bld_base, instr->src[0]);
   bld_base->load_mem(bld_base, nir_dest_num_components(instr->dest), nir_dest_bit_size(instr->dest),
                      NULL, offset, result);
}

static void visit_shared_store(struct lp_build_nir_context *bld_base,
                               nir_intrinsic_instr *instr)
{
   LLVMValueRef val = get_src(bld_base, instr->src[0]);
   LLVMValueRef offset = get_src(bld_base, instr->src[1]);
   int writemask = instr->const_index[1];
   int nc = nir_src_num_components(instr->src[0]);
   int bitsize = nir_src_bit_size(instr->src[0]);
   bld_base->store_mem(bld_base, writemask, nc, bitsize, NULL, offset, val);
}

static void visit_shared_atomic(struct lp_build_nir_context *bld_base,
                                nir_intrinsic_instr *instr,
                                LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
{
   LLVMValueRef offset = get_src(bld_base, instr->src[0]);
   LLVMValueRef val = get_src(bld_base, instr->src[1]);
   LLVMValueRef val2 = NULL;
   if (instr->intrinsic == nir_intrinsic_shared_atomic_comp_swap)
      val2 = get_src(bld_base, instr->src[2]);

   bld_base->atomic_mem(bld_base, instr->intrinsic, NULL, offset, val, val2, &result[0]);

}

static void visit_barrier(struct lp_build_nir_context *bld_base)
{
   bld_base->barrier(bld_base);
}

static void visit_discard(struct lp_build_nir_context *bld_base,
                          nir_intrinsic_instr *instr)
{
   LLVMValueRef cond = NULL;
   if (instr->intrinsic == nir_intrinsic_discard_if) {
      cond = get_src(bld_base, instr->src[0]);
      cond = cast_type(bld_base, cond, nir_type_int, 32);
   }
   bld_base->discard(bld_base, cond);
}

static void visit_load_kernel_input(struct lp_build_nir_context *bld_base,
                                    nir_intrinsic_instr *instr, LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
{
   LLVMValueRef offset = get_src(bld_base, instr->src[0]);

   bool offset_is_uniform = nir_src_is_dynamically_uniform(instr->src[0]);
   bld_base->load_kernel_arg(bld_base, nir_dest_num_components(instr->dest), nir_dest_bit_size(instr->dest),
                             nir_src_bit_size(instr->src[0]),
                             offset_is_uniform, offset, result);
}

static void visit_load_global(struct lp_build_nir_context *bld_base,
                              nir_intrinsic_instr *instr, LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
{
   LLVMValueRef addr = get_src(bld_base, instr->src[0]);
   bld_base->load_global(bld_base, nir_dest_num_components(instr->dest), nir_dest_bit_size(instr->dest),
                         nir_src_bit_size(instr->src[0]),
                         addr, result);
}

static void visit_store_global(struct lp_build_nir_context *bld_base,
                               nir_intrinsic_instr *instr)
{
   LLVMValueRef val = get_src(bld_base, instr->src[0]);
   int nc = nir_src_num_components(instr->src[0]);
   int bitsize = nir_src_bit_size(instr->src[0]);
   LLVMValueRef addr = get_src(bld_base, instr->src[1]);
   int addr_bitsize = nir_src_bit_size(instr->src[1]);
   int writemask = instr->const_index[0];
   bld_base->store_global(bld_base, writemask, nc, bitsize, addr_bitsize, addr, val);
}

static void visit_global_atomic(struct lp_build_nir_context *bld_base,
                                nir_intrinsic_instr *instr,
                                LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
{
   LLVMValueRef addr = get_src(bld_base, instr->src[0]);
   LLVMValueRef val = get_src(bld_base, instr->src[1]);
   LLVMValueRef val2 = NULL;
   int addr_bitsize = nir_src_bit_size(instr->src[0]);
   if (instr->intrinsic == nir_intrinsic_global_atomic_comp_swap)
      val2 = get_src(bld_base, instr->src[2]);

   bld_base->atomic_global(bld_base, instr->intrinsic, addr_bitsize, addr, val, val2, &result[0]);
}

static void visit_interp(struct lp_build_nir_context *bld_base,
                         nir_intrinsic_instr *instr,
                         LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
{
   struct gallivm_state *gallivm = bld_base->base.gallivm;
   LLVMBuilderRef builder = gallivm->builder;
   nir_deref_instr *deref = nir_instr_as_deref(instr->src[0].ssa->parent_instr);
   unsigned num_components = nir_dest_num_components(instr->dest);
   nir_variable *var = nir_deref_instr_get_variable(deref);
   unsigned const_index;
   LLVMValueRef indir_index;
   LLVMValueRef offsets[2] = { NULL, NULL };
   get_deref_offset(bld_base, deref, false, NULL, NULL,
                    &const_index, &indir_index);
   bool centroid = instr->intrinsic == nir_intrinsic_interp_deref_at_centroid;
   bool sample = false;
   if (instr->intrinsic == nir_intrinsic_interp_deref_at_offset) {
      for (unsigned i = 0; i < 2; i++) {
         offsets[i] = LLVMBuildExtractValue(builder, get_src(bld_base, instr->src[1]), i, "");
         offsets[i] = cast_type(bld_base, offsets[i], nir_type_float, 32);
      }
   } else if (instr->intrinsic == nir_intrinsic_interp_deref_at_sample) {
      offsets[0] = get_src(bld_base, instr->src[1]);
      offsets[0] = cast_type(bld_base, offsets[0], nir_type_int, 32);
      sample = true;
   }
   bld_base->interp_at(bld_base, num_components, var, centroid, sample, const_index, indir_index, offsets, result);
}

static void visit_intrinsic(struct lp_build_nir_context *bld_base,
                            nir_intrinsic_instr *instr)
{
   LLVMValueRef result[NIR_MAX_VEC_COMPONENTS] = {0};
   switch (instr->intrinsic) {
   case nir_intrinsic_load_deref:
      visit_load_var(bld_base, instr, result);
      break;
   case nir_intrinsic_store_deref:
      visit_store_var(bld_base, instr);
      break;
   case nir_intrinsic_load_ubo:
      visit_load_ubo(bld_base, instr, result);
      break;
   case nir_intrinsic_load_ssbo:
      visit_load_ssbo(bld_base, instr, result);
      break;
   case nir_intrinsic_store_ssbo:
      visit_store_ssbo(bld_base, instr);
      break;
   case nir_intrinsic_get_buffer_size:
      visit_get_buffer_size(bld_base, instr, result);
      break;
   case nir_intrinsic_load_vertex_id:
   case nir_intrinsic_load_primitive_id:
   case nir_intrinsic_load_instance_id:
   case nir_intrinsic_load_base_instance:
   case nir_intrinsic_load_base_vertex:
   case nir_intrinsic_load_work_group_id:
   case nir_intrinsic_load_local_invocation_id:
   case nir_intrinsic_load_num_work_groups:
   case nir_intrinsic_load_invocation_id:
   case nir_intrinsic_load_front_face:
   case nir_intrinsic_load_draw_id:
   case nir_intrinsic_load_local_group_size:
   case nir_intrinsic_load_work_dim:
   case nir_intrinsic_load_tess_coord:
   case nir_intrinsic_load_tess_level_outer:
   case nir_intrinsic_load_tess_level_inner:
   case nir_intrinsic_load_patch_vertices_in:
   case nir_intrinsic_load_sample_id:
   case nir_intrinsic_load_sample_pos:
   case nir_intrinsic_load_sample_mask_in:
      bld_base->sysval_intrin(bld_base, instr, result);
      break;
   case nir_intrinsic_load_helper_invocation:
      bld_base->helper_invocation(bld_base, &result[0]);
      break;
   case nir_intrinsic_discard_if:
   case nir_intrinsic_discard:
      visit_discard(bld_base, instr);
      break;
   case nir_intrinsic_emit_vertex:
      bld_base->emit_vertex(bld_base, nir_intrinsic_stream_id(instr));
      break;
   case nir_intrinsic_end_primitive:
      bld_base->end_primitive(bld_base, nir_intrinsic_stream_id(instr));
      break;
   case nir_intrinsic_ssbo_atomic_add:
   case nir_intrinsic_ssbo_atomic_imin:
   case nir_intrinsic_ssbo_atomic_imax:
   case nir_intrinsic_ssbo_atomic_umin:
   case nir_intrinsic_ssbo_atomic_umax:
   case nir_intrinsic_ssbo_atomic_and:
   case nir_intrinsic_ssbo_atomic_or:
   case nir_intrinsic_ssbo_atomic_xor:
   case nir_intrinsic_ssbo_atomic_exchange:
   case nir_intrinsic_ssbo_atomic_comp_swap:
      visit_ssbo_atomic(bld_base, instr, result);
      break;
   case nir_intrinsic_image_deref_load:
      visit_load_image(bld_base, instr, result);
      break;
   case nir_intrinsic_image_deref_store:
      visit_store_image(bld_base, instr);
      break;
   case nir_intrinsic_image_deref_atomic_add:
   case nir_intrinsic_image_deref_atomic_imin:
   case nir_intrinsic_image_deref_atomic_imax:
   case nir_intrinsic_image_deref_atomic_umin:
   case nir_intrinsic_image_deref_atomic_umax:
   case nir_intrinsic_image_deref_atomic_and:
   case nir_intrinsic_image_deref_atomic_or:
   case nir_intrinsic_image_deref_atomic_xor:
   case nir_intrinsic_image_deref_atomic_exchange:
   case nir_intrinsic_image_deref_atomic_comp_swap:
      visit_atomic_image(bld_base, instr, result);
      break;
   case nir_intrinsic_image_deref_size:
      visit_image_size(bld_base, instr, result);
      break;
   case nir_intrinsic_image_deref_samples:
      visit_image_samples(bld_base, instr, result);
      break;
   case nir_intrinsic_load_shared:
      visit_shared_load(bld_base, instr, result);
      break;
   case nir_intrinsic_store_shared:
      visit_shared_store(bld_base, instr);
      break;
   case nir_intrinsic_shared_atomic_add:
   case nir_intrinsic_shared_atomic_imin:
   case nir_intrinsic_shared_atomic_umin:
   case nir_intrinsic_shared_atomic_imax:
   case nir_intrinsic_shared_atomic_umax:
   case nir_intrinsic_shared_atomic_and:
   case nir_intrinsic_shared_atomic_or:
   case nir_intrinsic_shared_atomic_xor:
   case nir_intrinsic_shared_atomic_exchange:
   case nir_intrinsic_shared_atomic_comp_swap:
      visit_shared_atomic(bld_base, instr, result);
      break;
   case nir_intrinsic_control_barrier:
      visit_barrier(bld_base);
      break;
   case nir_intrinsic_group_memory_barrier:
   case nir_intrinsic_memory_barrier:
   case nir_intrinsic_memory_barrier_shared:
   case nir_intrinsic_memory_barrier_buffer:
   case nir_intrinsic_memory_barrier_image:
   case nir_intrinsic_memory_barrier_tcs_patch:
      break;
   case nir_intrinsic_load_kernel_input:
      visit_load_kernel_input(bld_base, instr, result);
     break;
   case nir_intrinsic_load_global:
      visit_load_global(bld_base, instr, result);
      break;
   case nir_intrinsic_store_global:
      visit_store_global(bld_base, instr);
      break;
   case nir_intrinsic_global_atomic_add:
   case nir_intrinsic_global_atomic_imin:
   case nir_intrinsic_global_atomic_umin:
   case nir_intrinsic_global_atomic_imax:
   case nir_intrinsic_global_atomic_umax:
   case nir_intrinsic_global_atomic_and:
   case nir_intrinsic_global_atomic_or:
   case nir_intrinsic_global_atomic_xor:
   case nir_intrinsic_global_atomic_exchange:
   case nir_intrinsic_global_atomic_comp_swap:
      visit_global_atomic(bld_base, instr, result);
      break;
   case nir_intrinsic_vote_all:
   case nir_intrinsic_vote_any:
   case nir_intrinsic_vote_ieq:
      bld_base->vote(bld_base, cast_type(bld_base, get_src(bld_base, instr->src[0]), nir_type_int, 32), instr, result);
      break;
   case nir_intrinsic_interp_deref_at_offset:
   case nir_intrinsic_interp_deref_at_centroid:
   case nir_intrinsic_interp_deref_at_sample:
      visit_interp(bld_base, instr, result);
      break;
   default:
      assert(0);
      break;
   }
   if (result[0]) {
      assign_dest(bld_base, &instr->dest, result);
   }
}

static void visit_txs(struct lp_build_nir_context *bld_base, nir_tex_instr *instr)
{
   struct lp_sampler_size_query_params params = { 0 };
   LLVMValueRef sizes_out[NIR_MAX_VEC_COMPONENTS];
   LLVMValueRef explicit_lod = NULL;

   for (unsigned i = 0; i < instr->num_srcs; i++) {
      switch (instr->src[i].src_type) {
      case nir_tex_src_lod:
         explicit_lod = cast_type(bld_base, get_src(bld_base, instr->src[i].src), nir_type_int, 32);
         break;
      default:
         break;
      }
   }

   params.target = glsl_sampler_to_pipe(instr->sampler_dim, instr->is_array);
   params.texture_unit = instr->texture_index;
   params.explicit_lod = explicit_lod;
   params.is_sviewinfo = TRUE;
   params.sizes_out = sizes_out;
   params.samples_only = (instr->op == nir_texop_texture_samples);

   if (instr->op == nir_texop_query_levels)
      params.explicit_lod = bld_base->uint_bld.zero;
   bld_base->tex_size(bld_base, &params);
   assign_dest(bld_base, &instr->dest, &sizes_out[instr->op == nir_texop_query_levels ? 3 : 0]);
}

static enum lp_sampler_lod_property lp_build_nir_lod_property(struct lp_build_nir_context *bld_base,
                                                              nir_src lod_src)
{
   enum lp_sampler_lod_property lod_property;

   if (nir_src_is_dynamically_uniform(lod_src))
      lod_property = LP_SAMPLER_LOD_SCALAR;
   else if (bld_base->shader->info.stage == MESA_SHADER_FRAGMENT) {
      if (gallivm_perf & GALLIVM_PERF_NO_QUAD_LOD)
         lod_property = LP_SAMPLER_LOD_PER_ELEMENT;
      else
         lod_property = LP_SAMPLER_LOD_PER_QUAD;
   }
   else
      lod_property = LP_SAMPLER_LOD_PER_ELEMENT;
   return lod_property;
}

static void visit_tex(struct lp_build_nir_context *bld_base, nir_tex_instr *instr)
{
   struct gallivm_state *gallivm = bld_base->base.gallivm;
   LLVMBuilderRef builder = gallivm->builder;
   LLVMValueRef coords[5];
   LLVMValueRef offsets[3] = { NULL };
   LLVMValueRef explicit_lod = NULL, projector = NULL, ms_index = NULL;
   struct lp_sampler_params params;
   struct lp_derivatives derivs;
   unsigned sample_key = 0;
   nir_deref_instr *texture_deref_instr = NULL;
   nir_deref_instr *sampler_deref_instr = NULL;
   LLVMValueRef texel[NIR_MAX_VEC_COMPONENTS];
   unsigned lod_src = 0;
   LLVMValueRef coord_undef = LLVMGetUndef(bld_base->base.int_vec_type);

   memset(&params, 0, sizeof(params));
   enum lp_sampler_lod_property lod_property = LP_SAMPLER_LOD_SCALAR;

   if (instr->op == nir_texop_txs || instr->op == nir_texop_query_levels || instr->op == nir_texop_texture_samples) {
      visit_txs(bld_base, instr);
      return;
   }
   if (instr->op == nir_texop_txf || instr->op == nir_texop_txf_ms)
      sample_key |= LP_SAMPLER_OP_FETCH << LP_SAMPLER_OP_TYPE_SHIFT;
   else if (instr->op == nir_texop_tg4) {
      sample_key |= LP_SAMPLER_OP_GATHER << LP_SAMPLER_OP_TYPE_SHIFT;
      sample_key |= (instr->component << LP_SAMPLER_GATHER_COMP_SHIFT);
   } else if (instr->op == nir_texop_lod)
      sample_key |= LP_SAMPLER_OP_LODQ << LP_SAMPLER_OP_TYPE_SHIFT;
   for (unsigned i = 0; i < instr->num_srcs; i++) {
      switch (instr->src[i].src_type) {
      case nir_tex_src_coord: {
         LLVMValueRef coord = get_src(bld_base, instr->src[i].src);
         if (instr->coord_components == 1)
            coords[0] = coord;
         else {
            for (unsigned chan = 0; chan < instr->coord_components; ++chan)
               coords[chan] = LLVMBuildExtractValue(builder, coord,
                                                    chan, "");
         }
         for (unsigned chan = instr->coord_components; chan < 5; chan++)
            coords[chan] = coord_undef;

         break;
      }
      case nir_tex_src_texture_deref:
         texture_deref_instr = nir_src_as_deref(instr->src[i].src);
         break;
      case nir_tex_src_sampler_deref:
         sampler_deref_instr = nir_src_as_deref(instr->src[i].src);
         break;
      case nir_tex_src_projector:
         projector = lp_build_rcp(&bld_base->base, cast_type(bld_base, get_src(bld_base, instr->src[i].src), nir_type_float, 32));
         break;
      case nir_tex_src_comparator:
         sample_key |= LP_SAMPLER_SHADOW;
         coords[4] = get_src(bld_base, instr->src[i].src);
         coords[4] = cast_type(bld_base, coords[4], nir_type_float, 32);
         break;
      case nir_tex_src_bias:
         sample_key |= LP_SAMPLER_LOD_BIAS << LP_SAMPLER_LOD_CONTROL_SHIFT;
         lod_src = i;
         explicit_lod = cast_type(bld_base, get_src(bld_base, instr->src[i].src), nir_type_float, 32);
         break;
      case nir_tex_src_lod:
         sample_key |= LP_SAMPLER_LOD_EXPLICIT << LP_SAMPLER_LOD_CONTROL_SHIFT;
         lod_src = i;
         if (instr->op == nir_texop_txf)
            explicit_lod = cast_type(bld_base, get_src(bld_base, instr->src[i].src), nir_type_int, 32);
         else
            explicit_lod = cast_type(bld_base, get_src(bld_base, instr->src[i].src), nir_type_float, 32);
         break;
      case nir_tex_src_ddx: {
         int deriv_cnt = instr->coord_components;
         if (instr->is_array)
            deriv_cnt--;
         LLVMValueRef deriv_val = get_src(bld_base, instr->src[i].src);
         if (deriv_cnt == 1)
            derivs.ddx[0] = deriv_val;
         else
            for (unsigned chan = 0; chan < deriv_cnt; ++chan)
               derivs.ddx[chan] = LLVMBuildExtractValue(builder, deriv_val,
                                                        chan, "");
         for (unsigned chan = 0; chan < deriv_cnt; ++chan)
            derivs.ddx[chan] = cast_type(bld_base, derivs.ddx[chan], nir_type_float, 32);
         break;
      }
      case nir_tex_src_ddy: {
         int deriv_cnt = instr->coord_components;
         if (instr->is_array)
            deriv_cnt--;
         LLVMValueRef deriv_val = get_src(bld_base, instr->src[i].src);
         if (deriv_cnt == 1)
            derivs.ddy[0] = deriv_val;
         else
            for (unsigned chan = 0; chan < deriv_cnt; ++chan)
               derivs.ddy[chan] = LLVMBuildExtractValue(builder, deriv_val,
                                                        chan, "");
         for (unsigned chan = 0; chan < deriv_cnt; ++chan)
            derivs.ddy[chan] = cast_type(bld_base, derivs.ddy[chan], nir_type_float, 32);
         break;
      }
      case nir_tex_src_offset: {
         int offset_cnt = instr->coord_components;
         if (instr->is_array)
            offset_cnt--;
         LLVMValueRef offset_val = get_src(bld_base, instr->src[i].src);
         sample_key |= LP_SAMPLER_OFFSETS;
         if (offset_cnt == 1)
            offsets[0] = cast_type(bld_base, offset_val, nir_type_int, 32);
         else {
            for (unsigned chan = 0; chan < offset_cnt; ++chan) {
               offsets[chan] = LLVMBuildExtractValue(builder, offset_val,
                                                     chan, "");
               offsets[chan] = cast_type(bld_base, offsets[chan], nir_type_int, 32);
            }
         }
         break;
      }
      case nir_tex_src_ms_index:
         sample_key |= LP_SAMPLER_FETCH_MS;
         ms_index = cast_type(bld_base, get_src(bld_base, instr->src[i].src), nir_type_int, 32);
         break;
      default:
         assert(0);
         break;
      }
   }
   if (!sampler_deref_instr)
      sampler_deref_instr = texture_deref_instr;

   if (explicit_lod)
      lod_property = lp_build_nir_lod_property(bld_base, instr->src[lod_src].src);

   if (instr->op == nir_texop_tex || instr->op == nir_texop_tg4 || instr->op == nir_texop_txb ||
       instr->op == nir_texop_txl || instr->op == nir_texop_txd || instr->op == nir_texop_lod)
      for (unsigned chan = 0; chan < instr->coord_components; ++chan)
         coords[chan] = cast_type(bld_base, coords[chan], nir_type_float, 32);
   else if (instr->op == nir_texop_txf || instr->op == nir_texop_txf_ms)
      for (unsigned chan = 0; chan < instr->coord_components; ++chan)
         coords[chan] = cast_type(bld_base, coords[chan], nir_type_int, 32);

   if (instr->is_array && instr->sampler_dim == GLSL_SAMPLER_DIM_1D) {
      /* move layer coord for 1d arrays. */
      coords[2] = coords[1];
      coords[1] = coord_undef;
   }

   if (projector) {
      for (unsigned chan = 0; chan < instr->coord_components; ++chan)
         coords[chan] = lp_build_mul(&bld_base->base, coords[chan], projector);
      if (sample_key & LP_SAMPLER_SHADOW)
         coords[4] = lp_build_mul(&bld_base->base, coords[4], projector);
   }

   uint32_t base_index = 0;
   if (!texture_deref_instr) {
      int samp_src_index = nir_tex_instr_src_index(instr, nir_tex_src_sampler_handle);
      if (samp_src_index == -1) {
         base_index = instr->sampler_index;
      }
   }

   if (instr->op == nir_texop_txd) {
      sample_key |= LP_SAMPLER_LOD_DERIVATIVES << LP_SAMPLER_LOD_CONTROL_SHIFT;
      params.derivs = &derivs;
      if (bld_base->shader->info.stage == MESA_SHADER_FRAGMENT) {
         if (gallivm_perf & GALLIVM_PERF_NO_QUAD_LOD)
            lod_property = LP_SAMPLER_LOD_PER_ELEMENT;
         else
            lod_property = LP_SAMPLER_LOD_PER_QUAD;
      } else
         lod_property = LP_SAMPLER_LOD_PER_ELEMENT;
   }

   sample_key |= lod_property << LP_SAMPLER_LOD_PROPERTY_SHIFT;
   params.sample_key = sample_key;
   params.offsets = offsets;
   params.texture_index = base_index;
   params.sampler_index = base_index;
   params.coords = coords;
   params.texel = texel;
   params.lod = explicit_lod;
   params.ms_index = ms_index;
   bld_base->tex(bld_base, &params);
   assign_dest(bld_base, &instr->dest, texel);
}

static void visit_ssa_undef(struct lp_build_nir_context *bld_base,
                            const nir_ssa_undef_instr *instr)
{
   unsigned num_components = instr->def.num_components;
   LLVMValueRef undef[NIR_MAX_VEC_COMPONENTS];
   struct lp_build_context *undef_bld = get_int_bld(bld_base, true, instr->def.bit_size);
   for (unsigned i = 0; i < num_components; i++)
      undef[i] = LLVMGetUndef(undef_bld->vec_type);
   assign_ssa_dest(bld_base, &instr->def, undef);
}

static void visit_jump(struct lp_build_nir_context *bld_base,
                       const nir_jump_instr *instr)
{
   switch (instr->type) {
   case nir_jump_break:
      bld_base->break_stmt(bld_base);
      break;
   case nir_jump_continue:
      bld_base->continue_stmt(bld_base);
      break;
   default:
      unreachable("Unknown jump instr\n");
   }
}

static void visit_deref(struct lp_build_nir_context *bld_base,
                        nir_deref_instr *instr)
{
   if (instr->mode != nir_var_mem_shared &&
       instr->mode != nir_var_mem_global)
      return;
   LLVMValueRef result = NULL;
   switch(instr->deref_type) {
   case nir_deref_type_var: {
      struct hash_entry *entry = _mesa_hash_table_search(bld_base->vars, instr->var);
      result = entry->data;
      break;
   }
   default:
      unreachable("Unhandled deref_instr deref type");
   }

   assign_ssa(bld_base, instr->dest.ssa.index, result);
}

static void visit_block(struct lp_build_nir_context *bld_base, nir_block *block)
{
   nir_foreach_instr(instr, block)
   {
      switch (instr->type) {
      case nir_instr_type_alu:
         visit_alu(bld_base, nir_instr_as_alu(instr));
         break;
      case nir_instr_type_load_const:
         visit_load_const(bld_base, nir_instr_as_load_const(instr));
         break;
      case nir_instr_type_intrinsic:
         visit_intrinsic(bld_base, nir_instr_as_intrinsic(instr));
         break;
      case nir_instr_type_tex:
         visit_tex(bld_base, nir_instr_as_tex(instr));
         break;
      case nir_instr_type_phi:
         assert(0);
         break;
      case nir_instr_type_ssa_undef:
         visit_ssa_undef(bld_base, nir_instr_as_ssa_undef(instr));
         break;
      case nir_instr_type_jump:
         visit_jump(bld_base, nir_instr_as_jump(instr));
         break;
      case nir_instr_type_deref:
         visit_deref(bld_base, nir_instr_as_deref(instr));
         break;
      default:
         fprintf(stderr, "Unknown NIR instr type: ");
         nir_print_instr(instr, stderr);
         fprintf(stderr, "\n");
         abort();
      }
   }
}

static void visit_if(struct lp_build_nir_context *bld_base, nir_if *if_stmt)
{
   LLVMValueRef cond = get_src(bld_base, if_stmt->condition);

   bld_base->if_cond(bld_base, cond);
   visit_cf_list(bld_base, &if_stmt->then_list);

   if (!exec_list_is_empty(&if_stmt->else_list)) {
      bld_base->else_stmt(bld_base);
      visit_cf_list(bld_base, &if_stmt->else_list);
   }
   bld_base->endif_stmt(bld_base);
}

static void visit_loop(struct lp_build_nir_context *bld_base, nir_loop *loop)
{
   bld_base->bgnloop(bld_base);
   visit_cf_list(bld_base, &loop->body);
   bld_base->endloop(bld_base);
}

static void visit_cf_list(struct lp_build_nir_context *bld_base,
                          struct exec_list *list)
{
   foreach_list_typed(nir_cf_node, node, node, list)
   {
      switch (node->type) {
      case nir_cf_node_block:
         visit_block(bld_base, nir_cf_node_as_block(node));
         break;

      case nir_cf_node_if:
         visit_if(bld_base, nir_cf_node_as_if(node));
         break;

      case nir_cf_node_loop:
         visit_loop(bld_base, nir_cf_node_as_loop(node));
         break;

      default:
         assert(0);
      }
   }
}

static void
handle_shader_output_decl(struct lp_build_nir_context *bld_base,
                          struct nir_shader *nir,
                          struct nir_variable *variable)
{
   bld_base->emit_var_decl(bld_base, variable);
}

/* vector registers are stored as arrays in LLVM side,
   so we can use GEP on them, as to do exec mask stores
   we need to operate on a single components.
   arrays are:
   0.x, 1.x, 2.x, 3.x
   0.y, 1.y, 2.y, 3.y
   ....
*/
static LLVMTypeRef get_register_type(struct lp_build_nir_context *bld_base,
                                     nir_register *reg)
{
   struct lp_build_context *int_bld = get_int_bld(bld_base, true, reg->bit_size);

   LLVMTypeRef type = int_bld->vec_type;
   if (reg->num_array_elems)
      type = LLVMArrayType(type, reg->num_array_elems);
   if (reg->num_components > 1)
      type = LLVMArrayType(type, reg->num_components);

   return type;
}


bool lp_build_nir_llvm(
   struct lp_build_nir_context *bld_base,
   struct nir_shader *nir)
{
   struct nir_function *func;

   nir_convert_from_ssa(nir, true);
   nir_lower_locals_to_regs(nir);
   nir_remove_dead_derefs(nir);
   nir_remove_dead_variables(nir, nir_var_function_temp, NULL);

   nir_foreach_variable(variable, &nir->outputs)
      handle_shader_output_decl(bld_base, nir, variable);

   bld_base->regs = _mesa_hash_table_create(NULL, _mesa_hash_pointer,
                                            _mesa_key_pointer_equal);
   bld_base->vars = _mesa_hash_table_create(NULL, _mesa_hash_pointer,
                                            _mesa_key_pointer_equal);

   func = (struct nir_function *)exec_list_get_head(&nir->functions);

   nir_foreach_register(reg, &func->impl->registers) {
      LLVMTypeRef type = get_register_type(bld_base, reg);
      LLVMValueRef reg_alloc = lp_build_alloca_undef(bld_base->base.gallivm,
                                                     type, "reg");
      _mesa_hash_table_insert(bld_base->regs, reg, reg_alloc);
   }
   nir_index_ssa_defs(func->impl);
   bld_base->ssa_defs = calloc(func->impl->ssa_alloc, sizeof(LLVMValueRef));
   visit_cf_list(bld_base, &func->impl->body);

   free(bld_base->ssa_defs);
   ralloc_free(bld_base->vars);
   ralloc_free(bld_base->regs);
   return true;
}

/* do some basic opts to remove some things we don't want to see. */
void lp_build_opt_nir(struct nir_shader *nir)
{
   bool progress;
   do {
      progress = false;
      NIR_PASS_V(nir, nir_opt_constant_folding);
      NIR_PASS_V(nir, nir_opt_algebraic);
      NIR_PASS_V(nir, nir_lower_pack);

      nir_lower_tex_options options = { .lower_tex_without_implicit_lod = true };
      NIR_PASS_V(nir, nir_lower_tex, &options);
   } while (progress);
   nir_lower_bool_to_int32(nir);
}