nir: Move compute system value lowering to a separate pass
The actual variable -> intrinsic lowering stays where it is, but
ops which convert one intrinsic to be implemented in terms of
another have moved.
Reviewed-by: Karol Herbst <kherbst@redhat.com>
Reviewed-by: Jason Ekstrand <jason@jlekstrand.net>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/5891>
diff --git a/src/amd/vulkan/radv_shader.c b/src/amd/vulkan/radv_shader.c
index 03cfe99..01a22b7 100644
--- a/src/amd/vulkan/radv_shader.c
+++ b/src/amd/vulkan/radv_shader.c
@@ -540,6 +540,8 @@
NIR_PASS_V(nir, nir_propagate_invariant);
NIR_PASS_V(nir, nir_lower_system_values);
+ NIR_PASS_V(nir, nir_lower_compute_system_values);
+
NIR_PASS_V(nir, nir_lower_clip_cull_distance_arrays);
if (device->instance->debug_flags & RADV_DEBUG_DISCARD_TO_DEMOTE)
diff --git a/src/broadcom/compiler/vir.c b/src/broadcom/compiler/vir.c
index d06f789..83eb4d1 100644
--- a/src/broadcom/compiler/vir.c
+++ b/src/broadcom/compiler/vir.c
@@ -586,6 +586,7 @@
NIR_PASS_V(c->s, nir_lower_tex, &tex_options);
NIR_PASS_V(c->s, nir_lower_system_values);
+ NIR_PASS_V(c->s, nir_lower_compute_system_values);
NIR_PASS_V(c->s, nir_lower_vars_to_scratch,
nir_var_function_temp,
diff --git a/src/compiler/nir/nir.h b/src/compiler/nir/nir.h
index bb43ab1..7432afd 100644
--- a/src/compiler/nir/nir.h
+++ b/src/compiler/nir/nir.h
@@ -4276,6 +4276,8 @@
bool nir_lower_system_values(nir_shader *shader);
+bool nir_lower_compute_system_values(nir_shader *shader);
+
enum PACKED nir_lower_tex_packing {
nir_lower_tex_packing_none = 0,
/* The sampler returns up to 2 32-bit words of half floats or 16-bit signed
diff --git a/src/compiler/nir/nir_lower_system_values.c b/src/compiler/nir/nir_lower_system_values.c
index bc6a393..bc80d18 100644
--- a/src/compiler/nir/nir_lower_system_values.c
+++ b/src/compiler/nir/nir_lower_system_values.c
@@ -92,6 +92,150 @@
return NULL;
}
+ case nir_intrinsic_load_helper_invocation:
+ if (b->shader->options->lower_helper_invocation) {
+ nir_ssa_def *tmp;
+ tmp = nir_ishl(b, nir_imm_int(b, 1),
+ nir_load_sample_id_no_per_sample(b));
+ tmp = nir_iand(b, nir_load_sample_mask_in(b), tmp);
+ return nir_inot(b, nir_i2b(b, tmp));
+ } else {
+ return NULL;
+ }
+
+ case nir_intrinsic_load_local_invocation_id:
+ case nir_intrinsic_load_local_invocation_index:
+ case nir_intrinsic_load_local_group_size:
+ return sanitize_32bit_sysval(b, intrin);
+
+ case nir_intrinsic_load_deref: {
+ nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]);
+ if (deref->mode != nir_var_system_value)
+ return NULL;
+
+ if (deref->deref_type != nir_deref_type_var) {
+ /* The only one system value that is an array and that is
+ * gl_SampleMask which is always an array of one element.
+ */
+ assert(deref->deref_type == nir_deref_type_array);
+ deref = nir_deref_instr_parent(deref);
+ assert(deref->deref_type == nir_deref_type_var);
+ assert(deref->var->data.location == SYSTEM_VALUE_SAMPLE_MASK_IN);
+ }
+ nir_variable *var = deref->var;
+
+ switch (var->data.location) {
+ case SYSTEM_VALUE_INSTANCE_INDEX:
+ return nir_iadd(b, nir_load_instance_id(b),
+ nir_load_base_instance(b));
+
+ case SYSTEM_VALUE_SUBGROUP_EQ_MASK:
+ case SYSTEM_VALUE_SUBGROUP_GE_MASK:
+ case SYSTEM_VALUE_SUBGROUP_GT_MASK:
+ case SYSTEM_VALUE_SUBGROUP_LE_MASK:
+ case SYSTEM_VALUE_SUBGROUP_LT_MASK: {
+ nir_intrinsic_op op =
+ nir_intrinsic_from_system_value(var->data.location);
+ nir_intrinsic_instr *load = nir_intrinsic_instr_create(b->shader, op);
+ nir_ssa_dest_init_for_type(&load->instr, &load->dest,
+ var->type, NULL);
+ load->num_components = load->dest.ssa.num_components;
+ nir_builder_instr_insert(b, &load->instr);
+ return &load->dest.ssa;
+ }
+
+ case SYSTEM_VALUE_DEVICE_INDEX:
+ if (b->shader->options->lower_device_index_to_zero)
+ return nir_imm_int(b, 0);
+ break;
+
+ case SYSTEM_VALUE_GLOBAL_GROUP_SIZE:
+ return build_global_group_size(b, bit_size);
+
+ case SYSTEM_VALUE_BARYCENTRIC_LINEAR_PIXEL:
+ return nir_load_barycentric(b, nir_intrinsic_load_barycentric_pixel,
+ INTERP_MODE_NOPERSPECTIVE);
+
+ case SYSTEM_VALUE_BARYCENTRIC_LINEAR_CENTROID:
+ return nir_load_barycentric(b, nir_intrinsic_load_barycentric_centroid,
+ INTERP_MODE_NOPERSPECTIVE);
+
+ case SYSTEM_VALUE_BARYCENTRIC_LINEAR_SAMPLE:
+ return nir_load_barycentric(b, nir_intrinsic_load_barycentric_sample,
+ INTERP_MODE_NOPERSPECTIVE);
+
+ case SYSTEM_VALUE_BARYCENTRIC_PERSP_PIXEL:
+ return nir_load_barycentric(b, nir_intrinsic_load_barycentric_pixel,
+ INTERP_MODE_SMOOTH);
+
+ case SYSTEM_VALUE_BARYCENTRIC_PERSP_CENTROID:
+ return nir_load_barycentric(b, nir_intrinsic_load_barycentric_centroid,
+ INTERP_MODE_SMOOTH);
+
+ case SYSTEM_VALUE_BARYCENTRIC_PERSP_SAMPLE:
+ return nir_load_barycentric(b, nir_intrinsic_load_barycentric_sample,
+ INTERP_MODE_SMOOTH);
+
+ case SYSTEM_VALUE_BARYCENTRIC_PULL_MODEL:
+ return nir_load_barycentric(b, nir_intrinsic_load_barycentric_model,
+ INTERP_MODE_NONE);
+
+ default:
+ break;
+ }
+
+ nir_intrinsic_op sysval_op =
+ nir_intrinsic_from_system_value(var->data.location);
+ return nir_load_system_value(b, sysval_op, 0,
+ intrin->dest.ssa.num_components,
+ intrin->dest.ssa.bit_size);
+ }
+
+ default:
+ return NULL;
+ }
+}
+
+bool
+nir_lower_system_values(nir_shader *shader)
+{
+ bool progress = nir_shader_lower_instructions(shader,
+ lower_system_value_filter,
+ lower_system_value_instr,
+ NULL);
+
+ /* We're going to delete the variables so we need to clean up all those
+ * derefs we left lying around.
+ */
+ if (progress)
+ nir_remove_dead_derefs(shader);
+
+ nir_foreach_variable_with_modes_safe(var, shader, nir_var_system_value)
+ exec_node_remove(&var->node);
+
+ return progress;
+}
+
+static bool
+lower_compute_system_value_filter(const nir_instr *instr, const void *_options)
+{
+ return instr->type == nir_instr_type_intrinsic;
+}
+
+static nir_ssa_def *
+lower_compute_system_value_instr(nir_builder *b,
+ nir_instr *instr, void *_options)
+{
+ nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
+
+ /* All the intrinsics we care about are loads */
+ if (!nir_intrinsic_infos[intrin->intrinsic].has_dest)
+ return NULL;
+
+ assert(intrin->dest.is_ssa);
+ const unsigned bit_size = intrin->dest.ssa.bit_size;
+
+ switch (intrin->intrinsic) {
case nir_intrinsic_load_local_invocation_id:
/* If lower_cs_local_id_from_index is true, then we derive the local
* index from the local id.
@@ -215,121 +359,20 @@
return index;
}
- case nir_intrinsic_load_helper_invocation:
- if (b->shader->options->lower_helper_invocation) {
- nir_ssa_def *tmp;
- tmp = nir_ishl(b, nir_imm_int(b, 1),
- nir_load_sample_id_no_per_sample(b));
- tmp = nir_iand(b, nir_load_sample_mask_in(b), tmp);
- return nir_inot(b, nir_i2b(b, tmp));
- } else {
- return NULL;
- }
-
- case nir_intrinsic_load_deref: {
- nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]);
- if (deref->mode != nir_var_system_value)
- return NULL;
-
- if (deref->deref_type != nir_deref_type_var) {
- /* The only one system value that is an array and that is
- * gl_SampleMask which is always an array of one element.
- */
- assert(deref->deref_type == nir_deref_type_array);
- deref = nir_deref_instr_parent(deref);
- assert(deref->deref_type == nir_deref_type_var);
- assert(deref->var->data.location == SYSTEM_VALUE_SAMPLE_MASK_IN);
- }
- nir_variable *var = deref->var;
-
- switch (var->data.location) {
- case SYSTEM_VALUE_INSTANCE_INDEX:
- return nir_iadd(b, nir_load_instance_id(b),
- nir_load_base_instance(b));
-
- case SYSTEM_VALUE_SUBGROUP_EQ_MASK:
- case SYSTEM_VALUE_SUBGROUP_GE_MASK:
- case SYSTEM_VALUE_SUBGROUP_GT_MASK:
- case SYSTEM_VALUE_SUBGROUP_LE_MASK:
- case SYSTEM_VALUE_SUBGROUP_LT_MASK: {
- nir_intrinsic_op op =
- nir_intrinsic_from_system_value(var->data.location);
- nir_intrinsic_instr *load = nir_intrinsic_instr_create(b->shader, op);
- nir_ssa_dest_init_for_type(&load->instr, &load->dest,
- var->type, NULL);
- load->num_components = load->dest.ssa.num_components;
- nir_builder_instr_insert(b, &load->instr);
- return &load->dest.ssa;
- }
-
- case SYSTEM_VALUE_DEVICE_INDEX:
- if (b->shader->options->lower_device_index_to_zero)
- return nir_imm_int(b, 0);
- break;
-
- case SYSTEM_VALUE_GLOBAL_GROUP_SIZE:
- return build_global_group_size(b, bit_size);
-
- case SYSTEM_VALUE_BARYCENTRIC_LINEAR_PIXEL:
- return nir_load_barycentric(b, nir_intrinsic_load_barycentric_pixel,
- INTERP_MODE_NOPERSPECTIVE);
-
- case SYSTEM_VALUE_BARYCENTRIC_LINEAR_CENTROID:
- return nir_load_barycentric(b, nir_intrinsic_load_barycentric_centroid,
- INTERP_MODE_NOPERSPECTIVE);
-
- case SYSTEM_VALUE_BARYCENTRIC_LINEAR_SAMPLE:
- return nir_load_barycentric(b, nir_intrinsic_load_barycentric_sample,
- INTERP_MODE_NOPERSPECTIVE);
-
- case SYSTEM_VALUE_BARYCENTRIC_PERSP_PIXEL:
- return nir_load_barycentric(b, nir_intrinsic_load_barycentric_pixel,
- INTERP_MODE_SMOOTH);
-
- case SYSTEM_VALUE_BARYCENTRIC_PERSP_CENTROID:
- return nir_load_barycentric(b, nir_intrinsic_load_barycentric_centroid,
- INTERP_MODE_SMOOTH);
-
- case SYSTEM_VALUE_BARYCENTRIC_PERSP_SAMPLE:
- return nir_load_barycentric(b, nir_intrinsic_load_barycentric_sample,
- INTERP_MODE_SMOOTH);
-
- case SYSTEM_VALUE_BARYCENTRIC_PULL_MODEL:
- return nir_load_barycentric(b, nir_intrinsic_load_barycentric_model,
- INTERP_MODE_NONE);
-
- default:
- break;
- }
-
- nir_intrinsic_op sysval_op =
- nir_intrinsic_from_system_value(var->data.location);
- return nir_load_system_value(b, sysval_op, 0,
- intrin->dest.ssa.num_components,
- intrin->dest.ssa.bit_size);
- }
-
default:
return NULL;
}
}
bool
-nir_lower_system_values(nir_shader *shader)
+nir_lower_compute_system_values(nir_shader *shader)
{
- bool progress = nir_shader_lower_instructions(shader,
- lower_system_value_filter,
- lower_system_value_instr,
- NULL);
+ if (shader->info.stage != MESA_SHADER_COMPUTE &&
+ shader->info.stage != MESA_SHADER_KERNEL)
+ return false;
- /* We're going to delete the variables so we need to clean up all those
- * derefs we left lying around.
- */
- if (progress)
- nir_remove_dead_derefs(shader);
-
- nir_foreach_variable_with_modes_safe(var, shader, nir_var_system_value)
- exec_node_remove(&var->node);
-
- return progress;
+ return nir_shader_lower_instructions(shader,
+ lower_compute_system_value_filter,
+ lower_compute_system_value_instr,
+ NULL);
}
diff --git a/src/freedreno/vulkan/tu_shader.c b/src/freedreno/vulkan/tu_shader.c
index ec8e266..aca5ea0 100644
--- a/src/freedreno/vulkan/tu_shader.c
+++ b/src/freedreno/vulkan/tu_shader.c
@@ -765,6 +765,8 @@
nir_assign_io_var_locations(nir, nir_var_shader_out, &nir->num_outputs, stage);
NIR_PASS_V(nir, nir_lower_system_values);
+ NIR_PASS_V(nir, nir_lower_compute_system_values);
+
NIR_PASS_V(nir, nir_lower_frexp);
if (stage == MESA_SHADER_FRAGMENT) {
diff --git a/src/gallium/auxiliary/nir/tgsi_to_nir.c b/src/gallium/auxiliary/nir/tgsi_to_nir.c
index 3035f54..7b1b055 100644
--- a/src/gallium/auxiliary/nir/tgsi_to_nir.c
+++ b/src/gallium/auxiliary/nir/tgsi_to_nir.c
@@ -2559,6 +2559,7 @@
NIR_PASS_V(nir, nir_split_var_copies);
NIR_PASS_V(nir, nir_lower_var_copies);
NIR_PASS_V(nir, nir_lower_system_values);
+ NIR_PASS_V(nir, nir_lower_compute_system_values);
if (c->cap_packed_uniforms)
NIR_PASS_V(nir, nir_lower_uniforms_to_ubo, 16);
diff --git a/src/gallium/drivers/freedreno/ir3/ir3_cmdline.c b/src/gallium/drivers/freedreno/ir3/ir3_cmdline.c
index 5dad4da..54227e4 100644
--- a/src/gallium/drivers/freedreno/ir3/ir3_cmdline.c
+++ b/src/gallium/drivers/freedreno/ir3/ir3_cmdline.c
@@ -185,6 +185,8 @@
ir3_glsl_type_size);
NIR_PASS_V(nir, nir_lower_system_values);
+ NIR_PASS_V(nir, nir_lower_compute_system_values);
+
NIR_PASS_V(nir, nir_lower_frexp);
NIR_PASS_V(nir, nir_lower_io,
nir_var_shader_in | nir_var_shader_out | nir_var_uniform,
@@ -401,6 +403,7 @@
/* TODO do this somewhere else */
nir_lower_int64(nir);
nir_lower_system_values(nir);
+ nir_lower_compute_system_values(nir);
} else if (num_files > 0) {
nir = load_glsl(num_files, filenames, stage);
} else {
diff --git a/src/gallium/frontends/clover/nir/invocation.cpp b/src/gallium/frontends/clover/nir/invocation.cpp
index 032fb03..f574720 100644
--- a/src/gallium/frontends/clover/nir/invocation.cpp
+++ b/src/gallium/frontends/clover/nir/invocation.cpp
@@ -169,6 +169,8 @@
spirv_options.global_addr_format);
NIR_PASS_V(nir, nir_lower_system_values);
+ NIR_PASS_V(nir, nir_lower_compute_system_values);
+
if (compiler_options->lower_int64_options)
NIR_PASS_V(nir, nir_lower_int64);
diff --git a/src/gallium/frontends/vallium/val_pipeline.c b/src/gallium/frontends/vallium/val_pipeline.c
index 136ac93..779e2ec 100644
--- a/src/gallium/frontends/vallium/val_pipeline.c
+++ b/src/gallium/frontends/vallium/val_pipeline.c
@@ -562,6 +562,7 @@
if (stage == MESA_SHADER_FRAGMENT)
val_lower_input_attachments(nir, false);
NIR_PASS_V(nir, nir_lower_system_values);
+ NIR_PASS_V(nir, nir_lower_compute_system_values);
NIR_PASS_V(nir, nir_lower_clip_cull_distance_arrays);
nir_remove_dead_variables(nir, nir_var_uniform, NULL);
diff --git a/src/intel/compiler/brw_nir.c b/src/intel/compiler/brw_nir.c
index 567bb42..024cea7 100644
--- a/src/intel/compiler/brw_nir.c
+++ b/src/intel/compiler/brw_nir.c
@@ -709,6 +709,7 @@
}
OPT(nir_lower_system_values);
+ OPT(nir_lower_compute_system_values);
const nir_lower_subgroups_options subgroups_options = {
.ballot_bit_size = 32,
diff --git a/src/mesa/state_tracker/st_glsl_to_nir.cpp b/src/mesa/state_tracker/st_glsl_to_nir.cpp
index 99bbbef..b5b85ae 100644
--- a/src/mesa/state_tracker/st_glsl_to_nir.cpp
+++ b/src/mesa/state_tracker/st_glsl_to_nir.cpp
@@ -771,6 +771,8 @@
st->pipe->screen);
NIR_PASS_V(nir, nir_lower_system_values);
+ NIR_PASS_V(nir, nir_lower_compute_system_values);
+
NIR_PASS_V(nir, nir_lower_clip_cull_distance_arrays);
st_shader_gather_info(nir, shader->Program);
diff --git a/src/mesa/state_tracker/st_nir_builtins.c b/src/mesa/state_tracker/st_nir_builtins.c
index 77ec872..cd078f3 100644
--- a/src/mesa/state_tracker/st_nir_builtins.c
+++ b/src/mesa/state_tracker/st_nir_builtins.c
@@ -43,6 +43,7 @@
NIR_PASS_V(nir, nir_split_var_copies);
NIR_PASS_V(nir, nir_lower_var_copies);
NIR_PASS_V(nir, nir_lower_system_values);
+ NIR_PASS_V(nir, nir_lower_compute_system_values);
if (nir->options->lower_to_scalar) {
nir_variable_mode mask =
diff --git a/src/mesa/state_tracker/st_program.c b/src/mesa/state_tracker/st_program.c
index 7e7373d..7ba6344 100644
--- a/src/mesa/state_tracker/st_program.c
+++ b/src/mesa/state_tracker/st_program.c
@@ -388,6 +388,7 @@
NIR_PASS_V(nir, st_nir_lower_wpos_ytransform, prog, screen);
NIR_PASS_V(nir, nir_lower_system_values);
+ NIR_PASS_V(nir, nir_lower_compute_system_values);
/* Optimise NIR */
NIR_PASS_V(nir, nir_opt_constant_folding);
