src/panfrost/lib/pan_blit.c - platform/external/mesa3d - Git at Google

 /*
  * Copyright (C) 2020 Collabora, Ltd.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
  * to deal in the Software without restriction, including without limitation
  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  * and/or sell copies of the Software, and to permit persons to whom the
  * Software is furnished to do so, subject to the following conditions:
  *
  * The above copyright notice and this permission notice (including the next
  * paragraph) shall be included in all copies or substantial portions of the
  * Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  * SOFTWARE.
  *
  * Authors:
  *   Alyssa Rosenzweig <alyssa.rosenzweig@collabora.com>
  */

 #include <math.h>
 #include <stdio.h>
 #include "pan_encoder.h"
 #include "pan_pool.h"
 #include "pan_scoreboard.h"
 #include "pan_texture.h"
 #include "panfrost-quirks.h"
 #include "../midgard/midgard_compile.h"
 #include "compiler/nir/nir_builder.h"
 #include "util/u_math.h"

 /* On Midgard, the native blit infrastructure (via MFBD preloads) is broken or
  * missing in many cases. We instead use software paths as fallbacks to
  * implement blits, which are done as TILER jobs. No vertex shader is
  * necessary since we can supply screen-space coordinates directly.
  *
  * This is primarily designed as a fallback for preloads but could be extended
  * for other clears/blits if needed in the future. */

 static void
 panfrost_build_blit_shader(panfrost_program *program, unsigned gpu_id, gl_frag_result loc, nir_alu_type T, bool ms)
 {
         bool is_colour = loc >= FRAG_RESULT_DATA0;

         nir_builder _b;
         nir_builder_init_simple_shader(&_b, NULL, MESA_SHADER_FRAGMENT, &midgard_nir_options);
         nir_builder *b = &_b;
         nir_shader *shader = b->shader;

         shader->info.internal = true;

         nir_variable *c_src = nir_variable_create(shader, nir_var_shader_in, glsl_vector_type(GLSL_TYPE_FLOAT, 2), "coord");
         nir_variable *c_out = nir_variable_create(shader, nir_var_shader_out, glsl_vector_type(
                                 GLSL_TYPE_FLOAT, is_colour ? 4 : 1), "out");

         c_src->data.location = VARYING_SLOT_TEX0;
         c_out->data.location = loc;

         nir_ssa_def *coord = nir_load_var(b, c_src);

         nir_tex_instr *tex = nir_tex_instr_create(shader, ms ? 3 : 1);

         tex->dest_type = T;

         if (ms) {
                 tex->src[0].src_type = nir_tex_src_coord;
                 tex->src[0].src = nir_src_for_ssa(nir_f2i32(b, coord));
                 tex->coord_components = 2;

                 tex->src[1].src_type = nir_tex_src_ms_index;
                 tex->src[1].src = nir_src_for_ssa(nir_load_sample_id(b));

                 tex->src[2].src_type = nir_tex_src_lod;
                 tex->src[2].src = nir_src_for_ssa(nir_imm_int(b, 0));
                 tex->sampler_dim = GLSL_SAMPLER_DIM_MS;
                 tex->op = nir_texop_txf_ms;
         } else {
                 tex->op = nir_texop_tex;

                 tex->src[0].src_type = nir_tex_src_coord;
                 tex->src[0].src = nir_src_for_ssa(coord);
                 tex->coord_components = 2;

                 tex->sampler_dim = GLSL_SAMPLER_DIM_2D;
         }

         nir_ssa_dest_init(&tex->instr, &tex->dest, 4, 32, NULL);
         nir_builder_instr_insert(b, &tex->instr);

         if (is_colour)
                 nir_store_var(b, c_out, &tex->dest.ssa, 0xFF);
         else
                 nir_store_var(b, c_out, nir_channel(b, &tex->dest.ssa, 0), 0xFF);

         struct panfrost_compile_inputs inputs = {
                 .gpu_id = gpu_id,
         };

         midgard_compile_shader_nir(shader, program, &inputs);
         ralloc_free(shader);
 }

 /* Compile and upload all possible blit shaders ahead-of-time to reduce draw
  * time overhead. There's only ~30 of them at the moment, so this is fine */

 void
 panfrost_init_blit_shaders(struct panfrost_device *dev)
 {
         static const struct {
                 gl_frag_result loc;
                 unsigned types;
         } shader_descs[] = {
                 { FRAG_RESULT_DEPTH,   1 << PAN_BLIT_FLOAT },
                 { FRAG_RESULT_STENCIL, 1 << PAN_BLIT_UINT },
                 { FRAG_RESULT_DATA0,  ~0 },
                 { FRAG_RESULT_DATA1,  ~0 },
                 { FRAG_RESULT_DATA2,  ~0 },
                 { FRAG_RESULT_DATA3,  ~0 },
                 { FRAG_RESULT_DATA4,  ~0 },
                 { FRAG_RESULT_DATA5,  ~0 },
                 { FRAG_RESULT_DATA6,  ~0 },
                 { FRAG_RESULT_DATA7,  ~0 }
         };

         nir_alu_type nir_types[PAN_BLIT_NUM_TYPES] = {
                 nir_type_float,
                 nir_type_uint,
                 nir_type_int
         };

         /* Total size = # of shaders * bytes per shader. There are
          * shaders for each RT (so up to DATA7 -- overestimate is
          * okay) and up to NUM_TYPES variants of each, * 2 for multisampling
          * variants. These shaders are simple enough that they should be less
          * than 8 quadwords each (again, overestimate is fine). */

         unsigned offset = 0;
         unsigned total_size = (FRAG_RESULT_DATA7 * PAN_BLIT_NUM_TYPES)
                 * (8 * 16) * 2;

         dev->blit_shaders.bo = panfrost_bo_create(dev, total_size, PAN_BO_EXECUTE);

         /* Don't bother generating multisampling variants if we don't actually
          * support multisampling */
         bool has_ms = !(dev->quirks & MIDGARD_SFBD);

         for (unsigned ms = 0; ms <= has_ms; ++ms) {
                 for (unsigned i = 0; i < ARRAY_SIZE(shader_descs); ++i) {
                         unsigned loc = shader_descs[i].loc;

                         for (enum pan_blit_type T = 0; T < PAN_BLIT_NUM_TYPES; ++T) {
                                 if (!(shader_descs[i].types & (1 << T)))
                                         continue;

                                 panfrost_program program;
                                 panfrost_build_blit_shader(&program, dev->gpu_id, loc,
                                                 nir_types[T], ms);

                                 assert(offset + program.compiled.size < total_size);
                                 memcpy(dev->blit_shaders.bo->cpu + offset, program.compiled.data, program.compiled.size);

                                 dev->blit_shaders.loads[loc][T][ms] = (dev->blit_shaders.bo->gpu + offset) | program.first_tag;
                                 offset += ALIGN_POT(program.compiled.size, 64);
                                 util_dynarray_fini(&program.compiled);
                         }
                 }
         }
 }

 /* Add a shader-based load on Midgard (draw-time for GL). Shaders are
  * precached */

 void
 panfrost_load_midg(
                 struct pan_pool *pool,
                 struct pan_scoreboard *scoreboard,
                 mali_ptr blend_shader,
                 mali_ptr fbd,
                 mali_ptr coordinates, unsigned vertex_count,
                 struct pan_image *image,
                 unsigned loc)
 {
         bool srgb = util_format_is_srgb(image->format);
         unsigned width = u_minify(image->width0, image->first_level);
         unsigned height = u_minify(image->height0, image->first_level);

         struct panfrost_transfer viewport = panfrost_pool_alloc(pool, MALI_VIEWPORT_LENGTH);
         struct panfrost_transfer sampler = panfrost_pool_alloc(pool, MALI_MIDGARD_SAMPLER_LENGTH);
         struct panfrost_transfer varying = panfrost_pool_alloc(pool, MALI_ATTRIBUTE_LENGTH);
         struct panfrost_transfer varying_buffer  = panfrost_pool_alloc(pool, MALI_ATTRIBUTE_BUFFER_LENGTH);

         pan_pack(viewport.cpu, VIEWPORT, cfg) {
                 cfg.scissor_maximum_x = width - 1; /* Inclusive */
                 cfg.scissor_maximum_y = height - 1;
         }

         pan_pack(varying_buffer.cpu, ATTRIBUTE_BUFFER, cfg) {
                 cfg.pointer = coordinates;
                 cfg.stride = 4 * sizeof(float);
                 cfg.size = cfg.stride * vertex_count;
         }

         pan_pack(varying.cpu, ATTRIBUTE, cfg) {
                 cfg.buffer_index = 0;
                 cfg.format = (MALI_CHANNEL_R << 0) | (MALI_CHANNEL_G << 3) | (MALI_RGBA32F << 12);
         }

         /* Determine the sampler type needed. Stencil is always sampled as
          * UINT. Pure (U)INT is always (U)INT. Everything else is FLOAT. */

         enum pan_blit_type T =
                 (loc == FRAG_RESULT_STENCIL) ? PAN_BLIT_UINT :
                 (util_format_is_pure_uint(image->format)) ? PAN_BLIT_UINT :
                 (util_format_is_pure_sint(image->format)) ? PAN_BLIT_INT :
                 PAN_BLIT_FLOAT;

         bool ms = image->nr_samples > 1;

         struct panfrost_transfer shader_meta_t =
                 panfrost_pool_alloc_aligned(pool,
                                             MALI_RENDERER_STATE_LENGTH +
                                             8 * MALI_BLEND_LENGTH,
                                             128);

         pan_pack(shader_meta_t.cpu, RENDERER_STATE, cfg) {
                 cfg.shader.shader = pool->dev->blit_shaders.loads[loc][T][ms];
                 cfg.shader.varying_count = 1;
                 cfg.shader.texture_count = 1;
                 cfg.shader.sampler_count = 1;

                 cfg.properties.midgard.work_register_count = 4;
                 cfg.properties.midgard.force_early_z = (loc >= FRAG_RESULT_DATA0);
                 cfg.properties.stencil_from_shader = (loc == FRAG_RESULT_STENCIL);
                 cfg.properties.depth_source = (loc == FRAG_RESULT_DEPTH) ?
                                               MALI_DEPTH_SOURCE_SHADER :
                                               MALI_DEPTH_SOURCE_FIXED_FUNCTION;

                 cfg.multisample_misc.sample_mask = 0xFFFF;
                 cfg.multisample_misc.multisample_enable = ms;
                 cfg.multisample_misc.evaluate_per_sample = ms;
                 cfg.multisample_misc.depth_write_mask = (loc == FRAG_RESULT_DEPTH);
                 cfg.multisample_misc.depth_function = MALI_FUNC_ALWAYS;

                 cfg.stencil_mask_misc.stencil_enable = (loc == FRAG_RESULT_STENCIL);
                 cfg.stencil_mask_misc.stencil_mask_front = 0xFF;
                 cfg.stencil_mask_misc.stencil_mask_back = 0xFF;
                 cfg.stencil_mask_misc.unknown_1 = 0x7;

                 cfg.stencil_front.compare_function = MALI_FUNC_ALWAYS;
                 cfg.stencil_front.stencil_fail = MALI_STENCIL_OP_REPLACE;
                 cfg.stencil_front.depth_fail = MALI_STENCIL_OP_REPLACE;
                 cfg.stencil_front.depth_pass = MALI_STENCIL_OP_REPLACE;
                 cfg.stencil_front.mask = 0xFF;

                 cfg.stencil_back = cfg.stencil_front;

                 if (pool->dev->quirks & MIDGARD_SFBD) {
                         cfg.stencil_mask_misc.sfbd_write_enable = true;
                         cfg.stencil_mask_misc.sfbd_dither_disable = true;
                         cfg.stencil_mask_misc.sfbd_srgb = srgb;
                         cfg.multisample_misc.sfbd_blend_shader = !!blend_shader;
                         if (cfg.multisample_misc.sfbd_blend_shader) {
                                 cfg.sfbd_blend_shader = blend_shader;
                         } else {
                                 cfg.sfbd_blend_equation.rgb.a = MALI_BLEND_OPERAND_A_SRC;
                                 cfg.sfbd_blend_equation.rgb.b = MALI_BLEND_OPERAND_B_SRC;
                                 cfg.sfbd_blend_equation.rgb.c = MALI_BLEND_OPERAND_C_ZERO;
                                 cfg.sfbd_blend_equation.alpha.a = MALI_BLEND_OPERAND_A_SRC;
                                 cfg.sfbd_blend_equation.alpha.b = MALI_BLEND_OPERAND_B_SRC;
                                 cfg.sfbd_blend_equation.alpha.c = MALI_BLEND_OPERAND_C_ZERO;

                                 if (loc >= FRAG_RESULT_DATA0)
                                         cfg.sfbd_blend_equation.color_mask = 0xf;
                                 cfg.sfbd_blend_constant = 0;
                         }
                 } else if (!(pool->dev->quirks & IS_BIFROST)) {
                         cfg.sfbd_blend_shader = blend_shader;
                 }

                 assert(cfg.shader.shader);
         }

         /* Create the texture descriptor. We partially compute the base address
          * ourselves to account for layer, such that the texture descriptor
          * itself is for a 2D texture with array size 1 even for 3D/array
          * textures, removing the need to separately key the blit shaders for
          * 2D and 3D variants */

         struct panfrost_transfer texture_t = panfrost_pool_alloc_aligned(
                         pool, MALI_MIDGARD_TEXTURE_LENGTH + sizeof(mali_ptr) * 2 * MAX2(image->nr_samples, 1), 128);

         panfrost_new_texture(texture_t.cpu,
                         image->width0, image->height0,
                         MAX2(image->nr_samples, 1), 1,
                         image->format, MALI_TEXTURE_DIMENSION_2D,
                         image->modifier,
                         image->first_level, image->last_level,
                         0, 0,
                         image->nr_samples,
                         0,
                         (MALI_CHANNEL_R << 0) | (MALI_CHANNEL_G << 3) | (MALI_CHANNEL_B << 6) | (MALI_CHANNEL_A << 9),
                         image->bo->gpu + image->first_layer *
                                 panfrost_get_layer_stride(image->slices,
                                         image->dim == MALI_TEXTURE_DIMENSION_3D,
                                         image->cubemap_stride, image->first_level),
                         image->slices);

         pan_pack(sampler.cpu, MIDGARD_SAMPLER, cfg)
                 cfg.normalized_coordinates = false;

         for (unsigned i = 0; i < 8; ++i) {
                 void *dest = shader_meta_t.cpu + MALI_RENDERER_STATE_LENGTH +
                              MALI_BLEND_LENGTH * i;

                 if (loc != (FRAG_RESULT_DATA0 + i)) {
                         memset(dest, 0x0, MALI_BLEND_LENGTH);
                         continue;
                 }

                 pan_pack(dest, BLEND, cfg) {
                         cfg.round_to_fb_precision = true;
                         cfg.srgb = srgb;
                         if (blend_shader) {
                                 cfg.midgard.blend_shader = true;
                                 cfg.midgard.shader_pc = blend_shader;
                         } else {
                                 cfg.midgard.equation.rgb.a = MALI_BLEND_OPERAND_A_SRC;
                                 cfg.midgard.equation.rgb.b = MALI_BLEND_OPERAND_B_SRC;
                                 cfg.midgard.equation.rgb.c = MALI_BLEND_OPERAND_C_ZERO;
                                 cfg.midgard.equation.alpha.a = MALI_BLEND_OPERAND_A_SRC;
                                 cfg.midgard.equation.alpha.b = MALI_BLEND_OPERAND_B_SRC;
                                 cfg.midgard.equation.alpha.c = MALI_BLEND_OPERAND_C_ZERO;
                                 cfg.midgard.equation.color_mask = 0xf;
                         }
                 }
         }

         struct panfrost_transfer t =
                 panfrost_pool_alloc_aligned(pool, MALI_MIDGARD_TILER_JOB_LENGTH, 64);

         pan_section_pack(t.cpu, MIDGARD_TILER_JOB, DRAW, cfg) {
                 cfg.four_components_per_vertex = true;
                 cfg.draw_descriptor_is_64b = true;
                 cfg.texture_descriptor_is_64b = true;
                 cfg.position = coordinates;
                 cfg.textures = panfrost_pool_upload(pool, &texture_t.gpu, sizeof(texture_t.gpu));
                 cfg.samplers = sampler.gpu;
                 cfg.state = shader_meta_t.gpu;
                 cfg.varying_buffers = varying_buffer.gpu;
                 cfg.varyings = varying.gpu;
                 cfg.viewport = viewport.gpu;
                 cfg.fbd = fbd;
         }

         pan_section_pack(t.cpu, MIDGARD_TILER_JOB, PRIMITIVE, cfg) {
                 cfg.draw_mode = MALI_DRAW_MODE_TRIANGLES;
                 cfg.index_count = vertex_count;
                 cfg.job_task_split = 6;
         }

         panfrost_pack_work_groups_compute(pan_section_ptr(t.cpu, MIDGARD_TILER_JOB, INVOCATION),
                                           1, vertex_count, 1, 1, 1, 1, true);

         panfrost_add_job(pool, scoreboard, MALI_JOB_TYPE_TILER, false, 0, &t, true);
 }
	/*
	* Copyright (C) 2020 Collabora, Ltd.
	*
	* Permission is hereby granted, free of charge, to any person obtaining a
	* copy of this software and associated documentation files (the "Software"),
	* to deal in the Software without restriction, including without limitation
	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
	* and/or sell copies of the Software, and to permit persons to whom the
	* Software is furnished to do so, subject to the following conditions:
	*
	* The above copyright notice and this permission notice (including the next
	* paragraph) shall be included in all copies or substantial portions of the
	* Software.
	*
	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
	* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	* SOFTWARE.
	*
	* Authors:
	* Alyssa Rosenzweig <alyssa.rosenzweig@collabora.com>
	*/

	#include <math.h>
	#include <stdio.h>
	#include "pan_encoder.h"
	#include "pan_pool.h"
	#include "pan_scoreboard.h"
	#include "pan_texture.h"
	#include "panfrost-quirks.h"
	#include "../midgard/midgard_compile.h"
	#include "compiler/nir/nir_builder.h"
	#include "util/u_math.h"

	/* On Midgard, the native blit infrastructure (via MFBD preloads) is broken or
	* missing in many cases. We instead use software paths as fallbacks to
	* implement blits, which are done as TILER jobs. No vertex shader is
	* necessary since we can supply screen-space coordinates directly.
	*
	* This is primarily designed as a fallback for preloads but could be extended
	* for other clears/blits if needed in the future. */

	static void
	panfrost_build_blit_shader(panfrost_program *program, unsigned gpu_id, gl_frag_result loc, nir_alu_type T, bool ms)
	{
	bool is_colour = loc >= FRAG_RESULT_DATA0;

	nir_builder _b;
	nir_builder_init_simple_shader(&_b, NULL, MESA_SHADER_FRAGMENT, &midgard_nir_options);
	nir_builder *b = &_b;
	nir_shader *shader = b->shader;

	shader->info.internal = true;

	nir_variable *c_src = nir_variable_create(shader, nir_var_shader_in, glsl_vector_type(GLSL_TYPE_FLOAT, 2), "coord");
	nir_variable *c_out = nir_variable_create(shader, nir_var_shader_out, glsl_vector_type(
	GLSL_TYPE_FLOAT, is_colour ? 4 : 1), "out");

	c_src->data.location = VARYING_SLOT_TEX0;
	c_out->data.location = loc;

	nir_ssa_def *coord = nir_load_var(b, c_src);

	nir_tex_instr *tex = nir_tex_instr_create(shader, ms ? 3 : 1);

	tex->dest_type = T;

	if (ms) {
	tex->src[0].src_type = nir_tex_src_coord;
	tex->src[0].src = nir_src_for_ssa(nir_f2i32(b, coord));
	tex->coord_components = 2;

	tex->src[1].src_type = nir_tex_src_ms_index;
	tex->src[1].src = nir_src_for_ssa(nir_load_sample_id(b));

	tex->src[2].src_type = nir_tex_src_lod;
	tex->src[2].src = nir_src_for_ssa(nir_imm_int(b, 0));
	tex->sampler_dim = GLSL_SAMPLER_DIM_MS;
	tex->op = nir_texop_txf_ms;
	} else {
	tex->op = nir_texop_tex;

	tex->src[0].src_type = nir_tex_src_coord;
	tex->src[0].src = nir_src_for_ssa(coord);
	tex->coord_components = 2;

	tex->sampler_dim = GLSL_SAMPLER_DIM_2D;
	}

	nir_ssa_dest_init(&tex->instr, &tex->dest, 4, 32, NULL);
	nir_builder_instr_insert(b, &tex->instr);

	if (is_colour)
	nir_store_var(b, c_out, &tex->dest.ssa, 0xFF);
	else
	nir_store_var(b, c_out, nir_channel(b, &tex->dest.ssa, 0), 0xFF);

	struct panfrost_compile_inputs inputs = {
	.gpu_id = gpu_id,
	};

	midgard_compile_shader_nir(shader, program, &inputs);
	ralloc_free(shader);
	}

	/* Compile and upload all possible blit shaders ahead-of-time to reduce draw
	* time overhead. There's only ~30 of them at the moment, so this is fine */

	void
	panfrost_init_blit_shaders(struct panfrost_device *dev)
	{
	static const struct {
	gl_frag_result loc;
	unsigned types;
	} shader_descs[] = {
	{ FRAG_RESULT_DEPTH, 1 << PAN_BLIT_FLOAT },
	{ FRAG_RESULT_STENCIL, 1 << PAN_BLIT_UINT },
	{ FRAG_RESULT_DATA0, ~0 },
	{ FRAG_RESULT_DATA1, ~0 },
	{ FRAG_RESULT_DATA2, ~0 },
	{ FRAG_RESULT_DATA3, ~0 },
	{ FRAG_RESULT_DATA4, ~0 },
	{ FRAG_RESULT_DATA5, ~0 },
	{ FRAG_RESULT_DATA6, ~0 },
	{ FRAG_RESULT_DATA7, ~0 }
	};

	nir_alu_type nir_types[PAN_BLIT_NUM_TYPES] = {
	nir_type_float,
	nir_type_uint,
	nir_type_int
	};

	/* Total size = # of shaders * bytes per shader. There are
	* shaders for each RT (so up to DATA7 -- overestimate is
	* okay) and up to NUM_TYPES variants of each, * 2 for multisampling
	* variants. These shaders are simple enough that they should be less
	* than 8 quadwords each (again, overestimate is fine). */

	unsigned offset = 0;
	unsigned total_size = (FRAG_RESULT_DATA7 * PAN_BLIT_NUM_TYPES)
	* (8 * 16) * 2;

	dev->blit_shaders.bo = panfrost_bo_create(dev, total_size, PAN_BO_EXECUTE);

	/* Don't bother generating multisampling variants if we don't actually
	* support multisampling */
	bool has_ms = !(dev->quirks & MIDGARD_SFBD);

	for (unsigned ms = 0; ms <= has_ms; ++ms) {
	for (unsigned i = 0; i < ARRAY_SIZE(shader_descs); ++i) {
	unsigned loc = shader_descs[i].loc;

	for (enum pan_blit_type T = 0; T < PAN_BLIT_NUM_TYPES; ++T) {
	if (!(shader_descs[i].types & (1 << T)))
	continue;

	panfrost_program program;
	panfrost_build_blit_shader(&program, dev->gpu_id, loc,
	nir_types[T], ms);

	assert(offset + program.compiled.size < total_size);
	memcpy(dev->blit_shaders.bo->cpu + offset, program.compiled.data, program.compiled.size);

	dev->blit_shaders.loads[loc][T][ms] = (dev->blit_shaders.bo->gpu + offset) \| program.first_tag;
	offset += ALIGN_POT(program.compiled.size, 64);
	util_dynarray_fini(&program.compiled);
	}
	}
	}
	}

	/* Add a shader-based load on Midgard (draw-time for GL). Shaders are
	* precached */

	void
	panfrost_load_midg(
	struct pan_pool *pool,
	struct pan_scoreboard *scoreboard,
	mali_ptr blend_shader,
	mali_ptr fbd,
	mali_ptr coordinates, unsigned vertex_count,
	struct pan_image *image,
	unsigned loc)
	{
	bool srgb = util_format_is_srgb(image->format);
	unsigned width = u_minify(image->width0, image->first_level);
	unsigned height = u_minify(image->height0, image->first_level);

	struct panfrost_transfer viewport = panfrost_pool_alloc(pool, MALI_VIEWPORT_LENGTH);
	struct panfrost_transfer sampler = panfrost_pool_alloc(pool, MALI_MIDGARD_SAMPLER_LENGTH);
	struct panfrost_transfer varying = panfrost_pool_alloc(pool, MALI_ATTRIBUTE_LENGTH);
	struct panfrost_transfer varying_buffer = panfrost_pool_alloc(pool, MALI_ATTRIBUTE_BUFFER_LENGTH);

	pan_pack(viewport.cpu, VIEWPORT, cfg) {
	cfg.scissor_maximum_x = width - 1; /* Inclusive */
	cfg.scissor_maximum_y = height - 1;
	}

	pan_pack(varying_buffer.cpu, ATTRIBUTE_BUFFER, cfg) {
	cfg.pointer = coordinates;
	cfg.stride = 4 * sizeof(float);
	cfg.size = cfg.stride * vertex_count;
	}

	pan_pack(varying.cpu, ATTRIBUTE, cfg) {
	cfg.buffer_index = 0;
	cfg.format = (MALI_CHANNEL_R << 0) \| (MALI_CHANNEL_G << 3) \| (MALI_RGBA32F << 12);
	}

	/* Determine the sampler type needed. Stencil is always sampled as
	* UINT. Pure (U)INT is always (U)INT. Everything else is FLOAT. */

	enum pan_blit_type T =
	(loc == FRAG_RESULT_STENCIL) ? PAN_BLIT_UINT :
	(util_format_is_pure_uint(image->format)) ? PAN_BLIT_UINT :
	(util_format_is_pure_sint(image->format)) ? PAN_BLIT_INT :
	PAN_BLIT_FLOAT;

	bool ms = image->nr_samples > 1;

	struct panfrost_transfer shader_meta_t =
	panfrost_pool_alloc_aligned(pool,
	MALI_RENDERER_STATE_LENGTH +
	8 * MALI_BLEND_LENGTH,
	128);

	pan_pack(shader_meta_t.cpu, RENDERER_STATE, cfg) {
	cfg.shader.shader = pool->dev->blit_shaders.loads[loc][T][ms];
	cfg.shader.varying_count = 1;
	cfg.shader.texture_count = 1;
	cfg.shader.sampler_count = 1;

	cfg.properties.midgard.work_register_count = 4;
	cfg.properties.midgard.force_early_z = (loc >= FRAG_RESULT_DATA0);
	cfg.properties.stencil_from_shader = (loc == FRAG_RESULT_STENCIL);
	cfg.properties.depth_source = (loc == FRAG_RESULT_DEPTH) ?
	MALI_DEPTH_SOURCE_SHADER :
	MALI_DEPTH_SOURCE_FIXED_FUNCTION;

	cfg.multisample_misc.sample_mask = 0xFFFF;
	cfg.multisample_misc.multisample_enable = ms;
	cfg.multisample_misc.evaluate_per_sample = ms;
	cfg.multisample_misc.depth_write_mask = (loc == FRAG_RESULT_DEPTH);
	cfg.multisample_misc.depth_function = MALI_FUNC_ALWAYS;

	cfg.stencil_mask_misc.stencil_enable = (loc == FRAG_RESULT_STENCIL);
	cfg.stencil_mask_misc.stencil_mask_front = 0xFF;
	cfg.stencil_mask_misc.stencil_mask_back = 0xFF;
	cfg.stencil_mask_misc.unknown_1 = 0x7;

	cfg.stencil_front.compare_function = MALI_FUNC_ALWAYS;
	cfg.stencil_front.stencil_fail = MALI_STENCIL_OP_REPLACE;
	cfg.stencil_front.depth_fail = MALI_STENCIL_OP_REPLACE;
	cfg.stencil_front.depth_pass = MALI_STENCIL_OP_REPLACE;
	cfg.stencil_front.mask = 0xFF;

	cfg.stencil_back = cfg.stencil_front;

	if (pool->dev->quirks & MIDGARD_SFBD) {
	cfg.stencil_mask_misc.sfbd_write_enable = true;
	cfg.stencil_mask_misc.sfbd_dither_disable = true;
	cfg.stencil_mask_misc.sfbd_srgb = srgb;
	cfg.multisample_misc.sfbd_blend_shader = !!blend_shader;
	if (cfg.multisample_misc.sfbd_blend_shader) {
	cfg.sfbd_blend_shader = blend_shader;
	} else {
	cfg.sfbd_blend_equation.rgb.a = MALI_BLEND_OPERAND_A_SRC;
	cfg.sfbd_blend_equation.rgb.b = MALI_BLEND_OPERAND_B_SRC;
	cfg.sfbd_blend_equation.rgb.c = MALI_BLEND_OPERAND_C_ZERO;
	cfg.sfbd_blend_equation.alpha.a = MALI_BLEND_OPERAND_A_SRC;
	cfg.sfbd_blend_equation.alpha.b = MALI_BLEND_OPERAND_B_SRC;
	cfg.sfbd_blend_equation.alpha.c = MALI_BLEND_OPERAND_C_ZERO;

	if (loc >= FRAG_RESULT_DATA0)
	cfg.sfbd_blend_equation.color_mask = 0xf;
	cfg.sfbd_blend_constant = 0;
	}
	} else if (!(pool->dev->quirks & IS_BIFROST)) {
	cfg.sfbd_blend_shader = blend_shader;
	}

	assert(cfg.shader.shader);
	}

	/* Create the texture descriptor. We partially compute the base address
	* ourselves to account for layer, such that the texture descriptor
	* itself is for a 2D texture with array size 1 even for 3D/array
	* textures, removing the need to separately key the blit shaders for
	* 2D and 3D variants */

	struct panfrost_transfer texture_t = panfrost_pool_alloc_aligned(
	pool, MALI_MIDGARD_TEXTURE_LENGTH + sizeof(mali_ptr) * 2 * MAX2(image->nr_samples, 1), 128);

	panfrost_new_texture(texture_t.cpu,
	image->width0, image->height0,
	MAX2(image->nr_samples, 1), 1,
	image->format, MALI_TEXTURE_DIMENSION_2D,
	image->modifier,
	image->first_level, image->last_level,
	0, 0,
	image->nr_samples,
	0,
	(MALI_CHANNEL_R << 0) \| (MALI_CHANNEL_G << 3) \| (MALI_CHANNEL_B << 6) \| (MALI_CHANNEL_A << 9),
	image->bo->gpu + image->first_layer *
	panfrost_get_layer_stride(image->slices,
	image->dim == MALI_TEXTURE_DIMENSION_3D,
	image->cubemap_stride, image->first_level),
	image->slices);

	pan_pack(sampler.cpu, MIDGARD_SAMPLER, cfg)
	cfg.normalized_coordinates = false;

	for (unsigned i = 0; i < 8; ++i) {
	void *dest = shader_meta_t.cpu + MALI_RENDERER_STATE_LENGTH +
	MALI_BLEND_LENGTH * i;

	if (loc != (FRAG_RESULT_DATA0 + i)) {
	memset(dest, 0x0, MALI_BLEND_LENGTH);
	continue;
	}

	pan_pack(dest, BLEND, cfg) {
	cfg.round_to_fb_precision = true;
	cfg.srgb = srgb;
	if (blend_shader) {
	cfg.midgard.blend_shader = true;
	cfg.midgard.shader_pc = blend_shader;
	} else {
	cfg.midgard.equation.rgb.a = MALI_BLEND_OPERAND_A_SRC;
	cfg.midgard.equation.rgb.b = MALI_BLEND_OPERAND_B_SRC;
	cfg.midgard.equation.rgb.c = MALI_BLEND_OPERAND_C_ZERO;
	cfg.midgard.equation.alpha.a = MALI_BLEND_OPERAND_A_SRC;
	cfg.midgard.equation.alpha.b = MALI_BLEND_OPERAND_B_SRC;
	cfg.midgard.equation.alpha.c = MALI_BLEND_OPERAND_C_ZERO;
	cfg.midgard.equation.color_mask = 0xf;
	}
	}
	}

	struct panfrost_transfer t =
	panfrost_pool_alloc_aligned(pool, MALI_MIDGARD_TILER_JOB_LENGTH, 64);

	pan_section_pack(t.cpu, MIDGARD_TILER_JOB, DRAW, cfg) {
	cfg.four_components_per_vertex = true;
	cfg.draw_descriptor_is_64b = true;
	cfg.texture_descriptor_is_64b = true;
	cfg.position = coordinates;
	cfg.textures = panfrost_pool_upload(pool, &texture_t.gpu, sizeof(texture_t.gpu));
	cfg.samplers = sampler.gpu;
	cfg.state = shader_meta_t.gpu;
	cfg.varying_buffers = varying_buffer.gpu;
	cfg.varyings = varying.gpu;
	cfg.viewport = viewport.gpu;
	cfg.fbd = fbd;
	}

	pan_section_pack(t.cpu, MIDGARD_TILER_JOB, PRIMITIVE, cfg) {
	cfg.draw_mode = MALI_DRAW_MODE_TRIANGLES;
	cfg.index_count = vertex_count;
	cfg.job_task_split = 6;
	}

	panfrost_pack_work_groups_compute(pan_section_ptr(t.cpu, MIDGARD_TILER_JOB, INVOCATION),
	1, vertex_count, 1, 1, 1, 1, true);

	panfrost_add_job(pool, scoreboard, MALI_JOB_TYPE_TILER, false, 0, &t, true);
	}