src/gallium/drivers/vc4/vc4_tiling_lt.c - platform/external/mesa3d - Git at Google

 /*
  * Copyright © 2017 Broadcom
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
  * to deal in the Software without restriction, including without limitation
  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  * and/or sell copies of the Software, and to permit persons to whom the
  * Software is furnished to do so, subject to the following conditions:
  *
  * The above copyright notice and this permission notice (including the next
  * paragraph) shall be included in all copies or substantial portions of the
  * Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  * IN THE SOFTWARE.
  */

 /** @file vc4_tiling_lt.c
  *
  * Helper functions from vc4_tiling.c that will be compiled for using NEON
  * assembly or not.
  *
  * If V3D_BUILD_NEON is set, then the functions will be suffixed with _neon.
  * They will only use NEON assembly if __ARM_ARCH is also set, to keep the x86
  * sim build working.
  */

 #include <string.h>
 #include "pipe/p_state.h"
 #include "vc4_tiling.h"
 #include "broadcom/common/v3d_cpu_tiling.h"

 #ifdef V3D_BUILD_NEON
 #define NEON_TAG(x) x ## _neon
 #else
 #define NEON_TAG(x) x ## _base
 #endif

 static inline uint32_t
 align_down(uint32_t val, uint32_t align)
 {
         return val & ~(align - 1);
 }

 /** Returns the stride in bytes of a 64-byte microtile. */
 static uint32_t
 vc4_utile_stride(int cpp)
 {
         switch (cpp) {
         case 1:
                 return 8;
         case 2:
         case 4:
         case 8:
                 return 16;
         default:
                 unreachable("bad cpp");
         }
 }

 /**
  * Returns the X value into the address bits for LT tiling.
  *
  * The LT tile load/stores rely on the X bits not intersecting with the Y
  * bits.  Because of this, we have to choose to put the utile index within the
  * LT tile into one of the two values, and we do so in swizzle_lt_x() to make
  * NPOT handling easier.
  */
 static uint32_t
 swizzle_lt_x(int x, int cpp)
 {
         switch (cpp) {
         case 1:
                 /* 8x8 inside of 4x4 */
                 return ((x & 0x7) << (0 - 0) |
                         (x & ~0x7) << (6 - 3));
         case 2:
                 /* 8x4 inside of 4x4 */
                 return ((x & 0x7) << (1 - 0) |
                         (x & ~0x7) << (6 - 3));
         case 4:
                 /* 4x4 inside of 4x4 */
                 return ((x & 0x3) << (2 - 0) |
                         (x & ~0x3) << (6 - 2));
         case 8:
                 /* 2x4 inside of 4x4 */
                 return ((x & 0x1) << (3 - 0) |
                         (x & ~0x1) << (6 - 1));
         default:
                 unreachable("bad cpp");
         }
 }

 /**
  * Returns the Y value into the address bits for LT tiling.
  *
  * The LT tile load/stores rely on the X bits not intersecting with the Y
  * bits.
  */
 static uint32_t
 swizzle_lt_y(int y, int cpp)
 {

         switch (cpp) {
         case 1:
                 /* 8x8 inside of 4x4 */
                 return ((y & 0x7) << 3);
         case 2:
                 /* 8x4 inside of 4x4 */
                 return ((y & 0x3) << 4);
         case 4:
                 /* 4x4 inside of 4x4 */
                 return ((y & 0x3) << 4);
         case 8:
                 /* 2x4 inside of 4x4 */
                 return ((y & 0x3) << 4);
         default:
                 unreachable("bad cpp");
         }
 }

 /**
  * Helper for loading or storing to an LT image, where the box is aligned
  * to utiles.
  *
  * This just breaks the box down into calls to the fast
  * vc4_load_utile/vc4_store_utile helpers.
  */
 static inline void
 vc4_lt_image_aligned(void *gpu, uint32_t gpu_stride,
                      void *cpu, uint32_t cpu_stride,
                      int cpp, const struct pipe_box *box, bool to_cpu)
 {
         uint32_t utile_w = vc4_utile_width(cpp);
         uint32_t utile_h = vc4_utile_height(cpp);
         uint32_t utile_stride = vc4_utile_stride(cpp);
         uint32_t xstart = box->x;
         uint32_t ystart = box->y;

         for (uint32_t y = 0; y < box->height; y += utile_h) {
                 for (uint32_t x = 0; x < box->width; x += utile_w) {
                         void *gpu_tile = gpu + ((ystart + y) * gpu_stride +
                                                 (xstart + x) * 64 / utile_w);
                         if (to_cpu) {
                                 v3d_load_utile(cpu + (cpu_stride * y +
                                                       x * cpp),
                                                cpu_stride,
                                                gpu_tile,
                                                utile_stride);
                         } else {
                                 v3d_store_utile(gpu_tile,
                                                 utile_stride,
                                                 cpu + (cpu_stride * y +
                                                        x * cpp),
                                                 cpu_stride);
                         }
                 }
         }
 }

 /**
  * Helper for loading or storing to an LT image, where the box is not aligned
  * to utiles.
  *
  * This walks through the raster-order data, copying to/from the corresponding
  * tiled pixel.  This means we don't get write-combining on stores, but the
  * loop is very few CPU instructions since the memcpy will be inlined.
  */
 static inline void
 vc4_lt_image_unaligned(void *gpu, uint32_t gpu_stride,
                        void *cpu, uint32_t cpu_stride,
                        int cpp, const struct pipe_box *box, bool to_cpu)
 {

         /* These are the address bits for the start of the box, split out into
          * x/y so that they can be incremented separately in their loops.
          */
         uint32_t offs_x0 = swizzle_lt_x(box->x, cpp);
         uint32_t offs_y = swizzle_lt_y(box->y, cpp);
         /* The *_mask values are "what bits of the address are from x or y" */
         uint32_t x_mask = swizzle_lt_x(~0, cpp);
         uint32_t y_mask = swizzle_lt_y(~0, cpp);
         uint32_t incr_y = swizzle_lt_x(gpu_stride / cpp, cpp);

         assert(!(x_mask & y_mask));

         offs_x0 += incr_y * (box->y / vc4_utile_height(cpp));

         for (uint32_t y = 0; y < box->height; y++) {
                 void *gpu_row = gpu + offs_y;

                 uint32_t offs_x = offs_x0;

                 for (uint32_t x = 0; x < box->width; x++) {
                         /* Use a memcpy here to move a pixel's worth of data.
                          * We're relying on this function to be inlined, so
                          * this will get expanded into the appropriate 1, 2,
                          * or 4-byte move.
                          */
                         if (to_cpu) {
                                 memcpy(cpu + x * cpp, gpu_row + offs_x, cpp);
                         } else {
                                 memcpy(gpu_row + offs_x, cpu + x * cpp, cpp);
                         }

                         /* This math trick with x_mask increments offs_x by 1
                          * in x.
                          */
                         offs_x = (offs_x - x_mask) & x_mask;
                 }

                 offs_y = (offs_y - y_mask) & y_mask;
                 /* When offs_y wraps (we hit the end of the utile), we
                  * increment offs_x0 by effectively the utile stride.
                  */
                 if (!offs_y)
                         offs_x0 += incr_y;

                 cpu += cpu_stride;
         }
 }

 /**
  * General LT image load/store helper.
  */
 static inline void
 vc4_lt_image_helper(void *gpu, uint32_t gpu_stride,
                     void *cpu, uint32_t cpu_stride,
                     int cpp, const struct pipe_box *box, bool to_cpu)
 {
         if (box->x & (vc4_utile_width(cpp) - 1) ||
             box->y & (vc4_utile_height(cpp) - 1) ||
             box->width & (vc4_utile_width(cpp) - 1) ||
             box->height & (vc4_utile_height(cpp) - 1)) {
                 vc4_lt_image_unaligned(gpu, gpu_stride,
                                        cpu, cpu_stride,
                                        cpp, box, to_cpu);
         } else {
                 vc4_lt_image_aligned(gpu, gpu_stride,
                                      cpu, cpu_stride,
                                      cpp, box, to_cpu);
         }
 }

 static inline void
 vc4_lt_image_cpp_helper(void *gpu, uint32_t gpu_stride,
                         void *cpu, uint32_t cpu_stride,
                         int cpp, const struct pipe_box *box, bool to_cpu)
 {
         switch (cpp) {
         case 1:
                 vc4_lt_image_helper(gpu, gpu_stride, cpu, cpu_stride, 1, box,
                                     to_cpu);
                 break;
         case 2:
                 vc4_lt_image_helper(gpu, gpu_stride, cpu, cpu_stride, 2, box,
                                     to_cpu);
                 break;
         case 4:
                 vc4_lt_image_helper(gpu, gpu_stride, cpu, cpu_stride, 4, box,
                                     to_cpu);
                 break;
         case 8:
                 vc4_lt_image_helper(gpu, gpu_stride, cpu, cpu_stride, 8, box,
                                     to_cpu);
                 break;
         default:
                 unreachable("bad cpp");
         }
 }

 void
 NEON_TAG(vc4_load_lt_image)(void *dst, uint32_t dst_stride,
                             void *src, uint32_t src_stride,
                             int cpp, const struct pipe_box *box)
 {
         vc4_lt_image_cpp_helper(src, src_stride, dst, dst_stride, cpp, box,
                                 true);
 }

 void
 NEON_TAG(vc4_store_lt_image)(void *dst, uint32_t dst_stride,
                              void *src, uint32_t src_stride,
                              int cpp, const struct pipe_box *box)
 {
         vc4_lt_image_cpp_helper(dst, dst_stride, src, src_stride, cpp, box,
                                 false);
 }
	/*
	* Copyright © 2017 Broadcom
	*
	* Permission is hereby granted, free of charge, to any person obtaining a
	* copy of this software and associated documentation files (the "Software"),
	* to deal in the Software without restriction, including without limitation
	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
	* and/or sell copies of the Software, and to permit persons to whom the
	* Software is furnished to do so, subject to the following conditions:
	*
	* The above copyright notice and this permission notice (including the next
	* paragraph) shall be included in all copies or substantial portions of the
	* Software.
	*
	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
	* IN THE SOFTWARE.
	*/

	/** @file vc4_tiling_lt.c
	*
	* Helper functions from vc4_tiling.c that will be compiled for using NEON
	* assembly or not.
	*
	* If V3D_BUILD_NEON is set, then the functions will be suffixed with _neon.
	* They will only use NEON assembly if __ARM_ARCH is also set, to keep the x86
	* sim build working.
	*/

	#include <string.h>
	#include "pipe/p_state.h"
	#include "vc4_tiling.h"
	#include "broadcom/common/v3d_cpu_tiling.h"

	#ifdef V3D_BUILD_NEON
	#define NEON_TAG(x) x ## _neon
	#else
	#define NEON_TAG(x) x ## _base
	#endif

	static inline uint32_t
	align_down(uint32_t val, uint32_t align)
	{
	return val & ~(align - 1);
	}

	/** Returns the stride in bytes of a 64-byte microtile. */
	static uint32_t
	vc4_utile_stride(int cpp)
	{
	switch (cpp) {
	case 1:
	return 8;
	case 2:
	case 4:
	case 8:
	return 16;
	default:
	unreachable("bad cpp");
	}
	}

	/**
	* Returns the X value into the address bits for LT tiling.
	*
	* The LT tile load/stores rely on the X bits not intersecting with the Y
	* bits. Because of this, we have to choose to put the utile index within the
	* LT tile into one of the two values, and we do so in swizzle_lt_x() to make
	* NPOT handling easier.
	*/
	static uint32_t
	swizzle_lt_x(int x, int cpp)
	{
	switch (cpp) {
	case 1:
	/* 8x8 inside of 4x4 */
	return ((x & 0x7) << (0 - 0) \|
	(x & ~0x7) << (6 - 3));
	case 2:
	/* 8x4 inside of 4x4 */
	return ((x & 0x7) << (1 - 0) \|
	(x & ~0x7) << (6 - 3));
	case 4:
	/* 4x4 inside of 4x4 */
	return ((x & 0x3) << (2 - 0) \|
	(x & ~0x3) << (6 - 2));
	case 8:
	/* 2x4 inside of 4x4 */
	return ((x & 0x1) << (3 - 0) \|
	(x & ~0x1) << (6 - 1));
	default:
	unreachable("bad cpp");
	}
	}

	/**
	* Returns the Y value into the address bits for LT tiling.
	*
	* The LT tile load/stores rely on the X bits not intersecting with the Y
	* bits.
	*/
	static uint32_t
	swizzle_lt_y(int y, int cpp)
	{

	switch (cpp) {
	case 1:
	/* 8x8 inside of 4x4 */
	return ((y & 0x7) << 3);
	case 2:
	/* 8x4 inside of 4x4 */
	return ((y & 0x3) << 4);
	case 4:
	/* 4x4 inside of 4x4 */
	return ((y & 0x3) << 4);
	case 8:
	/* 2x4 inside of 4x4 */
	return ((y & 0x3) << 4);
	default:
	unreachable("bad cpp");
	}
	}

	/**
	* Helper for loading or storing to an LT image, where the box is aligned
	* to utiles.
	*
	* This just breaks the box down into calls to the fast
	* vc4_load_utile/vc4_store_utile helpers.
	*/
	static inline void
	vc4_lt_image_aligned(void *gpu, uint32_t gpu_stride,
	void *cpu, uint32_t cpu_stride,
	int cpp, const struct pipe_box *box, bool to_cpu)
	{
	uint32_t utile_w = vc4_utile_width(cpp);
	uint32_t utile_h = vc4_utile_height(cpp);
	uint32_t utile_stride = vc4_utile_stride(cpp);
	uint32_t xstart = box->x;
	uint32_t ystart = box->y;

	for (uint32_t y = 0; y < box->height; y += utile_h) {
	for (uint32_t x = 0; x < box->width; x += utile_w) {
	void gpu_tile = gpu + ((ystart + y) gpu_stride +
	(xstart + x) * 64 / utile_w);
	if (to_cpu) {
	v3d_load_utile(cpu + (cpu_stride * y +
	x * cpp),
	cpu_stride,
	gpu_tile,
	utile_stride);
	} else {
	v3d_store_utile(gpu_tile,
	utile_stride,
	cpu + (cpu_stride * y +
	x * cpp),
	cpu_stride);
	}
	}
	}
	}

	/**
	* Helper for loading or storing to an LT image, where the box is not aligned
	* to utiles.
	*
	* This walks through the raster-order data, copying to/from the corresponding
	* tiled pixel. This means we don't get write-combining on stores, but the
	* loop is very few CPU instructions since the memcpy will be inlined.
	*/
	static inline void
	vc4_lt_image_unaligned(void *gpu, uint32_t gpu_stride,
	void *cpu, uint32_t cpu_stride,
	int cpp, const struct pipe_box *box, bool to_cpu)
	{

	/* These are the address bits for the start of the box, split out into
	* x/y so that they can be incremented separately in their loops.
	*/
	uint32_t offs_x0 = swizzle_lt_x(box->x, cpp);
	uint32_t offs_y = swizzle_lt_y(box->y, cpp);
	/* The _mask values are "what bits of the address are from x or y" /
	uint32_t x_mask = swizzle_lt_x(~0, cpp);
	uint32_t y_mask = swizzle_lt_y(~0, cpp);
	uint32_t incr_y = swizzle_lt_x(gpu_stride / cpp, cpp);

	assert(!(x_mask & y_mask));

	offs_x0 += incr_y * (box->y / vc4_utile_height(cpp));

	for (uint32_t y = 0; y < box->height; y++) {
	void *gpu_row = gpu + offs_y;

	uint32_t offs_x = offs_x0;

	for (uint32_t x = 0; x < box->width; x++) {
	/* Use a memcpy here to move a pixel's worth of data.
	* We're relying on this function to be inlined, so
	* this will get expanded into the appropriate 1, 2,
	* or 4-byte move.
	*/
	if (to_cpu) {
	memcpy(cpu + x * cpp, gpu_row + offs_x, cpp);
	} else {
	memcpy(gpu_row + offs_x, cpu + x * cpp, cpp);
	}

	/* This math trick with x_mask increments offs_x by 1
	* in x.
	*/
	offs_x = (offs_x - x_mask) & x_mask;
	}

	offs_y = (offs_y - y_mask) & y_mask;
	/* When offs_y wraps (we hit the end of the utile), we
	* increment offs_x0 by effectively the utile stride.
	*/
	if (!offs_y)
	offs_x0 += incr_y;

	cpu += cpu_stride;
	}
	}

	/**
	* General LT image load/store helper.
	*/
	static inline void
	vc4_lt_image_helper(void *gpu, uint32_t gpu_stride,
	void *cpu, uint32_t cpu_stride,
	int cpp, const struct pipe_box *box, bool to_cpu)
	{
	if (box->x & (vc4_utile_width(cpp) - 1) \|\|
	box->y & (vc4_utile_height(cpp) - 1) \|\|
	box->width & (vc4_utile_width(cpp) - 1) \|\|
	box->height & (vc4_utile_height(cpp) - 1)) {
	vc4_lt_image_unaligned(gpu, gpu_stride,
	cpu, cpu_stride,
	cpp, box, to_cpu);
	} else {
	vc4_lt_image_aligned(gpu, gpu_stride,
	cpu, cpu_stride,
	cpp, box, to_cpu);
	}
	}

	static inline void
	vc4_lt_image_cpp_helper(void *gpu, uint32_t gpu_stride,
	void *cpu, uint32_t cpu_stride,
	int cpp, const struct pipe_box *box, bool to_cpu)
	{
	switch (cpp) {
	case 1:
	vc4_lt_image_helper(gpu, gpu_stride, cpu, cpu_stride, 1, box,
	to_cpu);
	break;
	case 2:
	vc4_lt_image_helper(gpu, gpu_stride, cpu, cpu_stride, 2, box,
	to_cpu);
	break;
	case 4:
	vc4_lt_image_helper(gpu, gpu_stride, cpu, cpu_stride, 4, box,
	to_cpu);
	break;
	case 8:
	vc4_lt_image_helper(gpu, gpu_stride, cpu, cpu_stride, 8, box,
	to_cpu);
	break;
	default:
	unreachable("bad cpp");
	}
	}

	void
	NEON_TAG(vc4_load_lt_image)(void *dst, uint32_t dst_stride,
	void *src, uint32_t src_stride,
	int cpp, const struct pipe_box *box)
	{
	vc4_lt_image_cpp_helper(src, src_stride, dst, dst_stride, cpp, box,
	true);
	}

	void
	NEON_TAG(vc4_store_lt_image)(void *dst, uint32_t dst_stride,
	void *src, uint32_t src_stride,
	int cpp, const struct pipe_box *box)
	{
	vc4_lt_image_cpp_helper(dst, dst_stride, src, src_stride, cpp, box,
	false);
	}