radeonsi: fix guardband handling for large values
When minx = -32768 (ViewportBounds.Min) we'll hit the "left <= -1, ..."
assert because 'left' is computed as:
(-65535 / 2 - translate_x) / (minx - translate_x)
This commit fixes the problem by using the full max_viewport_size => [-32768, 32767]
instead of [-32767, 32767] for SI_QUANT_MODE_16_8_FIXED_POINT_1_256TH.
Closes: https://gitlab.freedesktop.org/mesa/mesa/-/issues/3502
Reviewed-by: Marek Olšák <marek.olsak@amd.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/6685>
diff --git a/src/gallium/drivers/radeonsi/si_state_viewport.c b/src/gallium/drivers/radeonsi/si_state_viewport.c
index 2db1e2b..9d62b2c 100644
--- a/src/gallium/drivers/radeonsi/si_state_viewport.c
+++ b/src/gallium/drivers/radeonsi/si_state_viewport.c
@@ -273,13 +273,15 @@
* This is done by applying the inverse viewport transformation
* on the viewport limits to get those limits in clip space.
*
- * The viewport range is [-max_viewport_size/2, max_viewport_size/2].
+ * The viewport range is [-max_viewport_size/2 - 1, max_viewport_size/2].
+ * (-1 to the min coord because max_viewport_size is odd and ViewportBounds
+ * Min/Max are -32768, 32767).
*/
assert(vp_as_scissor.quant_mode < ARRAY_SIZE(max_viewport_size));
max_range = max_viewport_size[vp_as_scissor.quant_mode] / 2;
- left = (-max_range - vp.translate[0]) / vp.scale[0];
+ left = (-max_range - 1 - vp.translate[0]) / vp.scale[0];
right = (max_range - vp.translate[0]) / vp.scale[0];
- top = (-max_range - vp.translate[1]) / vp.scale[1];
+ top = (-max_range - 1 - vp.translate[1]) / vp.scale[1];
bottom = (max_range - vp.translate[1]) / vp.scale[1];
assert(left <= -1 && top <= -1 && right >= 1 && bottom >= 1);
