/*
 * Copyright © 2010 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 */

#include "brw_context.h"
#include "brw_defines.h"
#include "intel_batchbuffer.h"
#include "intel_fbo.h"

/**
 * According to the latest documentation, any PIPE_CONTROL with the
 * "Command Streamer Stall" bit set must also have another bit set,
 * with five different options:
 *
 *  - Render Target Cache Flush
 *  - Depth Cache Flush
 *  - Stall at Pixel Scoreboard
 *  - Post-Sync Operation
 *  - Depth Stall
 *  - DC Flush Enable
 *
 * I chose "Stall at Pixel Scoreboard" since we've used it effectively
 * in the past, but the choice is fairly arbitrary.
 */
static void
gen8_add_cs_stall_workaround_bits(uint32_t *flags)
{
   uint32_t wa_bits = PIPE_CONTROL_RENDER_TARGET_FLUSH |
                      PIPE_CONTROL_DEPTH_CACHE_FLUSH |
                      PIPE_CONTROL_WRITE_IMMEDIATE |
                      PIPE_CONTROL_WRITE_DEPTH_COUNT |
                      PIPE_CONTROL_WRITE_TIMESTAMP |
                      PIPE_CONTROL_STALL_AT_SCOREBOARD |
                      PIPE_CONTROL_DEPTH_STALL |
                      PIPE_CONTROL_DATA_CACHE_FLUSH;

   /* If we're doing a CS stall, and don't already have one of the
    * workaround bits set, add "Stall at Pixel Scoreboard."
    */
   if ((*flags & PIPE_CONTROL_CS_STALL) != 0 && (*flags & wa_bits) == 0)
      *flags |= PIPE_CONTROL_STALL_AT_SCOREBOARD;
}

/* Implement the WaCsStallAtEveryFourthPipecontrol workaround on IVB, BYT:
 *
 * "Every 4th PIPE_CONTROL command, not counting the PIPE_CONTROL with
 *  only read-cache-invalidate bit(s) set, must have a CS_STALL bit set."
 *
 * Note that the kernel does CS stalls between batches, so we only need
 * to count them within a batch.
 */
static uint32_t
gen7_cs_stall_every_four_pipe_controls(struct brw_context *brw, uint32_t flags)
{
   if (brw->gen == 7 && !brw->is_haswell) {
      if (flags & PIPE_CONTROL_CS_STALL) {
         /* If we're doing a CS stall, reset the counter and carry on. */
         brw->pipe_controls_since_last_cs_stall = 0;
         return 0;
      }

      /* If this is the fourth pipe control without a CS stall, do one now. */
      if (++brw->pipe_controls_since_last_cs_stall == 4) {
         brw->pipe_controls_since_last_cs_stall = 0;
         return PIPE_CONTROL_CS_STALL;
      }
   }
   return 0;
}

/**
 * Emit a PIPE_CONTROL with various flushing flags.
 *
 * The caller is responsible for deciding what flags are appropriate for the
 * given generation.
 */
void
brw_emit_pipe_control_flush(struct brw_context *brw, uint32_t flags)
{
   if (brw->gen >= 6 &&
       (flags & PIPE_CONTROL_CACHE_FLUSH_BITS) &&
       (flags & PIPE_CONTROL_CACHE_INVALIDATE_BITS)) {
      /* A pipe control command with flush and invalidate bits set
       * simultaneously is an inherently racy operation on Gen6+ if the
       * contents of the flushed caches were intended to become visible from
       * any of the invalidated caches.  Split it in two PIPE_CONTROLs, the
       * first one should stall the pipeline to make sure that the flushed R/W
       * caches are coherent with memory once the specified R/O caches are
       * invalidated.  On pre-Gen6 hardware the (implicit) R/O cache
       * invalidation seems to happen at the bottom of the pipeline together
       * with any write cache flush, so this shouldn't be a concern.
       */
      brw_emit_pipe_control_flush(brw, (flags & PIPE_CONTROL_CACHE_FLUSH_BITS) |
                                       PIPE_CONTROL_CS_STALL);
      flags &= ~(PIPE_CONTROL_CACHE_FLUSH_BITS | PIPE_CONTROL_CS_STALL);
   }

   if (brw->gen >= 8) {
      if (brw->gen == 8)
         gen8_add_cs_stall_workaround_bits(&flags);

      if (brw->gen == 9 &&
          (flags & PIPE_CONTROL_VF_CACHE_INVALIDATE)) {
         /* Hardware workaround: SKL
          *
          * Emit Pipe Control with all bits set to zero before emitting
          * a Pipe Control with VF Cache Invalidate set.
          */
         brw_emit_pipe_control_flush(brw, 0);
      }

      BEGIN_BATCH(6);
      OUT_BATCH(_3DSTATE_PIPE_CONTROL | (6 - 2));
      OUT_BATCH(flags);
      OUT_BATCH(0);
      OUT_BATCH(0);
      OUT_BATCH(0);
      OUT_BATCH(0);
      ADVANCE_BATCH();
   } else if (brw->gen >= 6) {
      if (brw->gen == 6 &&
          (flags & PIPE_CONTROL_RENDER_TARGET_FLUSH)) {
         /* Hardware workaround: SNB B-Spec says:
          *
          *   [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush
          *   Enable = 1, a PIPE_CONTROL with any non-zero post-sync-op is
          *   required.
          */
         brw_emit_post_sync_nonzero_flush(brw);
      }

      flags |= gen7_cs_stall_every_four_pipe_controls(brw, flags);

      BEGIN_BATCH(5);
      OUT_BATCH(_3DSTATE_PIPE_CONTROL | (5 - 2));
      OUT_BATCH(flags);
      OUT_BATCH(0);
      OUT_BATCH(0);
      OUT_BATCH(0);
      ADVANCE_BATCH();
   } else {
      BEGIN_BATCH(4);
      OUT_BATCH(_3DSTATE_PIPE_CONTROL | flags | (4 - 2));
      OUT_BATCH(0);
      OUT_BATCH(0);
      OUT_BATCH(0);
      ADVANCE_BATCH();
   }
}

/**
 * Emit a PIPE_CONTROL that writes to a buffer object.
 *
 * \p flags should contain one of the following items:
 *  - PIPE_CONTROL_WRITE_IMMEDIATE
 *  - PIPE_CONTROL_WRITE_TIMESTAMP
 *  - PIPE_CONTROL_WRITE_DEPTH_COUNT
 */
void
brw_emit_pipe_control_write(struct brw_context *brw, uint32_t flags,
                            drm_intel_bo *bo, uint32_t offset,
                            uint32_t imm_lower, uint32_t imm_upper)
{
   if (brw->gen >= 8) {
      if (brw->gen == 8)
         gen8_add_cs_stall_workaround_bits(&flags);

      BEGIN_BATCH(6);
      OUT_BATCH(_3DSTATE_PIPE_CONTROL | (6 - 2));
      OUT_BATCH(flags);
      OUT_RELOC64(bo, I915_GEM_DOMAIN_INSTRUCTION, I915_GEM_DOMAIN_INSTRUCTION,
                  offset);
      OUT_BATCH(imm_lower);
      OUT_BATCH(imm_upper);
      ADVANCE_BATCH();
   } else if (brw->gen >= 6) {
      flags |= gen7_cs_stall_every_four_pipe_controls(brw, flags);

      /* PPGTT/GGTT is selected by DW2 bit 2 on Sandybridge, but DW1 bit 24
       * on later platforms.  We always use PPGTT on Gen7+.
       */
      unsigned gen6_gtt = brw->gen == 6 ? PIPE_CONTROL_GLOBAL_GTT_WRITE : 0;

      BEGIN_BATCH(5);
      OUT_BATCH(_3DSTATE_PIPE_CONTROL | (5 - 2));
      OUT_BATCH(flags);
      OUT_RELOC(bo, I915_GEM_DOMAIN_INSTRUCTION, I915_GEM_DOMAIN_INSTRUCTION,
                gen6_gtt | offset);
      OUT_BATCH(imm_lower);
      OUT_BATCH(imm_upper);
      ADVANCE_BATCH();
   } else {
      BEGIN_BATCH(4);
      OUT_BATCH(_3DSTATE_PIPE_CONTROL | flags | (4 - 2));
      OUT_RELOC(bo, I915_GEM_DOMAIN_INSTRUCTION, I915_GEM_DOMAIN_INSTRUCTION,
                PIPE_CONTROL_GLOBAL_GTT_WRITE | offset);
      OUT_BATCH(imm_lower);
      OUT_BATCH(imm_upper);
      ADVANCE_BATCH();
   }
}

/**
 * Restriction [DevSNB, DevIVB]:
 *
 * Prior to changing Depth/Stencil Buffer state (i.e. any combination of
 * 3DSTATE_DEPTH_BUFFER, 3DSTATE_CLEAR_PARAMS, 3DSTATE_STENCIL_BUFFER,
 * 3DSTATE_HIER_DEPTH_BUFFER) SW must first issue a pipelined depth stall
 * (PIPE_CONTROL with Depth Stall bit set), followed by a pipelined depth
 * cache flush (PIPE_CONTROL with Depth Flush Bit set), followed by
 * another pipelined depth stall (PIPE_CONTROL with Depth Stall bit set),
 * unless SW can otherwise guarantee that the pipeline from WM onwards is
 * already flushed (e.g., via a preceding MI_FLUSH).
 */
void
brw_emit_depth_stall_flushes(struct brw_context *brw)
{
   assert(brw->gen >= 6);

   /* Starting on BDW, these pipe controls are unnecessary.
    *
    *   WM HW will internally manage the draining pipe and flushing of the caches
    *   when this command is issued. The PIPE_CONTROL restrictions are removed.
    */
   if (brw->gen >= 8)
      return;

   brw_emit_pipe_control_flush(brw, PIPE_CONTROL_DEPTH_STALL);
   brw_emit_pipe_control_flush(brw, PIPE_CONTROL_DEPTH_CACHE_FLUSH);
   brw_emit_pipe_control_flush(brw, PIPE_CONTROL_DEPTH_STALL);
}

/**
 * From the Ivybridge PRM, Volume 2 Part 1, Section 3.2 (VS Stage Input):
 * "A PIPE_CONTROL with Post-Sync Operation set to 1h and a depth
 *  stall needs to be sent just prior to any 3DSTATE_VS, 3DSTATE_URB_VS,
 *  3DSTATE_CONSTANT_VS, 3DSTATE_BINDING_TABLE_POINTER_VS,
 *  3DSTATE_SAMPLER_STATE_POINTER_VS command.  Only one PIPE_CONTROL needs
 *  to be sent before any combination of VS associated 3DSTATE."
 */
void
gen7_emit_vs_workaround_flush(struct brw_context *brw)
{
   assert(brw->gen == 7);
   brw_emit_pipe_control_write(brw,
                               PIPE_CONTROL_WRITE_IMMEDIATE
                               | PIPE_CONTROL_DEPTH_STALL,
                               brw->workaround_bo, 0,
                               0, 0);
}


/**
 * Emit a PIPE_CONTROL command for gen7 with the CS Stall bit set.
 */
void
gen7_emit_cs_stall_flush(struct brw_context *brw)
{
   brw_emit_pipe_control_write(brw,
                               PIPE_CONTROL_CS_STALL
                               | PIPE_CONTROL_WRITE_IMMEDIATE,
                               brw->workaround_bo, 0,
                               0, 0);
}


/**
 * Emits a PIPE_CONTROL with a non-zero post-sync operation, for
 * implementing two workarounds on gen6.  From section 1.4.7.1
 * "PIPE_CONTROL" of the Sandy Bridge PRM volume 2 part 1:
 *
 * [DevSNB-C+{W/A}] Before any depth stall flush (including those
 * produced by non-pipelined state commands), software needs to first
 * send a PIPE_CONTROL with no bits set except Post-Sync Operation !=
 * 0.
 *
 * [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush Enable
 * =1, a PIPE_CONTROL with any non-zero post-sync-op is required.
 *
 * And the workaround for these two requires this workaround first:
 *
 * [Dev-SNB{W/A}]: Pipe-control with CS-stall bit set must be sent
 * BEFORE the pipe-control with a post-sync op and no write-cache
 * flushes.
 *
 * And this last workaround is tricky because of the requirements on
 * that bit.  From section 1.4.7.2.3 "Stall" of the Sandy Bridge PRM
 * volume 2 part 1:
 *
 *     "1 of the following must also be set:
 *      - Render Target Cache Flush Enable ([12] of DW1)
 *      - Depth Cache Flush Enable ([0] of DW1)
 *      - Stall at Pixel Scoreboard ([1] of DW1)
 *      - Depth Stall ([13] of DW1)
 *      - Post-Sync Operation ([13] of DW1)
 *      - Notify Enable ([8] of DW1)"
 *
 * The cache flushes require the workaround flush that triggered this
 * one, so we can't use it.  Depth stall would trigger the same.
 * Post-sync nonzero is what triggered this second workaround, so we
 * can't use that one either.  Notify enable is IRQs, which aren't
 * really our business.  That leaves only stall at scoreboard.
 */
void
brw_emit_post_sync_nonzero_flush(struct brw_context *brw)
{
   brw_emit_pipe_control_flush(brw,
                               PIPE_CONTROL_CS_STALL |
                               PIPE_CONTROL_STALL_AT_SCOREBOARD);

   brw_emit_pipe_control_write(brw, PIPE_CONTROL_WRITE_IMMEDIATE,
                               brw->workaround_bo, 0, 0, 0);
}

/* Emit a pipelined flush to either flush render and texture cache for
 * reading from a FBO-drawn texture, or flush so that frontbuffer
 * render appears on the screen in DRI1.
 *
 * This is also used for the always_flush_cache driconf debug option.
 */
void
brw_emit_mi_flush(struct brw_context *brw)
{
   if (brw->batch.ring == BLT_RING && brw->gen >= 6) {
      BEGIN_BATCH_BLT(4);
      OUT_BATCH(MI_FLUSH_DW);
      OUT_BATCH(0);
      OUT_BATCH(0);
      OUT_BATCH(0);
      ADVANCE_BATCH();
   } else {
      int flags = PIPE_CONTROL_NO_WRITE | PIPE_CONTROL_RENDER_TARGET_FLUSH;
      if (brw->gen >= 6) {
         flags |= PIPE_CONTROL_INSTRUCTION_INVALIDATE |
                  PIPE_CONTROL_CONST_CACHE_INVALIDATE |
                  PIPE_CONTROL_DEPTH_CACHE_FLUSH |
                  PIPE_CONTROL_VF_CACHE_INVALIDATE |
                  PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
                  PIPE_CONTROL_CS_STALL;
      }
      brw_emit_pipe_control_flush(brw, flags);
   }
}

int
brw_init_pipe_control(struct brw_context *brw,
                      const struct gen_device_info *devinfo)
{
   if (devinfo->gen < 6)
      return 0;

   /* We can't just use brw_state_batch to get a chunk of space for
    * the gen6 workaround because it involves actually writing to
    * the buffer, and the kernel doesn't let us write to the batch.
    */
   brw->workaround_bo = drm_intel_bo_alloc(brw->bufmgr,
                                           "pipe_control workaround",
                                           4096, 4096);
   if (brw->workaround_bo == NULL)
      return -ENOMEM;

   brw->pipe_controls_since_last_cs_stall = 0;

   return 0;
}

void
brw_fini_pipe_control(struct brw_context *brw)
{
   drm_intel_bo_unreference(brw->workaround_bo);
}