// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "test.h"

#include <unistd.h>
#include <cstdint>
#include <cstdlib>
#include <ctime>
#include <iostream>
#include <memory>
#include <string>
#include <vector>
#ifdef __APPLE__
#include <TargetConditionals.h>
#endif

#include "../eight_bit_int_gemm/eight_bit_int_gemm.h"
#include "../internal/kernel_reference.h"
#include "test_data.h"

namespace gemmlowp {

void ReferenceEightBitIntGemm(bool transpose_a, bool transpose_b,
                              bool transpose_c, int m, int n, int k,
                              const uint8_t* a, int32_t a_offset, int lda,
                              const uint8_t* b, int32_t b_offset, int ldb,
                              uint8_t* c, int32_t c_offset, int32_t c_mult_int,
                              int32_t c_shift, int ldc) {
  assert((c_shift >= 0) && (c_shift <= 32));

  assert(a != nullptr);
  assert(b != nullptr);
  assert(c != nullptr);

  int a_i_stride;
  int a_l_stride;
  if (transpose_a) {
    a_i_stride = lda;
    a_l_stride = 1;
  } else {
    a_i_stride = 1;
    a_l_stride = lda;
  }
  int b_j_stride;
  int b_l_stride;
  if (transpose_b) {
    b_j_stride = 1;
    b_l_stride = ldb;
  } else {
    b_j_stride = ldb;
    b_l_stride = 1;
  }
  int c_i_stride;
  int c_j_stride;
  if (transpose_c) {
    c_i_stride = ldc;
    c_j_stride = 1;
  } else {
    c_i_stride = 1;
    c_j_stride = ldc;
  }
  int i, j, l;

  const std::int32_t kRoundingTerm = (c_shift < 1) ? 0 : (1 << (c_shift - 1));

  for (j = 0; j < n; j++) {
    for (i = 0; i < m; i++) {
      int32_t total = 0;
      for (l = 0; l < k; l++) {
        const int a_index = i * a_i_stride + l * a_l_stride;
        const uint8_t a_as_byte = a[a_index];
        const int32_t a_as_int = static_cast<int32_t>(a_as_byte) + a_offset;
        const int b_index = j * b_j_stride + l * b_l_stride;
        const uint8_t b_as_byte = b[b_index];
        const int32_t b_as_int = static_cast<int32_t>(b_as_byte) + b_offset;
        const int32_t mult_as_int = a_as_int * b_as_int;
        total += mult_as_int;
      }
      int32_t output =
          (((total + c_offset) * c_mult_int) + kRoundingTerm) >> c_shift;
      if (output > 255) {
        output = 255;
      }
      if (output < 0) {
        output = 0;
      }
      const int c_index = i * c_i_stride + j * c_j_stride;
      c[c_index] = static_cast<uint8_t>(output);
    }
  }
}

// *GemmWrapper's allow to wrap various Gemm functions in a uniform
// interface, so we can use the same testing code to test all of them

template <typename Kernel, typename Scalar, typename tBitDepthParams>
struct SingleThreadGemmWrapper {
  typedef tBitDepthParams BitDepthParams;

  static const char* Name() {
    static char buf[256];
    snprintf(buf, sizeof(buf), "SingleThreadGemm, Kernel: %s", Kernel().Name());
    return buf;
  }

  typedef SingleThreadGemmContext Context;

  template <MapOrder LhsOrder, MapOrder RhsOrder, MapOrder ResultOrder>
  static void Gemm(Context* context,
                   const MatrixMap<const Scalar, LhsOrder>& lhs,
                   const MatrixMap<const Scalar, RhsOrder>& rhs,
                   MatrixMap<Scalar, ResultOrder>* result, int lhs_offset,
                   int rhs_offset, int result_offset, int result_mult_int,
                   int result_shift) {
    const OffsetColDup lhs_offset_vector(lhs_offset, lhs.rows());
    const OffsetRowDup rhs_offset_vector(rhs_offset, rhs.cols());
    SingleThreadGemm<typename Kernel::Format, Scalar, Scalar, BitDepthParams,
                     LhsOrder, RhsOrder, ResultOrder,
                     OffsetColDup, OffsetRowDup>(
        context, Kernel(), lhs, rhs, result, lhs_offset_vector,
        rhs_offset_vector,
        MakeStandardOutputPipeline(result_offset, result_mult_int,
                                   result_shift));
  }
};

template <typename Kernel, typename Scalar, typename tBitDepthParams>
struct MultiThreadGemmWrapper {
  typedef tBitDepthParams BitDepthParams;

  static const char* Name() {
    static char buf[256];
    snprintf(buf, sizeof(buf), "MultiThreadGemm, Kernel: %s", Kernel().Name());
    return buf;
  }

  typedef MultiThreadGemmContext Context;

  template <MapOrder LhsOrder, MapOrder RhsOrder, MapOrder ResultOrder>
  static void Gemm(Context* context,
                   const MatrixMap<const Scalar, LhsOrder>& lhs,
                   const MatrixMap<const Scalar, RhsOrder>& rhs,
                   MatrixMap<Scalar, ResultOrder>* result, int lhs_offset,
                   int rhs_offset, int result_offset, int result_mult_int,
                   int result_shift) {
    const OffsetColDup lhs_offset_vector(lhs_offset, lhs.rows());
    const OffsetRowDup rhs_offset_vector(rhs_offset, rhs.cols());
    MultiThreadGemm<typename Kernel::Format, Scalar, Scalar, BitDepthParams,
                    LhsOrder, RhsOrder, ResultOrder,
                    OffsetColDup, OffsetRowDup>(
        context, Kernel(), lhs, rhs, result, lhs_offset_vector,
        rhs_offset_vector,
        MakeStandardOutputPipeline(result_offset, result_mult_int,
                                   result_shift));
  }
};

template <typename Scalar, typename tBitDepthParams>
struct PublicGemmWrapper {
  typedef tBitDepthParams BitDepthParams;

  static const char* Name() { return "public Gemm"; }

  typedef GemmContext Context;

  template <MapOrder LhsOrder, MapOrder RhsOrder, MapOrder ResultOrder>
  static void Gemm(Context* context,
                   const MatrixMap<const Scalar, LhsOrder>& lhs,
                   const MatrixMap<const Scalar, RhsOrder>& rhs,
                   MatrixMap<Scalar, ResultOrder>* result, int lhs_offset,
                   int rhs_offset, int result_offset, int result_mult_int,
                   int result_shift) {
    gemmlowp::Gemm<uint8_t, BitDepthParams, LhsOrder, RhsOrder, ResultOrder>(
        context, lhs, rhs, result, lhs_offset, rhs_offset, result_offset,
        result_mult_int, result_shift);
  }
};

template <eight_bit_int_gemm::BitDepthSetting BitDepth>
struct BitDepthParamsForSettings {};

template <>
struct BitDepthParamsForSettings<eight_bit_int_gemm::BitDepthSetting::A8B8>
    : DefaultL8R8BitDepthParams {};

template <>
struct BitDepthParamsForSettings<eight_bit_int_gemm::BitDepthSetting::A5B7>
    : DefaultL7R5BitDepthParams {};

template <typename Scalar, eight_bit_int_gemm::BitDepthSetting BitDepth>
struct EightBitIntGemmWrapper {
  typedef BitDepthParamsForSettings<BitDepth> BitDepthParams;

  static const char* Name() { return "EightBitIntGemm"; }

  typedef void Context;

  template <MapOrder LhsOrder, MapOrder RhsOrder, MapOrder ResultOrder>
  static void Gemm(Context*, const MatrixMap<const Scalar, LhsOrder>& lhs,
                   const MatrixMap<const Scalar, RhsOrder>& rhs,
                   MatrixMap<Scalar, ResultOrder>* result, int lhs_offset,
                   int rhs_offset, int result_offset, int result_mult_int,
                   int result_shift) {
    const bool transpose_c = ResultOrder == MapOrder::RowMajor;
    const bool transpose_a = LhsOrder == MapOrder::RowMajor;
    const bool transpose_b = RhsOrder == MapOrder::RowMajor;
    eight_bit_int_gemm::EightBitIntGemm(
        transpose_a, transpose_b, transpose_c, lhs.rows(), rhs.cols(),
        lhs.cols(), lhs.data(), lhs_offset, lhs.stride(), rhs.data(),
        rhs_offset, rhs.stride(), result->data(), result_offset,
        result_mult_int, result_shift, result->stride(), BitDepth);
  }
};

template <typename Scalar>
struct ReferenceEightBitIntGemmWrapper {
  typedef DefaultL8R8BitDepthParams BitDepthParams;

  static const char* Name() { return "ReferenceEightBitIntGemm"; }

  template <MapOrder LhsOrder, MapOrder RhsOrder, MapOrder ResultOrder>
  static void Gemm(bool transpose_a, bool transpose_b, bool transpose_c,
                   const MatrixMap<const Scalar, LhsOrder>& lhs,
                   const MatrixMap<const Scalar, RhsOrder>& rhs,
                   MatrixMap<Scalar, ResultOrder>* result, int lhs_offset,
                   int rhs_offset, int result_offset, int result_mult_int,
                   int result_shift) {
    ReferenceEightBitIntGemm(transpose_a, transpose_b, transpose_c, lhs.rows(),
                             rhs.cols(), lhs.cols(), lhs.data(), lhs_offset,
                             lhs.stride(), rhs.data(), rhs_offset, rhs.stride(),
                             result->data(), result_offset, result_mult_int,
                             result_shift, result->stride());
  }
};

const char* OrderName(MapOrder order) {
  return order == MapOrder::ColMajor ? "ColMajor" : "RowMajor";
}

struct ResultStats {
  ResultStats()
      : count(0),
        med_val(0),
        mean_signed_diff(0),
        med_signed_diff(0),
        med_unsigned_diff(0),
        max_unsigned_diff(0) {}

  int count;
  int med_val;
  float mean_signed_diff;
  int med_signed_diff;
  int med_unsigned_diff;
  int max_unsigned_diff;

  std::vector<int> count_diff_by_pot_slice;
};

void GetResultStats(const uint8_t* actual, const uint8_t* expected,
                    size_t count, ResultStats* stats) {
  std::vector<uint8_t> results;
  std::vector<int16_t> signed_diffs;
  std::vector<uint8_t> unsigned_diffs;
  int64_t signed_diffs_sum = 0;
  for (size_t i = 0; i < count; i++) {
    results.push_back(actual[i]);
    int16_t signed_diff = actual[i] - expected[i];
    signed_diffs.push_back(signed_diff);
    unsigned_diffs.push_back(std::abs(signed_diff));
    signed_diffs_sum += signed_diff;
  }

  std::sort(results.begin(), results.end());
  std::sort(signed_diffs.begin(), signed_diffs.end());
  std::sort(unsigned_diffs.begin(), unsigned_diffs.end());

  const size_t middle = count / 2;

  stats->count = count;
  stats->med_val = results[middle];
  stats->mean_signed_diff = float(signed_diffs_sum) / count;
  stats->med_signed_diff = signed_diffs[middle];
  stats->med_unsigned_diff = unsigned_diffs[middle];
  stats->max_unsigned_diff = unsigned_diffs.back();

  // Size 9 for 9 different POT values: 2^0, ..., 2^8
  stats->count_diff_by_pot_slice.resize(9);
  auto cur = unsigned_diffs.begin();
  size_t checksum = 0;
  for (int exponent = 0; exponent < 9; exponent++) {
    int pot = 1 << exponent;
    auto next = std::lower_bound(cur, unsigned_diffs.end(), pot);
    checksum += stats->count_diff_by_pot_slice[exponent] = next - cur;
    cur = next;
  }
  assert(checksum == count);
}

struct ResultStatsBounds {
  ResultStatsBounds()
      : mean_signed_diff(0),
        med_signed_diff(0),
        med_unsigned_diff(0),
        max_unsigned_diff(0) {}

  float mean_signed_diff;
  int med_signed_diff;
  int med_unsigned_diff;
  int max_unsigned_diff;
};

bool CheckResultStatsBounds(const ResultStats& stats,
                            const ResultStatsBounds& bounds) {
  return stats.max_unsigned_diff <= bounds.max_unsigned_diff &&
         stats.med_unsigned_diff <= bounds.med_unsigned_diff &&
         std::abs(stats.med_signed_diff) <= bounds.med_signed_diff &&
         std::abs(stats.mean_signed_diff) <= bounds.mean_signed_diff;
}

void ReportResultStats(const ResultStats& stats,
                       const ResultStatsBounds& bounds) {
  printf("    number of matrix entries: %d\n", stats.count);
  printf("    median value: %d\n", stats.med_val);
  printf("    median unsigned diff: %d (tolerating %d)\n",
         stats.med_unsigned_diff, bounds.med_unsigned_diff);
  printf("    max unsigned diff: %d (tolerating %d)\n", stats.max_unsigned_diff,
         bounds.max_unsigned_diff);
  printf("    median signed diff: %d (tolerating %d)\n", stats.med_signed_diff,
         bounds.med_signed_diff);
  printf("    mean signed diff: %.3g (tolerating %.3g)\n",
         stats.mean_signed_diff, bounds.mean_signed_diff);

  printf("No error: %.2f %% of entries\n",
         100.f * stats.count_diff_by_pot_slice[0] / stats.count);
  for (int exponent = 1; exponent < 9; exponent++) {
    printf("Error in %d..%d range: %.2f %% of entries\n", 1 << (exponent - 1),
           (1 << exponent) - 1,
           100.f * stats.count_diff_by_pot_slice[exponent] / stats.count);
  }
}

// Our approach to choosing result_shift values for testing, is bisection.
// This function takes an interval, [result_shift_min .. result_shift_max].
// If too much saturation occurred in either direction, it bisects accordingly,
// recursing until the interval contains only one value.
// The primary reason why we prefer this over computing optimal shift values,
// is that we actually want to exercise some saturation, as there is nontrivial
// code handling that in gemmlowp.
// Secondarily, this is faster than computing optimal shifts, since in 90% of
// cases the first-tried shift value 16 turns out to be good enough.
template <typename GemmWrapper, typename LhsType, typename RhsType,
          typename ResultType>
void test_gemm_impl(typename GemmWrapper::Context* context, const LhsType& lhs,
                    const RhsType& rhs, ResultType* result, int lhs_offset,
                    int rhs_offset, int result_offset, int result_mult_int,
                    int result_shift_min, int result_shift_max) {
  const int rows = lhs.rows();
  const int cols = rhs.cols();
  Check(lhs.cols() == rhs.rows());
  const int depth = lhs.cols();

  const int result_shift = (result_shift_min + result_shift_max) / 2;

  GemmWrapper::Gemm(context, lhs.const_map(), rhs.const_map(), &result->map(),
                    lhs_offset, rhs_offset, result_offset, result_mult_int,
                    result_shift);

  typedef typename ResultType::Scalar Scalar;
  static const MapOrder kLhsOrder = LhsType::kOrder;
  static const MapOrder kRhsOrder = RhsType::kOrder;
  static const MapOrder kResultOrder = ResultType::kOrder;
  ResultType ref_result(rows, cols);
  const bool transpose_c = kResultOrder == MapOrder::RowMajor;
  const bool transpose_a = kLhsOrder == MapOrder::RowMajor;
  const bool transpose_b = kRhsOrder == MapOrder::RowMajor;
  ReferenceEightBitIntGemmWrapper<Scalar>::Gemm(
      transpose_a, transpose_b, transpose_c, lhs.const_map(), rhs.const_map(),
      &ref_result.map(), lhs_offset, rhs_offset, result_offset, result_mult_int,
      result_shift);

  typedef typename GemmWrapper::BitDepthParams BitDepthParams;

  ResultStats stats;
  GetResultStats(result->data(), ref_result.data(), rows * cols, &stats);

  // Adjust shifts until we get meaningful results
  int new_result_shift_min = result_shift_min;
  int new_result_shift_max = result_shift_max;
  bool retry = false;

  if (stats.med_val < 32) {
    new_result_shift_max = (result_shift_min + result_shift_max) / 2;
    retry = true;
  }

  if (stats.med_val > 224) {
    new_result_shift_min = (result_shift_min + result_shift_max) / 2;
    retry = true;
  }

  if (retry) {
    if (result_shift_min != result_shift_max) {
      test_gemm_impl<GemmWrapper>(context, lhs, rhs, result, lhs_offset,
                                  rhs_offset, result_offset, result_mult_int,
                                  new_result_shift_min, new_result_shift_max);
    }
    return;
  }

  ResultStatsBounds bounds;

  if (BitDepthParams::LhsBitDepth::kBits < 8 ||
      BitDepthParams::RhsBitDepth::kBits < 8) {
    // We have very lax requirements on unsigned diff.
    // We have tighter requirements on signed diff (bias), but only
    // if the matrix is large enough for things to average out.
    // For very small sizes, we... basically don't test anything.
    // The problem is that this test uses unrealistic combinations of
    // result_mult_int
    // and result_shift, resulting in potentially wild requantization artifacts
    // on small GEMMs.
    int adjust_for_small_sizes = 1000 / (rows * cols);
    bounds.max_unsigned_diff =
        std::max(stats.med_val / 2, adjust_for_small_sizes);
    bounds.med_unsigned_diff =
        std::max(stats.med_val / 8, adjust_for_small_sizes);
    bounds.med_signed_diff = std::max(2, adjust_for_small_sizes);
    bounds.mean_signed_diff = std::max(2, adjust_for_small_sizes);
  }

  // Check results
  const bool good = CheckResultStatsBounds(stats, bounds);

  printf(
      "%s: %dx%dx%d %s x %s -> %s, %s, offsets %d/%d/%d, mult %d, shift %d\n",
      good ? "PASS" : "FAIL", rows, depth, cols, OrderName(kLhsOrder),
      OrderName(kRhsOrder), OrderName(kResultOrder), GemmWrapper::Name(),
      lhs_offset, rhs_offset, result_offset, result_mult_int, result_shift);

  if (!good) {
    ReportResultStats(stats, bounds);

    int bad_coeffs_printed = 0;
    for (int c = 0; c < result->cols() && bad_coeffs_printed < 20; c++) {
      for (int r = 0; r < result->rows() && bad_coeffs_printed < 20; r++) {
        if (ref_result(r, c) != (*result)(r, c)) {
          printf("bad coeff: at (%d, %d), expected %d, got %d\n", r, c,
                 ref_result(r, c), (*result)(r, c));
          bad_coeffs_printed++;
        }
      }
    }
  }

  Check(good);
}

template <typename GemmWrapper, typename LhsType, typename RhsType,
          typename ResultType>
void test_gemm(typename GemmWrapper::Context* context, const LhsType& lhs,
               const RhsType& rhs, ResultType* result, int lhs_offset,
               int rhs_offset, int result_offset, int result_mult_int) {
  test_gemm_impl<GemmWrapper>(context, lhs, rhs, result, lhs_offset, rhs_offset,
                              result_offset, result_mult_int, 0, 32);
}

enum class WhatParamsToTest {
  All,
  OnlyGenericCase,
};

template <typename GemmWrapper, MapOrder LhsOrder, MapOrder RhsOrder,
          MapOrder ResultOrder>
void test_gemm(typename GemmWrapper::Context* context, int rows, int depth,
               int cols, WhatParamsToTest params_to_test) {
  typedef std::uint8_t Scalar;
  typedef Matrix<Scalar, LhsOrder> LhsType;
  LhsType lhs(rows, depth);
  MakeRandom(&lhs, 8);
  typedef Matrix<Scalar, RhsOrder> RhsType;
  RhsType rhs(depth, cols);
  MakeRandom(&rhs, 8);
  typedef Matrix<Scalar, ResultOrder> ResultType;
  ResultType result(rows, cols);
  MakeZero(&result);

  if (params_to_test == WhatParamsToTest::All) {
    test_gemm<GemmWrapper>(context, lhs, rhs, &result, 0, 0, 0, 1);
    test_gemm<GemmWrapper>(context, lhs, rhs, &result, 10, 0, 0, 1);
    test_gemm<GemmWrapper>(context, lhs, rhs, &result, 0, 10, 0, 1);
    test_gemm<GemmWrapper>(context, lhs, rhs, &result, 0, 0, 10, 1);
    test_gemm<GemmWrapper>(context, lhs, rhs, &result, 0, 0, 0, 10);
    test_gemm<GemmWrapper>(context, lhs, rhs, &result, 10, 10, 10, 10);
    test_gemm<GemmWrapper>(context, lhs, rhs, &result, 256, 1, 17, 4);
  }
  test_gemm<GemmWrapper>(context, lhs, rhs, &result, -75, -91, 74980, 123);
}

enum class WhatOrdersToTest { All, OnlyRCC };

template <typename GemmWrapper>
void test_gemm(typename GemmWrapper::Context* context, int rows, int depth,
               int cols, WhatParamsToTest params_to_test,
               WhatOrdersToTest orders_to_test) {
#define GEMMLOWP_ONE_TEST(LhsOrder, RhsOrder, ResultOrder)         \
  do {                                                             \
    test_gemm<GemmWrapper, MapOrder::LhsOrder, MapOrder::RhsOrder, \
              MapOrder::ResultOrder>(context, rows, depth, cols,   \
                                     params_to_test);              \
  } while (false)

  if (orders_to_test == WhatOrdersToTest::All) {
    GEMMLOWP_ONE_TEST(ColMajor, ColMajor, ColMajor);
    GEMMLOWP_ONE_TEST(RowMajor, ColMajor, ColMajor);
    GEMMLOWP_ONE_TEST(ColMajor, RowMajor, ColMajor);
    GEMMLOWP_ONE_TEST(RowMajor, RowMajor, ColMajor);

    GEMMLOWP_ONE_TEST(ColMajor, ColMajor, RowMajor);
    GEMMLOWP_ONE_TEST(RowMajor, ColMajor, RowMajor);
    GEMMLOWP_ONE_TEST(ColMajor, RowMajor, RowMajor);
    GEMMLOWP_ONE_TEST(RowMajor, RowMajor, RowMajor);
  } else {
    GEMMLOWP_ONE_TEST(RowMajor, ColMajor, ColMajor);
  }

#undef GEMMLOWP_ONE_TEST
}

template <typename Kernel>
void test_gemm_kernel(MultiThreadGemmContext* context) {
  typedef MultiThreadGemmWrapper<Kernel, std::uint8_t,
                                 DefaultL8R8BitDepthParams>
      GemmWrapper;
  test_gemm<GemmWrapper>(context, 1, 1, 1, WhatParamsToTest::OnlyGenericCase,
                         WhatOrdersToTest::OnlyRCC);
  test_gemm<GemmWrapper>(context, 2, 2, 2, WhatParamsToTest::OnlyGenericCase,
                         WhatOrdersToTest::OnlyRCC);
  test_gemm<GemmWrapper>(context, 3, 3, 3, WhatParamsToTest::OnlyGenericCase,
                         WhatOrdersToTest::OnlyRCC);
  test_gemm<GemmWrapper>(context, 4, 4, 4, WhatParamsToTest::OnlyGenericCase,
                         WhatOrdersToTest::OnlyRCC);
  test_gemm<GemmWrapper>(context, 5, 5, 5, WhatParamsToTest::OnlyGenericCase,
                         WhatOrdersToTest::OnlyRCC);
  test_gemm<GemmWrapper>(context, 9, 11, 13, WhatParamsToTest::OnlyGenericCase,
                         WhatOrdersToTest::OnlyRCC);
  test_gemm<GemmWrapper>(context, 50, 50, 50, WhatParamsToTest::All,
                         WhatOrdersToTest::OnlyRCC);
  test_gemm<GemmWrapper>(context, 200, 200, 200,
                         WhatParamsToTest::OnlyGenericCase,
                         WhatOrdersToTest::All);
  test_gemm<GemmWrapper>(context, 50, 5000, 50,
                         WhatParamsToTest::OnlyGenericCase,
                         WhatOrdersToTest::OnlyRCC);
}

template <typename GemmWrapper>
void test_gemm(typename GemmWrapper::Context* context) {
  test_gemm<GemmWrapper>(context, 1, 1, 1, WhatParamsToTest::All,
                         WhatOrdersToTest::OnlyRCC);
  test_gemm<GemmWrapper>(context, 2, 1, 1, WhatParamsToTest::All,
                         WhatOrdersToTest::OnlyRCC);
  test_gemm<GemmWrapper>(context, 1, 2, 1, WhatParamsToTest::All,
                         WhatOrdersToTest::OnlyRCC);
  test_gemm<GemmWrapper>(context, 1, 1, 2, WhatParamsToTest::All,
                         WhatOrdersToTest::OnlyRCC);
  test_gemm<GemmWrapper>(context, 2, 2, 2, WhatParamsToTest::All,
                         WhatOrdersToTest::OnlyRCC);
  test_gemm<GemmWrapper>(context, 3, 3, 3, WhatParamsToTest::All,
                         WhatOrdersToTest::OnlyRCC);
  test_gemm<GemmWrapper>(context, 4, 4, 4, WhatParamsToTest::All,
                         WhatOrdersToTest::OnlyRCC);
  test_gemm<GemmWrapper>(context, 5, 5, 5, WhatParamsToTest::All,
                         WhatOrdersToTest::OnlyRCC);
  test_gemm<GemmWrapper>(context, 6, 6, 6, WhatParamsToTest::All,
                         WhatOrdersToTest::OnlyRCC);
  test_gemm<GemmWrapper>(context, 3, 5, 7, WhatParamsToTest::All,
                         WhatOrdersToTest::OnlyRCC);
  test_gemm<GemmWrapper>(context, 7, 3, 5, WhatParamsToTest::All,
                         WhatOrdersToTest::OnlyRCC);
  test_gemm<GemmWrapper>(context, 5, 7, 3, WhatParamsToTest::All,
                         WhatOrdersToTest::OnlyRCC);
  test_gemm<GemmWrapper>(context, 8, 8, 8, WhatParamsToTest::All,
                         WhatOrdersToTest::OnlyRCC);
  test_gemm<GemmWrapper>(context, 16, 16, 16, WhatParamsToTest::All,
                         WhatOrdersToTest::OnlyRCC);
  test_gemm<GemmWrapper>(context, 32, 32, 32, WhatParamsToTest::All,
                         WhatOrdersToTest::OnlyRCC);
  test_gemm<GemmWrapper>(context, 64, 64, 64, WhatParamsToTest::All,
                         WhatOrdersToTest::OnlyRCC);
  test_gemm<GemmWrapper>(context, 128, 128, 128, WhatParamsToTest::All,
                         WhatOrdersToTest::OnlyRCC);

  test_gemm<GemmWrapper>(context, 16, 17, 16, WhatParamsToTest::All,
                         WhatOrdersToTest::OnlyRCC);
  test_gemm<GemmWrapper>(context, 37, 55, 73, WhatParamsToTest::All,
                         WhatOrdersToTest::OnlyRCC);
  test_gemm<GemmWrapper>(context, 57, 87, 117, WhatParamsToTest::All,
                         WhatOrdersToTest::OnlyRCC);
  test_gemm<GemmWrapper>(context, 93, 83, 73, WhatParamsToTest::All,
                         WhatOrdersToTest::OnlyRCC);
  test_gemm<GemmWrapper>(context, 109, 89, 99, WhatParamsToTest::All,
                         WhatOrdersToTest::OnlyRCC);
  test_gemm<GemmWrapper>(context, 78, 101, 82, WhatParamsToTest::All,
                         WhatOrdersToTest::OnlyRCC);

  test_gemm<GemmWrapper>(context, 512, 512, 512,
                         WhatParamsToTest::OnlyGenericCase,
                         WhatOrdersToTest::OnlyRCC);
  test_gemm<GemmWrapper>(context, 1024, 1024, 1024,
                         WhatParamsToTest::OnlyGenericCase,
                         WhatOrdersToTest::OnlyRCC);
  test_gemm<GemmWrapper>(context, 567, 2345, 123,
                         WhatParamsToTest::OnlyGenericCase,
                         WhatOrdersToTest::OnlyRCC);
  test_gemm<GemmWrapper>(context, 100, 5000, 100,
                         WhatParamsToTest::OnlyGenericCase,
                         WhatOrdersToTest::OnlyRCC);
  test_gemm<GemmWrapper>(context, 1, 1, 1000, WhatParamsToTest::OnlyGenericCase,
                         WhatOrdersToTest::OnlyRCC);
  test_gemm<GemmWrapper>(context, 1000, 1, 1, WhatParamsToTest::OnlyGenericCase,
                         WhatOrdersToTest::OnlyRCC);
  test_gemm<GemmWrapper>(context, 1, 1000, 1, WhatParamsToTest::OnlyGenericCase,
                         WhatOrdersToTest::OnlyRCC);
  test_gemm<GemmWrapper>(context, 1, 1000, 1000,
                         WhatParamsToTest::OnlyGenericCase,
                         WhatOrdersToTest::OnlyRCC);
  test_gemm<GemmWrapper>(context, 1000, 1, 1000,
                         WhatParamsToTest::OnlyGenericCase,
                         WhatOrdersToTest::OnlyRCC);
  test_gemm<GemmWrapper>(context, 1000, 1000, 1,
                         WhatParamsToTest::OnlyGenericCase,
                         WhatOrdersToTest::OnlyRCC);
  test_gemm<GemmWrapper>(context, 777, 3456, 1,
                         WhatParamsToTest::OnlyGenericCase,
                         WhatOrdersToTest::OnlyRCC);
  test_gemm<GemmWrapper>(context, 4567, 555, 1,
                         WhatParamsToTest::OnlyGenericCase,
                         WhatOrdersToTest::OnlyRCC);

  // Test all storage orders
  test_gemm<GemmWrapper>(context, 70, 90, 110, WhatParamsToTest::All,
                         WhatOrdersToTest::All);
  test_gemm<GemmWrapper>(context, 300, 400, 500,
                         WhatParamsToTest::OnlyGenericCase,
                         WhatOrdersToTest::All);
}

template <typename GemmWrapper>
void test_gemv(typename GemmWrapper::Context* context) {
  test_gemm<GemmWrapper>(context, 2, 2, 1, WhatParamsToTest::All,
                         WhatOrdersToTest::OnlyRCC);
  test_gemm<GemmWrapper>(context, 3, 3, 1, WhatParamsToTest::All,
                         WhatOrdersToTest::OnlyRCC);
  test_gemm<GemmWrapper>(context, 4, 4, 1, WhatParamsToTest::All,
                         WhatOrdersToTest::OnlyRCC);
  test_gemm<GemmWrapper>(context, 5, 5, 1, WhatParamsToTest::All,
                         WhatOrdersToTest::OnlyRCC);
  test_gemm<GemmWrapper>(context, 6, 6, 1, WhatParamsToTest::All,
                         WhatOrdersToTest::OnlyRCC);
  test_gemm<GemmWrapper>(context, 3, 5, 1, WhatParamsToTest::All,
                         WhatOrdersToTest::OnlyRCC);
  test_gemm<GemmWrapper>(context, 7, 3, 1, WhatParamsToTest::All,
                         WhatOrdersToTest::OnlyRCC);
  test_gemm<GemmWrapper>(context, 5, 7, 1, WhatParamsToTest::All,
                         WhatOrdersToTest::OnlyRCC);
  test_gemm<GemmWrapper>(context, 8, 8, 1, WhatParamsToTest::All,
                         WhatOrdersToTest::OnlyRCC);
  test_gemm<GemmWrapper>(context, 32, 32, 1, WhatParamsToTest::All,
                         WhatOrdersToTest::OnlyRCC);
  test_gemm<GemmWrapper>(context, 128, 128, 1, WhatParamsToTest::All,
                         WhatOrdersToTest::OnlyRCC);
  test_gemm<GemmWrapper>(context, 321, 123, 1, WhatParamsToTest::All,
                         WhatOrdersToTest::OnlyRCC);

  // Test all storage orders
  test_gemm<GemmWrapper>(context, 70, 90, 1, WhatParamsToTest::All,
                         WhatOrdersToTest::All);
  test_gemm<GemmWrapper>(context, 300, 400, 1,
                         WhatParamsToTest::OnlyGenericCase,
                         WhatOrdersToTest::All);
}

const char* GetBitDepthName(eight_bit_int_gemm::BitDepthSetting b) {
  switch (b) {
    case eight_bit_int_gemm::BitDepthSetting::A8B8:
      return "Lhs: 8 bit, Rhs: 8 bit";
    case eight_bit_int_gemm::BitDepthSetting::A5B7:
      return "Lhs: 7 bit, Rhs: 5 bit";
    default:
      abort();
      return nullptr;
  }
}

// Runs a small set of hand-picked data for per-channel quantized data.
// This test case comes from a set of 2 2x2 convolution filters run over a 3x3
// image.
void TestWithSmallDataPerChannelQuantization() {
  const int m = 2;
  const int n = 9;
  const int k = 12;

  // 12 x 2, columnwise.
  const uint8_t a_data[] = {
     0,  0,  0,  0,  0,  0, 0, 0, 0, 255, 255, 255,
    64, 64, 64, 64, 64, 64, 0, 0, 0, 255, 255, 255
  };
  const int lda = k;
  int a_offset[] = {0, -64};
  MatrixMap<const std::uint8_t, MapOrder::RowMajor> lhs(a_data, m, k, lda);
  const OffsetColMap lhs_offset(a_offset, m);

  // 12 x 9, columnwise.
  const uint8_t b_data[] = {
      0,   0,   0,   0,   0,   0,   0,   0,   0, 255, 255, 255,
      0,   0,   0,   0,   0,   0, 255, 255, 255,   0,   0,   0,
      0,   0,   0, 127, 127, 127,   0,   0,   0, 127, 127, 127,
      0,   0,   0, 255, 255, 255,   0,   0,   0,   0,   0,   0,
    255, 255, 255,   0,   0,   0,   0,   0,   0,   0,   0,   0,
      0,   0,   0, 127, 127, 127,   0,   0,   0, 127, 127, 127,
      0,   0,   0,   0,   0,   0, 127, 127, 127, 127, 127, 127,
      0,   0,   0,   0,   0,   0, 127, 127, 127, 127, 127, 127,
      0,   0,   0, 127, 127, 127, 127, 127, 127, 127, 127, 127
  };
  const int ldb = k;
  int b_offset = -127;
  MatrixMap<const std::uint8_t, MapOrder::ColMajor> rhs(b_data, k, n, ldb);
  const OffsetRowDup rhs_offset(b_offset, rhs.cols());

  // 2 x 9, columnwise.
  const uint8_t expected_c_data[] = {
    255, 255,
      0,   0,
    127, 159,
      0,  64,
      0,  64,
    127, 159,
    127, 127,
    127, 127,
    127, 127
  };
  const int ldc = m;
  int c_offset[] = {97155, 97346};
  int c_mult_int[] = {2741, 2741};
  const int c_shift = 21;

  const int c_count = m * n;
  std::unique_ptr<uint8_t[]> output_data(new uint8_t[c_count]);
  MatrixMap<std::uint8_t, MapOrder::ColMajor> result(output_data.get(), m, n,
                                                     ldc);
  const OffsetColMap result_offset(c_offset, m);
  const OffsetColMap result_mult_int(c_mult_int, m);
  const int result_shift = c_shift;

  GemmContext gemm_context;
  auto output_pipeline = MakeStandardOutputPipeline<VectorShape::Col>(
      result_offset, result_mult_int, result_shift);
  GemmWithOutputPipelinePC<uint8_t, uint8_t, DefaultL8R8BitDepthParams>(
      &gemm_context, lhs, rhs, &result, lhs_offset, rhs_offset,
      output_pipeline);

  ResultStats stats;
  GetResultStats(output_data.get(), expected_c_data, c_count, &stats);

  ResultStatsBounds bounds;
  const bool good = CheckResultStatsBounds(stats, bounds);
  printf("TestWithSmallDataPerChannelQuantization: %s\n",
         good ? "PASS" : "FAIL");
  ReportResultStats(stats, bounds);
  Check(good);
}

// Runs a larger set of hand-picked data for per-channel quantized data.
// This test case comes from a set of 22 3x3 convolution filters run over a 5x5
// image.  Right now, I have 7 different filters and 15 copies of the first
// filter to make sure NEON code path that processes 16 rows at a time is
// covered.
void TestWithLargeDataPerChannelQuantization() {
  const int m = 22;
  const int n = 25;
  const int k = 27;

  // 27 x 22, column-wise.
  const uint8_t a_data[] = {
     0,  0,  0,   0,   0,   0,  0,  0,  0,   0,   0,   0, 255, 255, 255,
         0,   0,   0,  0,  0,  0,   0,   0,   0,  0,  0,  0,
     0,  0,  0,   0,   0,   0,  0,  0,  0, 127, 127, 127, 255, 255, 255,
       127, 127, 127,  0,  0,  0,   0,   0,   0,  0,  0,  0,
     0,  0,  0, 127, 127, 127,  0,  0,  0,   0,   0,   0, 255, 255, 255,
         0,   0,   0,  0,  0,  0, 127, 127, 127,  0,  0,  0,
    51, 51, 51,  51,  51,  51, 51, 51, 51,   0,   0,   0, 255, 255, 255,
         0,   0,   0, 51, 51, 51,  51,  51,  51, 51, 51, 51,
    51, 51, 51,   0,   0,   0, 51, 51, 51,  51,  51,  51, 255, 255, 255,
        51,  51,  51, 51, 51, 51,   0,   0,   0, 51, 51, 51,
     0,  0,  0,  64,  64,  64,  0,  0,  0,  64,  64,  64, 255, 255, 255,
        64,  64,  64,  0,  0,  0,  64,  64,  64,  0,  0,  0,
    36, 36, 36,   0,   0,   0, 36, 36, 36,   0,   0,   0, 255, 255, 255,
         0,   0,   0, 36, 36, 36,   0,   0,   0, 36, 36, 36,
     0,  0,  0,   0,   0,   0,  0,  0,  0,   0,   0,   0, 255, 255, 255,
         0,   0,   0,  0,  0,  0,   0,   0,   0,  0,  0,  0,
     0,  0,  0,   0,   0,   0,  0,  0,  0,   0,   0,   0, 255, 255, 255,
         0,   0,   0,  0,  0,  0,   0,   0,   0,  0,  0,  0,
     0,  0,  0,   0,   0,   0,  0,  0,  0,   0,   0,   0, 255, 255, 255,
         0,   0,   0,  0,  0,  0,   0,   0,   0,  0,  0,  0,
     0,  0,  0,   0,   0,   0,  0,  0,  0,   0,   0,   0, 255, 255, 255,
         0,   0,   0,  0,  0,  0,   0,   0,   0,  0,  0,  0,
     0,  0,  0,   0,   0,   0,  0,  0,  0,   0,   0,   0, 255, 255, 255,
         0,   0,   0,  0,  0,  0,   0,   0,   0,  0,  0,  0,
     0,  0,  0,   0,   0,   0,  0,  0,  0,   0,   0,   0, 255, 255, 255,
         0,   0,   0,  0,  0,  0,   0,   0,   0,  0,  0,  0,
     0,  0,  0,   0,   0,   0,  0,  0,  0,   0,   0,   0, 255, 255, 255,
         0,   0,   0,  0,  0,  0,   0,   0,   0,  0,  0,  0,
     0,  0,  0,   0,   0,   0,  0,  0,  0,   0,   0,   0, 255, 255, 255,
         0,   0,   0,  0,  0,  0,   0,   0,   0,  0,  0,  0,
     0,  0,  0,   0,   0,   0,  0,  0,  0,   0,   0,   0, 255, 255, 255,
         0,   0,   0,  0,  0,  0,   0,   0,   0,  0,  0,  0,
     0,  0,  0,   0,   0,   0,  0,  0,  0,   0,   0,   0, 255, 255, 255,
         0,   0,   0,  0,  0,  0,   0,   0,   0,  0,  0,  0,
     0,  0,  0,   0,   0,   0,  0,  0,  0,   0,   0,   0, 255, 255, 255,
         0,   0,   0,  0,  0,  0,   0,   0,   0,  0,  0,  0,
     0,  0,  0,   0,   0,   0,  0,  0,  0,   0,   0,   0, 255, 255, 255,
         0,   0,   0,  0,  0,  0,   0,   0,   0,  0,  0,  0,
     0,  0,  0,   0,   0,   0,  0,  0,  0,   0,   0,   0, 255, 255, 255,
         0,   0,   0,  0,  0,  0,   0,   0,   0,  0,  0,  0,
     0,  0,  0,   0,   0,   0,  0,  0,  0,   0,   0,   0, 255, 255, 255,
         0,   0,   0,  0,  0,  0,   0,   0,   0,  0,  0,  0,
     0,  0,  0,   0,   0,   0,  0,  0,  0,   0,   0,   0, 255, 255, 255,
         0,   0,   0,  0,  0,  0,   0,   0,   0,  0,  0,  0,
  };
  const int lda = k;
  int a_offset[] = {
      0, 0, 0, -51, -51, 0, -36, 0, 0, 0,
      0, 0, 0,   0,   0, 0,   0, 0, 0, 0,
      0, 0
  };
  MatrixMap<const std::uint8_t, MapOrder::RowMajor> lhs(a_data, m, k, lda);
  const OffsetColMap lhs_offset(a_offset, m);

  // 27 x 25, column-wise.
  const uint8_t b_data[] = {
    127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 119, 119, 119,
         119, 119, 119, 127, 127, 127, 119, 119, 119, 119, 119, 119,
    127, 127, 127, 127, 127, 127, 127, 127, 127, 119, 119, 119, 119, 119, 119,
         119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119,
    127, 127, 127, 127, 127, 127, 127, 127, 127, 119, 119, 119, 119, 119, 119,
         119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119,
    127, 127, 127, 127, 127, 127, 127, 127, 127, 119, 119, 119, 119, 119, 119,
         119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119,
    127, 127, 127, 127, 127, 127, 127, 127, 127, 119, 119, 119, 119, 119, 119,
         127, 127, 127, 119, 119, 119, 119, 119, 119, 127, 127, 127,
    127, 127, 127, 119, 119, 119, 119, 119, 119, 127, 127, 127, 119, 119, 119,
         119, 119, 119, 127, 127, 127, 119, 119, 119, 119, 119, 119,
    119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119,
         119, 119, 119, 119, 119, 119, 119, 119, 119, 136, 136, 136,
    119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119,
         119, 119, 119, 119, 119, 119, 136, 136, 136, 119, 119, 119,
    119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119,
         119, 119, 119, 136, 136, 136, 119, 119, 119, 119, 119, 119,
    119, 119, 119, 119, 119, 119, 127, 127, 127, 119, 119, 119, 119, 119, 119,
         127, 127, 127, 119, 119, 119, 119, 119, 119, 127, 127, 127,
    127, 127, 127, 119, 119, 119, 119, 119, 119, 127, 127, 127, 119, 119, 119,
         119, 119, 119, 127, 127, 127, 119, 119, 119, 119, 119, 119,
    119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119,
         136, 136, 136, 119, 119, 119, 119, 119, 119, 119, 119, 119,
    119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 136, 136, 136,
         119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119,
    119, 119, 119, 119, 119, 119, 119, 119, 119, 136, 136, 136, 119, 119, 119,
         119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119,
    119, 119, 119, 119, 119, 119, 127, 127, 127, 119, 119, 119, 119, 119, 119,
         127, 127, 127, 119, 119, 119, 119, 119, 119, 127, 127, 127,
    127, 127, 127, 119, 119, 119, 119, 119, 119, 127, 127, 127, 119, 119, 119,
         119, 119, 119, 127, 127, 127, 119, 119, 119, 119, 119, 119,
    119, 119, 119, 119, 119, 119, 136, 136, 136, 119, 119, 119, 119, 119, 119,
         119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119,
    119, 119, 119, 136, 136, 136, 119, 119, 119, 119, 119, 119, 119, 119, 119,
         119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119,
    136, 136, 136, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119,
         119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119,
    119, 119, 119, 119, 119, 119, 127, 127, 127, 119, 119, 119, 119, 119, 119,
         127, 127, 127, 119, 119, 119, 119, 119, 119, 127, 127, 127,
    127, 127, 127, 119, 119, 119, 119, 119, 119, 127, 127, 127, 119, 119, 119,
         119, 119, 119, 127, 127, 127, 127, 127, 127, 127, 127, 127,
    119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119,
         119, 119, 119, 127, 127, 127, 127, 127, 127, 127, 127, 127,
    119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119,
         119, 119, 119, 127, 127, 127, 127, 127, 127, 127, 127, 127,
    119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119, 119,
         119, 119, 119, 127, 127, 127, 127, 127, 127, 127, 127, 127,
    119, 119, 119, 119, 119, 119, 127, 127, 127, 119, 119, 119, 119, 119, 119,
         127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127
  };
  const int ldb = k;
  int b_offset = -127;
  MatrixMap<const std::uint8_t, MapOrder::ColMajor> rhs(b_data, k, n, ldb);
  const OffsetRowDup rhs_offset(b_offset, rhs.cols());

  // 22 x 25, column-wise.
  const uint8_t expected_c_data[] = {
      7,  37,  37,  67,  67,  39,  79,   7,   7,   7,   7,   7,   7,   7,   7,
           7,   7,   7,   7,   7,   7,   7,
      7,   7,  37,  87,  67,  23,  91,   7,   7,   7,   7,   7,   7,   7,   7,
           7,   7,   7,   7,   7,   7,   7,
      7,   7,  37,  87,  67,  23,  91,   7,   7,   7,   7,   7,   7,   7,   7,
           7,   7,   7,   7,   7,   7,   7,
      7,   7,  37,  87,  67,  23,  91,   7,   7,   7,   7,   7,   7,   7,   7,
           7,   7,   7,   7,   7,   7,   7,
      7,  37,  37,  67,  67,  39,  79,   7,   7,   7,   7,   7,   7,   7,   7,
           7,   7,   7,   7,   7,   7,   7,
      7,  37,   7,  67,  87,  23,  91,   7,   7,   7,   7,   7,   7,   7,   7,
           7,   7,   7,   7,   7,   7,   7,
      7,   7,   7,  87,  87,   7, 103,   7,   7,   7,   7,   7,   7,   7,   7,
           7,   7,   7,   7,   7,   7,   7,
      7,   7,  71,  87,  45,  41,  77,   7,   7,   7,   7,   7,   7,   7,   7,
           7,   7,   7,   7,   7,   7,   7,
      7,   7,   7,  87,  87,   7, 103,   7,   7,   7,   7,   7,   7,   7,   7,
           7,   7,   7,   7,   7,   7,   7,
      7,  37,   7,  67,  87,  23,  91,   7,   7,   7,   7,   7,   7,   7,   7,
           7,   7,   7,   7,   7,   7,   7,
      7,  37,   7,  67,  87,  23,  91,   7,   7,   7,   7,   7,   7,   7,   7,
           7,   7,   7,   7,   7,   7,   7,
      7,  71,   7,  45,  87,  41,  77,   7,   7,   7,   7,   7,   7,   7,   7,
           7,   7,   7,   7,   7,   7,   7,
    255, 135, 135, 255, 255, 143, 255, 255, 255, 255, 255, 255, 255, 255, 255,
         255, 255, 255, 255, 255, 255, 255,
      7,  71,   7,  45,  87,  41,  77,   7,   7,   7,   7,   7,   7,   7,   7,
           7,   7,   7,   7,   7,   7,   7,
      7,  37,   7,  67,  87,  23,  91,   7,   7,   7,   7,   7,   7,   7,   7,
           7,   7,   7,   7,   7,   7,   7,
      7,  37,   7,  67,  87,  23,  91,   7,   7,   7,   7,   7,   7,   7,   7,
           7,   7,   7,   7,   7,   7,   7,
      7,   7,   7,  87,  87,   7, 103,   7,   7,   7,   7,   7,   7,   7,   7,
           7,   7,   7,   7,   7,   7,   7,
      7,   7,  71,  87,  45,  41,  77,   7,   7,   7,   7,   7,   7,   7,   7,
           7,   7,   7,   7,   7,   7,   7,
      7,   7,   7,  87,  87,   7, 103,   7,   7,   7,   7,   7,   7,   7,   7,
           7,   7,   7,   7,   7,   7,   7,
      7,  37,   7,  67,  87,  23,  91,   7,   7,   7,   7,   7,   7,   7,   7,
           7,   7,   7,   7,   7,   7,   7,
      7,  37,  37,  67,  67,  39,  79,   7,   7,   7,   7,   7,   7,   7,   7,
           7,   7,   7,   7,   7,   7,   7,
      7,   7,  37,  87,  67,  23,  91,   7,   7,   7,   7,   7,   7,   7,   7,
           7,   7,   7,   7,   7,   7,   7,
      7,   7,  37,  87,  67,  23,  91,   7,   7,   7,   7,   7,   7,   7,   7,
           7,   7,   7,   7,   7,   7,   7,
      7,   7,  37,  87,  67,  23,  91,   7,   7,   7,   7,   7,   7,   7,   7,
           7,   7,   7,   7,   7,   7,   7,
      7,  37,  37,  67,  67,  39,  79,   7,   7,   7,   7,   7,   7,   7,   7,
           7,   7,   7,   7,   7,   7,   7,
     99,  99,  99,  99,  99,  99,  99,  99,  99,  99,  99,  99,  99,  99,  99,
          99,  99,  99,  99,  99,  99,  99,
    111, 111, 111, 111, 111, 111, 111, 111, 111, 111, 111, 111, 111, 111, 111,
         111, 111, 111, 111, 111, 111, 111,
  };
  const int ldc = m;
  int c_offset[] = {
      6477, 12954, 12954, 7793, 7793, 12954, 9282, 6477, 6477, 6477,
      6477,  6477,  6477, 6477, 6477,  6477, 6477, 6477, 6477, 6477,
      6477,  6477,
  };
  int c_mult_int[] = {
      41121, 20560, 20560, 34267, 34267, 21937, 28784, 41121, 41121, 41121,
      41121, 41121, 41121, 41121, 41121, 41121, 41121, 41121, 41121, 41121,
      41121, 41121,
  };
  const int c_shift = 21;

  const int c_count = m * n;
  std::unique_ptr<uint8_t[]> output_data(new uint8_t[c_count]);
  MatrixMap<std::uint8_t, MapOrder::ColMajor> result(output_data.get(), m, n,
                                                     ldc);
  const OffsetColMap result_offset(c_offset, m);
  const OffsetColMap result_mult_int(c_mult_int, m);
  const int result_shift = c_shift;

  GemmContext gemm_context;
  auto output_pipeline = MakeStandardOutputPipeline<VectorShape::Col>(
      result_offset, result_mult_int, result_shift);
  GemmWithOutputPipelinePC<uint8_t, uint8_t, DefaultL8R8BitDepthParams>(
      &gemm_context, lhs, rhs, &result, lhs_offset, rhs_offset,
      output_pipeline);

  ResultStats stats;
  GetResultStats(output_data.get(), expected_c_data, c_count, &stats);

  ResultStatsBounds bounds;
  const bool good = CheckResultStatsBounds(stats, bounds);
  printf("TestWithLargeDataPerChannelQuantization: %s\n",
         good ? "PASS" : "FAIL");
  ReportResultStats(stats, bounds);
  Check(good);
}

// Runs a small set of hand-calculated data through the implementation.
void TestWithSmallData() {
  const int m = 4;
  const int n = 2;
  const int k = 3;
  // Matrix A (LHS) is:
  // |  7 | 10 | 13 | 16 |
  // |  8 | 11 | 14 | 17 |
  // |  9 | 12 | 15 | 18 |
  const uint8_t a_data[] = {7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18};
  // Matrix B (RHS) is:
  // |  1 |  3 |  5 |
  // |  2 |  4 |  6 |
  const uint8_t b_data[] = {1, 2, 3, 4, 5, 6};
  // Here are the results we expect, from hand calculations:
  // (1 * 7) + (3 * 8) + (5 * 9) = 76
  // (2 * 7) + (4 * 8) + (6 * 9) = 100
  // (1 * 10) + (3 * 11) + (5 * 12) = 103
  // (2 * 10) + (4 * 11) + (6 * 12) = 136
  // (1 * 13) + (3 * 14) + (5 * 15) = 130
  // (2 * 13) + (4 * 14) + (6 * 15) = 172
  // (1 * 16) + (3 * 17) + (5 * 18) = 157
  // (2 * 16) + (4 * 17) + (6 * 18) = 208
  // That means matrix C should be:
  // |  76 | 103 | 130 | 157 |
  // | 100 | 136 | 172 | 208 |
  const uint8_t expected_data[] = {76, 100, 103, 136, 130, 172, 157, 208};

  const int c_count = m * n;
  std::unique_ptr<uint8_t[]> output_data(new uint8_t[c_count]);

  const bool is_a_transposed = true;
  const bool is_b_transposed = true;
  const bool is_c_transposed = true;
  const int lda = k;
  const int ldb = n;
  const int ldc = n;

  const int a_offset = 0;
  const int b_offset = 0;
  const int c_offset = 0;
  const int c_mult = 1;
  const int c_shift = 0;

  gemmlowp::eight_bit_int_gemm::EightBitIntGemm(
      is_a_transposed, is_b_transposed, is_c_transposed, m, n, k, a_data,
      a_offset, lda, b_data, b_offset, ldb, output_data.get(), c_offset, c_mult,
      c_shift, ldc, eight_bit_int_gemm::BitDepthSetting::A8B8);

  ResultStats stats;
  GetResultStats(output_data.get(), expected_data, c_count, &stats);

  ResultStatsBounds bounds;
  const bool good = CheckResultStatsBounds(stats, bounds);
  printf("TestWithSmallData: %s\n", good ? "PASS" : "FAIL");
  ReportResultStats(stats, bounds);
  Check(good);
}

// This is the most realistic test of how we'll be using the low-precision GEMM
// function in applications. It takes in large input matrices that have been
// captured from an actual neural network run.
void TestWithRealData(eight_bit_int_gemm::BitDepthSetting BitDepth,
                      int tolerance_median, int tolerance_max) {
  std::unique_ptr<uint8_t[]> output_data(new uint8_t[test_data::c_count]);
  gemmlowp::eight_bit_int_gemm::EightBitIntGemm(
      test_data::is_a_transposed, test_data::is_b_transposed,
      test_data::is_c_transposed, test_data::m, test_data::n, test_data::k,
      test_data::a_data, test_data::a_offset, test_data::k, test_data::b_data,
      test_data::b_offset, test_data::k, output_data.get(), test_data::c_offset,
      test_data::c_mult_int, test_data::c_shift, test_data::m, BitDepth);

  ResultStats stats;
  GetResultStats(output_data.get(), test_data::expected_c_data,
                 test_data::c_count, &stats);

  ResultStatsBounds bounds;
  if (BitDepth == eight_bit_int_gemm::BitDepthSetting::A5B7) {
    bounds.med_unsigned_diff = tolerance_median;
    bounds.max_unsigned_diff = tolerance_max;
    bounds.med_signed_diff = 0;
    bounds.mean_signed_diff = 0.2f;
  }

  const bool good = CheckResultStatsBounds(stats, bounds);
  printf("TestWithRealData: %s with %s\n", good ? "PASS" : "FAIL",
         GetBitDepthName(BitDepth));
  ReportResultStats(stats, bounds);
  Check(good);
}

template <MapOrder ResultOrder>
void TestOutputStages(int rows, int depth, int cols, int result_offset,
                      int result_mult_int, int result_shift) {
  Matrix<std::uint8_t, MapOrder::RowMajor> lhs(rows, depth);
  Matrix<std::uint8_t, MapOrder::ColMajor> rhs(depth, cols);
  Matrix<std::int32_t, ResultOrder> result_raw_int32(rows, cols);
  MakeRandom(&lhs, 8);
  MakeRandom(&rhs, 8);
  const int lhs_offset = 12;
  const int rhs_offset = -34;

  // Test an empty pipeline, i.e. returning raw int32 accumulators.
  auto empty_pipeline = std::make_tuple();
  GemmContext context;
  GemmWithOutputPipeline<std::uint8_t, std::int32_t, DefaultL8R8BitDepthParams>(
      &context, lhs.const_map(), rhs.const_map(), &result_raw_int32, lhs_offset,
      rhs_offset, empty_pipeline);

  for (int r = 0; r < rows; r++) {
    for (int c = 0; c < cols; c++) {
      std::int32_t expected = 0;
      for (int d = 0; d < depth; d++) {
        std::int32_t lhs_val =
            static_cast<std::int32_t>(lhs(r, d)) + lhs_offset;
        std::int32_t rhs_val =
            static_cast<std::int32_t>(rhs(d, c)) + rhs_offset;
        expected += lhs_val * rhs_val;
      }
      Check(expected == result_raw_int32(r, c));
    }
  }

  // Test a pipeline with only the quantize-down stage, still returning
  // unclamped (but scaled) int32's
  OutputStageQuantizeDownInt32ToUint8Scale quantize_down_stage;
  quantize_down_stage.result_offset = result_offset;
  quantize_down_stage.result_mult_int = result_mult_int;
  quantize_down_stage.result_shift = result_shift;
  auto quantize_down_pipeline = std::make_tuple(quantize_down_stage);
  Matrix<std::int32_t, ResultOrder> result_quantized_down_int32(rows, cols);
  GemmWithOutputPipeline<std::uint8_t, std::int32_t, DefaultL8R8BitDepthParams>(
      &context, lhs.const_map(), rhs.const_map(), &result_quantized_down_int32,
      lhs_offset, rhs_offset, quantize_down_pipeline);

  std::uint64_t sum = 0;
  for (int r = 0; r < rows; r++) {
    for (int c = 0; c < cols; c++) {
      std::int32_t raw = result_raw_int32(r, c);
      const std::int32_t rounding =
          (result_shift < 1) ? 0 : (1 << (result_shift - 1));
      std::int32_t expected =
          ((raw + result_offset) * result_mult_int + rounding) >> result_shift;
      Check(expected == result_quantized_down_int32(r, c));
      sum += expected;
    }
  }
  std::uint64_t avg = sum / (rows * cols);
  // Test that the average quantized-down value falls reasonably in the
  // middle of the [0..255] range. Otherwise, the multiplier / shift need to be
  // adjusted.
  Check(avg >= 64 && avg <= 192);

  // Test the familiar default pipeline consisting of quantize-down and
  // clamp-and-cast-to-uint8.
  OutputStageSaturatingCastToUint8 saturating_cast_stage;
  auto quantize_down_and_saturating_cast_pipeline =
      std::make_tuple(quantize_down_stage, saturating_cast_stage);
  Matrix<std::uint8_t, ResultOrder> result_quantized_down_saturated_uint8(rows,
                                                                          cols);
  GemmWithOutputPipeline<std::uint8_t, std::uint8_t, DefaultL8R8BitDepthParams>(
      &context, lhs.const_map(), rhs.const_map(),
      &result_quantized_down_saturated_uint8, lhs_offset, rhs_offset,
      quantize_down_and_saturating_cast_pipeline);

  for (int r = 0; r < rows; r++) {
    for (int c = 0; c < cols; c++) {
      std::int32_t quantized = result_quantized_down_int32(r, c);
      std::uint8_t expected = std::min(std::max(quantized, 0), 255);
      Check(expected == result_quantized_down_saturated_uint8(r, c));
    }
  }

  // Test a bias-addition with row-vector
  std::vector<std::int32_t> row_vector_data(cols);
  for (int i = 0; i < cols; i++) {
    row_vector_data[i] = (Random() % 1000) - 500;
  }
  typedef VectorMap<std::int32_t, VectorShape::Row> RowVectorMap;
  RowVectorMap row_vector_map(row_vector_data.data(), cols);
  OutputStageBiasAddition<RowVectorMap> row_bias_addition_stage;
  row_bias_addition_stage.bias_vector = row_vector_map;
  auto row_bias_addition_pipeline = std::make_tuple(row_bias_addition_stage);
  Matrix<std::int32_t, ResultOrder> result_of_row_bias_addition(rows, cols);
  GemmWithOutputPipeline<std::uint8_t, std::int32_t, DefaultL8R8BitDepthParams>(
      &context, lhs.const_map(), rhs.const_map(), &result_of_row_bias_addition,
      lhs_offset, rhs_offset, row_bias_addition_pipeline);
  for (int r = 0; r < rows; r++) {
    for (int c = 0; c < cols; c++) {
      std::int32_t expected = result_raw_int32(r, c) + row_vector_data[c];
      Check(expected == result_of_row_bias_addition(r, c));
    }
  }

  // Test a bias-addition with column-vector
  std::vector<std::int32_t> col_vector_data(rows);
  for (int i = 0; i < rows; i++) {
    col_vector_data[i] = (Random() % 1000) - 500;
  }
  typedef VectorMap<std::int32_t, VectorShape::Col> ColVectorMap;
  ColVectorMap col_vector_map(col_vector_data.data(), rows);
  OutputStageBiasAddition<ColVectorMap> col_bias_addition_stage;
  col_bias_addition_stage.bias_vector = col_vector_map;
  auto col_bias_addition_pipeline = std::make_tuple(col_bias_addition_stage);
  Matrix<std::int32_t, ResultOrder> result_of_col_bias_addition(rows, cols);
  GemmWithOutputPipeline<std::uint8_t, std::int32_t, DefaultL8R8BitDepthParams>(
      &context, lhs.const_map(), rhs.const_map(), &result_of_col_bias_addition,
      lhs_offset, rhs_offset, col_bias_addition_pipeline);
  for (int r = 0; r < rows; r++) {
    for (int c = 0; c < cols; c++) {
      std::int32_t expected = result_raw_int32(r, c) + col_vector_data[r];
      Check(expected == result_of_col_bias_addition(r, c));
    }
  }

  // Test a clamp
  OutputStageClamp clamp_stage;
  // Determine min and max of raw int32 accumulators
  std::int32_t raw_min = std::numeric_limits<std::int32_t>::max();
  std::int32_t raw_max = std::numeric_limits<std::int32_t>::min();
  for (int r = 0; r < rows; r++) {
    for (int c = 0; c < cols; c++) {
      raw_min = std::min(raw_min, result_raw_int32(r, c));
      raw_max = std::max(raw_max, result_raw_int32(r, c));
    }
  }
  // Pick some interesting clamp min/max bounds
  clamp_stage.min = static_cast<std::int32_t>(raw_min * 0.7 + raw_max * 0.3);
  clamp_stage.max = static_cast<std::int32_t>(raw_min * 0.3 + raw_max * 0.7);
  assert(raw_min <= clamp_stage.min && clamp_stage.min <= clamp_stage.max &&
         clamp_stage.max <= raw_max);
  auto clamp_pipeline = std::make_tuple(clamp_stage);
  Matrix<std::int32_t, ResultOrder> result_clamped(rows, cols);
  GemmWithOutputPipeline<std::uint8_t, std::int32_t, DefaultL8R8BitDepthParams>(
      &context, lhs.const_map(), rhs.const_map(), &result_clamped, lhs_offset,
      rhs_offset, clamp_pipeline);
  for (int r = 0; r < rows; r++) {
    for (int c = 0; c < cols; c++) {
      std::int32_t raw = result_raw_int32(r, c);
      std::int32_t expected =
          std::min(std::max(raw, clamp_stage.min), clamp_stage.max);
      Check(expected == result_clamped(r, c));
    }
  }

  // Test tanh
  OutputStageTanh tanh_stage;
  const std::int32_t real_zero_as_int32 = (raw_max + raw_min) / 2;
  const std::int32_t real_amplitude_as_int32 = (raw_max - raw_min) / 16;
  tanh_stage.real_zero_as_int32 = real_zero_as_int32;
  tanh_stage.real_amplitude_as_int32 = real_amplitude_as_int32;
  auto tanh_pipeline = std::make_tuple(tanh_stage);
  Matrix<std::int32_t, ResultOrder> result_tanh(rows, cols);
  GemmWithOutputPipeline<std::uint8_t, std::int32_t, DefaultL8R8BitDepthParams>(
      &context, lhs.const_map(), rhs.const_map(), &result_tanh, lhs_offset,
      rhs_offset, tanh_pipeline);
  for (int r = 0; r < rows; r++) {
    for (int c = 0; c < cols; c++) {
      std::int32_t raw = result_raw_int32(r, c);
      double real_input =
          double(raw - real_zero_as_int32) / real_amplitude_as_int32;
      double expected = std::tanh(real_input);
      std::int32_t actual_int32 = result_tanh(r, c);
      double actual =
          double(actual_int32 - real_zero_as_int32) / real_amplitude_as_int32;
      Check(std::abs(expected - actual) < 2e-4);
    }
  }

  // Test a pipeline with bias and clamp
  auto bias_clamp_pipeline =
      std::make_tuple(col_bias_addition_stage, clamp_stage);
  Matrix<std::int32_t, ResultOrder> result_biased_clamped(rows, cols);
  GemmWithOutputPipeline<std::uint8_t, std::int32_t, DefaultL8R8BitDepthParams>(
      &context, lhs.const_map(), rhs.const_map(), &result_biased_clamped,
      lhs_offset, rhs_offset, bias_clamp_pipeline);
  for (int r = 0; r < rows; r++) {
    for (int c = 0; c < cols; c++) {
      std::int32_t raw = result_raw_int32(r, c);
      std::int32_t biased = raw + col_vector_data[r];
      std::int32_t expected =
          std::min(std::max(biased, clamp_stage.min), clamp_stage.max);
      Check(expected == result_biased_clamped(r, c));
    }
  }

  // Test a full pipeline with bias and clamp and quantization down to 8bit
  // result
  auto bias_clamp_quantize_cast_pipeline =
      std::make_tuple(col_bias_addition_stage, clamp_stage, quantize_down_stage,
                      saturating_cast_stage);
  Matrix<std::uint8_t, ResultOrder> result_biased_clamped_quantized_casted(
      rows, cols);
  GemmWithOutputPipeline<std::uint8_t, std::uint8_t, DefaultL8R8BitDepthParams>(
      &context, lhs.const_map(), rhs.const_map(),
      &result_biased_clamped_quantized_casted, lhs_offset, rhs_offset,
      bias_clamp_quantize_cast_pipeline);
  for (int r = 0; r < rows; r++) {
    for (int c = 0; c < cols; c++) {
      const std::int32_t rounding =
          (result_shift < 1) ? 0 : (1 << (result_shift - 1));
      std::int32_t quantized =
          ((result_biased_clamped(r, c) + result_offset) * result_mult_int +
           rounding) >>
          result_shift;
      std::uint8_t expected = std::min(std::max(quantized, 0), 255);
      Check(expected == result_biased_clamped_quantized_casted(r, c));
    }
  }

  printf("TestOutputStages: PASS with ResultOrder=%s\n",
         OrderName(ResultOrder));
}

#ifndef GEMMLOWP_SKIP_EXHAUSTIVE_TESTS
void TestExhaustively() {
  GemmContext context;

  // Test the internal GEMM interfaces
  test_gemm<SingleThreadGemmWrapper<
      DefaultKernel<KernelFamily::Gemm, DefaultL8R8BitDepthParams>,
      std::uint8_t, DefaultL8R8BitDepthParams>>(&context);

  test_gemm<MultiThreadGemmWrapper<
      DefaultKernel<KernelFamily::Gemm, DefaultL8R8BitDepthParams>,
      std::uint8_t, DefaultL8R8BitDepthParams>>(&context);

  // Test the public GEMM interfaces
  test_gemm<PublicGemmWrapper<uint8_t, DefaultL8R8BitDepthParams>>(&context);

  test_gemm<EightBitIntGemmWrapper<uint8_t,
                                   eight_bit_int_gemm::BitDepthSetting::A8B8>>(
      &context);

  // Test GEMV cases (internal interfaces)
  test_gemv<SingleThreadGemmWrapper<
      DefaultKernel<KernelFamily::Gemv, DefaultL8R8BitDepthParams>,
      std::uint8_t, DefaultL8R8BitDepthParams>>(&context);

  test_gemv<MultiThreadGemmWrapper<
      DefaultKernel<KernelFamily::Gemv, DefaultL8R8BitDepthParams>,
      std::uint8_t, DefaultL8R8BitDepthParams>>(&context);

  // Test GEMV cases (public interfaces)
  test_gemv<PublicGemmWrapper<uint8_t, DefaultL8R8BitDepthParams>>(&context);

  test_gemv<EightBitIntGemmWrapper<uint8_t,
                                   eight_bit_int_gemm::BitDepthSetting::A8B8>>(
      &context);

  // Test other bit depths
  // L7R5
  test_gemm<SingleThreadGemmWrapper<
      DefaultKernel<KernelFamily::Gemm, DefaultL7R5BitDepthParams>,
      std::uint8_t, DefaultL7R5BitDepthParams>>(&context);

  test_gemv<SingleThreadGemmWrapper<
      DefaultKernel<KernelFamily::Gemv, DefaultL7R5BitDepthParams>,
      std::uint8_t, DefaultL7R5BitDepthParams>>(&context);

  test_gemm<EightBitIntGemmWrapper<std::uint8_t,
                                   eight_bit_int_gemm::BitDepthSetting::A5B7>>(
      &context);

  // Test specific kernels with various different formats,
  // to exercises corner cases especially in the packing code.
  test_gemm_kernel<
      ReferenceKernel<KernelFormat<KernelSideFormat<CellFormat<1, 1>, 1>,
                                   KernelSideFormat<CellFormat<1, 1>, 1>>>>(
      &context);

  test_gemm_kernel<
      ReferenceKernel<KernelFormat<KernelSideFormat<CellFormat<4, 2>, 1>,
                                   KernelSideFormat<CellFormat<4, 2>, 2>>>>(
      &context);

  test_gemm_kernel<
      ReferenceKernel<KernelFormat<KernelSideFormat<CellFormat<4, 2>, 4>,
                                   KernelSideFormat<CellFormat<4, 2>, 5>>>>(
      &context);

  test_gemm_kernel<ReferenceKernel<KernelFormat<
      KernelSideFormat<CellFormat<3, 4, CellOrder::DepthMajor>, 2>,
      KernelSideFormat<CellFormat<5, 4, CellOrder::DepthMajor>, 3>>>>(&context);

  test_gemm_kernel<ReferenceKernel<KernelFormat<
      KernelSideFormat<CellFormat<3, 4, CellOrder::WidthMajor>, 2>,
      KernelSideFormat<CellFormat<5, 4, CellOrder::WidthMajor>, 3>>>>(&context);

  test_gemm_kernel<ReferenceKernel<KernelFormat<
      KernelSideFormat<CellFormat<5, 2, CellOrder::WidthMajor>, 3>,
      KernelSideFormat<CellFormat<4, 2, CellOrder::DepthMajor>, 2>>>>(&context);

  test_gemm_kernel<ReferenceKernel<KernelFormat<
      KernelSideFormat<CellFormat<5, 2, CellOrder::DepthMajor>, 3>,
      KernelSideFormat<CellFormat<4, 2, CellOrder::WidthMajor>, 2>>>>(&context);

  test_gemm_kernel<ReferenceKernel<KernelFormat<
      KernelSideFormat<CellFormat<8, 8, CellOrder::Diagonal>, 2>,
      KernelSideFormat<CellFormat<3, 8, CellOrder::WidthMajor>, 1>>>>(&context);

  test_gemm_kernel<ReferenceKernel<KernelFormat<
      KernelSideFormat<CellFormat<1, 4, CellOrder::DepthMajor>, 1>,
      KernelSideFormat<CellFormat<4, 4, CellOrder::Diagonal>, 1>>>>(&context);
}
#endif  // not GEMMLOWP_SKIP_EXHAUSTIVE_TESTS

void test() {
#ifdef GEMMLOWP_TEST_PROFILE
  RegisterCurrentThreadForProfiling();
  StartProfiling();
#endif

  // Run a first quick test against hand-calculated data.
  TestWithSmallData();

#ifndef GEMMLOWP_SKIP_EXHAUSTIVE_TESTS
  TestExhaustively();
#endif

  // Run against actual data from a network evaluation.
  TestWithRealData(eight_bit_int_gemm::BitDepthSetting::A8B8, 0, 0);
  TestWithRealData(eight_bit_int_gemm::BitDepthSetting::A5B7, 2, 10);

  // Test non-default output pipelines with various combinations of
  // output stages.
  TestOutputStages<MapOrder::RowMajor>(63, 10, 127, 5, 17, 14);
  TestOutputStages<MapOrder::ColMajor>(63, 10, 127, 5, 17, 14);

  // Test per channel quantization.
  TestWithSmallDataPerChannelQuantization();
  TestWithLargeDataPerChannelQuantization();
#ifdef GEMMLOWP_TEST_PROFILE
  FinishProfiling();
#endif

  std::cerr << "All tests passed." << std::endl;

  // We have been testing the eight_bit_int_gemm, so we should free its
  // persistent
  // resources now to avoid having leak-checking tools report leaks.
  eight_bit_int_gemm::FreePersistentResources();
}

}  // end namespace gemmlowp

// For iOS, we need to define our own main(), so skip it here.
#if !(defined(__APPLE__) && (TARGET_OS_IPHONE || TARGET_IPHONE_SIMULATOR))
int main() { gemmlowp::test(); }
#endif