diff --git a/Android.bp b/Android.bp
index 946aa99..30eca18 100644
--- a/Android.bp
+++ b/Android.bp
@@ -40,13 +40,21 @@
         "libgav1/src/buffer_pool.cc",
         "libgav1/src/decoder.cc",
         "libgav1/src/decoder_impl.cc",
+        "libgav1/src/dsp/arm/average_blend_neon.cc",
+        "libgav1/src/dsp/arm/convolve_neon.cc",
+        "libgav1/src/dsp/arm/distance_weighted_blend_neon.cc",
+        "libgav1/src/dsp/arm/intra_edge_neon.cc",
         "libgav1/src/dsp/arm/intrapred_cfl_neon.cc",
         "libgav1/src/dsp/arm/intrapred_directional_neon.cc",
         "libgav1/src/dsp/arm/intrapred_filter_intra_neon.cc",
         "libgav1/src/dsp/arm/intrapred_neon.cc",
         "libgav1/src/dsp/arm/intrapred_smooth_neon.cc",
+        "libgav1/src/dsp/arm/inverse_transform_neon.cc",
         "libgav1/src/dsp/arm/loop_filter_neon.cc",
         "libgav1/src/dsp/arm/loop_restoration_neon.cc",
+        "libgav1/src/dsp/arm/mask_blend_neon.cc",
+        "libgav1/src/dsp/arm/obmc_neon.cc",
+        "libgav1/src/dsp/arm/warp_neon.cc",
         "libgav1/src/dsp/average_blend.cc",
         "libgav1/src/dsp/cdef.cc",
         "libgav1/src/dsp/constants.cc",
@@ -60,9 +68,12 @@
         "libgav1/src/dsp/inverse_transform.cc",
         "libgav1/src/dsp/loop_filter.cc",
         "libgav1/src/dsp/loop_restoration.cc",
-        "libgav1/src/dsp/mask_blending.cc",
+        "libgav1/src/dsp/mask_blend.cc",
         "libgav1/src/dsp/obmc.cc",
         "libgav1/src/dsp/warp.cc",
+        "libgav1/src/dsp/x86/average_blend_sse4.cc",
+        "libgav1/src/dsp/x86/convolve_sse4.cc",
+        "libgav1/src/dsp/x86/distance_weighted_blend_sse4.cc",
         "libgav1/src/dsp/x86/intra_edge_sse4.cc",
         "libgav1/src/dsp/x86/intrapred_cfl_sse4.cc",
         "libgav1/src/dsp/x86/intrapred_smooth_sse4.cc",
@@ -70,6 +81,7 @@
         "libgav1/src/dsp/x86/inverse_transform_sse4.cc",
         "libgav1/src/dsp/x86/loop_filter_sse4.cc",
         "libgav1/src/dsp/x86/loop_restoration_sse4.cc",
+        "libgav1/src/dsp/x86/obmc_sse4.cc",
         "libgav1/src/internal_frame_buffer_list.cc",
         "libgav1/src/loop_filter_mask.cc",
         "libgav1/src/loop_restoration_info.cc",
@@ -88,7 +100,6 @@
         "libgav1/src/tile/bitstream/transform_size.cc",
         "libgav1/src/tile/prediction.cc",
         "libgav1/src/tile/tile.cc",
-        "libgav1/src/utils/allocator.cc",
         "libgav1/src/utils/bit_reader.cc",
         "libgav1/src/utils/block_parameters_holder.cc",
         "libgav1/src/utils/constants.cc",
@@ -97,7 +108,6 @@
         "libgav1/src/utils/logging.cc",
         "libgav1/src/utils/parameter_tree.cc",
         "libgav1/src/utils/raw_bit_reader.cc",
-        "libgav1/src/utils/scan.cc",
         "libgav1/src/utils/segmentation.cc",
         "libgav1/src/utils/segmentation_map.cc",
         "libgav1/src/utils/threadpool.cc",
diff --git a/libgav1/src/decoder.cc b/libgav1/src/decoder.cc
index b4251f2..8767887 100644
--- a/libgav1/src/decoder.cc
+++ b/libgav1/src/decoder.cc
@@ -1,9 +1,6 @@
 #include "src/decoder.h"
 
-#include <new>
-
 #include "src/decoder_impl.h"
-#include "src/utils/logging.h"
 
 namespace libgav1 {
 
@@ -13,15 +10,9 @@
 
 StatusCode Decoder::Init(const DecoderSettings* const settings) {
   if (initialized_) return kLibgav1StatusAlready;
-  if (settings != nullptr) {
-    if (settings->threads < 0) return kLibgav1StatusInvalidArgument;
-    settings_ = *settings;
-  }
-  impl_.reset(new (std::nothrow) DecoderImpl(&settings_));
-  if (impl_ == nullptr) {
-    LIBGAV1_DLOG(ERROR, "Failed to allocate DecoderImpl.");
-    return kLibgav1StatusOutOfMemory;
-  }
+  if (settings != nullptr) settings_ = *settings;
+  const StatusCode status = DecoderImpl::Create(&settings_, &impl_);
+  if (status != kLibgav1StatusOk) return status;
   initialized_ = true;
   return kLibgav1StatusOk;
 }
@@ -41,4 +32,7 @@
   return settings_.frame_parallel ? settings_.threads : 1;
 }
 
+// static.
+int Decoder::GetMaxBitdepth() { return DecoderImpl::GetMaxBitdepth(); }
+
 }  // namespace libgav1
diff --git a/libgav1/src/decoder.h b/libgav1/src/decoder.h
index 274b41c..b01107d 100644
--- a/libgav1/src/decoder.h
+++ b/libgav1/src/decoder.h
@@ -8,13 +8,14 @@
 #include "src/decoder_buffer.h"
 #include "src/decoder_settings.h"
 #include "src/status_code.h"
+#include "src/symbol_visibility.h"
 
 namespace libgav1 {
 
 // Forward declaration.
 class DecoderImpl;
 
-class Decoder {
+class LIBGAV1_PUBLIC Decoder {
  public:
   Decoder();
   ~Decoder();
@@ -55,6 +56,9 @@
   // is false, then this function will always return 1.
   int GetMaxAllowedFrames() const;
 
+  // Returns the maximum bitdepth that is supported by this decoder.
+  static int GetMaxBitdepth();
+
  private:
   bool initialized_ = false;
   DecoderSettings settings_;
diff --git a/libgav1/src/decoder_buffer.h b/libgav1/src/decoder_buffer.h
index 477acb7..6806603 100644
--- a/libgav1/src/decoder_buffer.h
+++ b/libgav1/src/decoder_buffer.h
@@ -4,6 +4,7 @@
 #include <cstdint>
 
 #include "src/frame_buffer.h"
+#include "src/symbol_visibility.h"
 
 // All the declarations in this file are part of the public ABI.
 
@@ -23,7 +24,7 @@
   kImageFormatMonochrome400
 };
 
-struct DecoderBuffer {
+struct LIBGAV1_PUBLIC DecoderBuffer {
   int NumPlanes() const {
     return (image_format == kImageFormatMonochrome400) ? 1 : 3;
   }
diff --git a/libgav1/src/decoder_impl.cc b/libgav1/src/decoder_impl.cc
index a01d2ee..dbfaf4c 100644
--- a/libgav1/src/decoder_impl.cc
+++ b/libgav1/src/decoder_impl.cc
@@ -1,10 +1,9 @@
 #include "src/decoder_impl.h"
 
 #include <algorithm>
+#include <atomic>
 #include <cassert>
-#include <condition_variable>  // NOLINT (unapproved c++11 header)
 #include <iterator>
-#include <mutex>  // NOLINT (unapproved c++11 header)
 #include <new>
 #include <utility>
 
@@ -16,6 +15,7 @@
 #include "src/post_filter.h"
 #include "src/prediction_mask.h"
 #include "src/quantizer.h"
+#include "src/utils/blocking_counter.h"
 #include "src/utils/common.h"
 #include "src/utils/logging.h"
 #include "src/utils/parameter_tree.h"
@@ -50,6 +50,24 @@
 
 }  // namespace
 
+// static
+StatusCode DecoderImpl::Create(const DecoderSettings* settings,
+                               std::unique_ptr<DecoderImpl>* output) {
+  if (settings->threads <= 0) {
+    LIBGAV1_DLOG(ERROR, "Invalid settings->threads: %d.", settings->threads);
+    return kLibgav1StatusInvalidArgument;
+  }
+  std::unique_ptr<DecoderImpl> impl(new (std::nothrow) DecoderImpl(settings));
+  if (impl == nullptr) {
+    LIBGAV1_DLOG(ERROR, "Failed to allocate DecoderImpl.");
+    return kLibgav1StatusOutOfMemory;
+  }
+  const StatusCode status = impl->Init();
+  if (status != kLibgav1StatusOk) return status;
+  *output = std::move(impl);
+  return kLibgav1StatusOk;
+}
+
 DecoderImpl::DecoderImpl(const DecoderSettings* settings)
     : buffer_pool_(*settings), settings_(*settings) {
   dsp::DspInit();
@@ -66,17 +84,27 @@
   }
 }
 
+StatusCode DecoderImpl::Init() {
+  const int max_allowed_frames =
+      settings_.frame_parallel ? settings_.threads : 1;
+  assert(max_allowed_frames > 0);
+  if (!encoded_frames_.Init(max_allowed_frames)) {
+    LIBGAV1_DLOG(ERROR, "encoded_frames_.Init() failed.");
+    return kLibgav1StatusOutOfMemory;
+  }
+  return kLibgav1StatusOk;
+}
+
 StatusCode DecoderImpl::EnqueueFrame(const uint8_t* data, size_t size,
                                      int64_t user_private_data) {
   if (data == nullptr) {
     // This has to actually flush the decoder.
     return kLibgav1StatusOk;
   }
-  int max_allowed_frames = settings_.frame_parallel ? settings_.threads : 1;
-  if (encoded_frames_.size() >= static_cast<size_t>(max_allowed_frames)) {
+  if (encoded_frames_.Full()) {
     return kLibgav1StatusResourceExhausted;
   }
-  encoded_frames_.emplace_back(data, size, user_private_data);
+  encoded_frames_.Push(EncodedFrame(data, size, user_private_data));
   return kLibgav1StatusOk;
 }
 
@@ -96,19 +124,21 @@
   // We assume a call to DequeueFrame() indicates that the caller is no longer
   // using the previous output frame, so we can release it.
   ReleaseOutputFrame();
-  if (encoded_frames_.empty()) {
+  if (encoded_frames_.Empty()) {
     // No encoded frame to decode. Not an error.
     *out_ptr = nullptr;
     return kLibgav1StatusOk;
   }
-  const EncodedFrame encoded_frame = encoded_frames_[0];
-  encoded_frames_.erase(encoded_frames_.begin());
+  const EncodedFrame encoded_frame = encoded_frames_.Pop();
   std::unique_ptr<ObuParser> obu(new (std::nothrow) ObuParser(
       encoded_frame.data, encoded_frame.size, &state_));
   if (obu == nullptr) {
     LIBGAV1_DLOG(ERROR, "Failed to initialize OBU parser.");
     return kLibgav1StatusOutOfMemory;
   }
+  if (state_.has_sequence_header) {
+    obu->set_sequence_header(state_.sequence_header);
+  }
   RefCountedBufferPtrCleanup current_frame_cleanup(&state_.current_frame);
   RefCountedBufferPtr displayable_frame;
   StatusCode status;
@@ -119,14 +149,16 @@
       return kLibgav1StatusResourceExhausted;
     }
 
-    obu->set_sequence_header(state_.sequence_header);
     if (!obu->ParseOneFrame()) {
       LIBGAV1_DLOG(ERROR, "Failed to parse OBU.");
       return kLibgav1StatusUnknownError;
     }
-    if (std::find(obu->types().begin(), obu->types().end(),
-                  kObuSequenceHeader) != obu->types().end()) {
+    if (std::find_if(obu->obu_headers().begin(), obu->obu_headers().end(),
+                     [](const ObuHeader& obu_header) {
+                       return obu_header.type == kObuSequenceHeader;
+                     }) != obu->obu_headers().end()) {
       state_.sequence_header = obu->sequence_header();
+      state_.has_sequence_header = true;
     }
     if (!obu->frame_header().show_existing_frame) {
       if (obu->tile_groups().empty()) {
@@ -355,6 +387,9 @@
       obu->frame_header().rows4x4 + kMaxBlockHeight4x4,
       obu->frame_header().columns4x4 + kMaxBlockWidth4x4,
       obu->sequence_header().use_128x128_superblock);
+  if (!block_parameters_holder.Init()) {
+    return kLibgav1StatusOutOfMemory;
+  }
   const dsp::Dsp* const dsp =
       dsp::GetDspTable(obu->sequence_header().color_config.bitdepth);
   if (dsp == nullptr) {
@@ -384,8 +419,11 @@
   const uint8_t tile_size_bytes = obu->frame_header().tile_info.tile_size_bytes;
   const int tile_count = obu->tile_groups().back().end + 1;
   assert(tile_count >= 1);
-  std::vector<std::unique_ptr<Tile>> tiles;
-  tiles.reserve(tile_count);
+  Vector<std::unique_ptr<Tile>> tiles;
+  if (!tiles.reserve(tile_count)) {
+    LIBGAV1_DLOG(ERROR, "tiles.reserve(%d) failed.\n", tile_count);
+    return kLibgav1StatusOutOfMemory;
+  }
   if (!threading_strategy_.Reset(obu->frame_header(), settings_.threads)) {
     return kLibgav1StatusOutOfMemory;
   }
@@ -412,31 +450,60 @@
     }
   }
 
+  const bool do_cdef =
+      PostFilter::DoCdef(obu->frame_header(), settings_.post_filter_mask);
+  const int num_planes = obu->sequence_header().color_config.is_monochrome
+                             ? kMaxPlanesMonochrome
+                             : kMaxPlanes;
+  const bool do_restoration =
+      PostFilter::DoRestoration(obu->frame_header().loop_restoration,
+                                settings_.post_filter_mask, num_planes);
   if (threading_strategy_.post_filter_thread_pool() != nullptr &&
-      PostFilter::DoRestoration(
-          obu->frame_header().loop_restoration, settings_.post_filter_mask,
-          obu->sequence_header().color_config.is_monochrome
-              ? kMaxPlanesMonochrome
-              : kMaxPlanes)) {
-    const size_t threaded_loop_restoration_buffer_size =
-        PostFilter::kRestorationWindowWidth *
-        PostFilter::GetRestorationWindowHeight(
+      (do_cdef || do_restoration)) {
+    const int window_buffer_width = PostFilter::GetWindowBufferWidth(
+        threading_strategy_.post_filter_thread_pool(), obu->frame_header());
+    size_t threaded_window_buffer_size =
+        window_buffer_width *
+        PostFilter::GetWindowBufferHeight(
             threading_strategy_.post_filter_thread_pool(),
             obu->frame_header()) *
         (obu->sequence_header().color_config.bitdepth == 8 ? sizeof(uint8_t)
                                                            : sizeof(uint16_t));
-    if (threaded_loop_restoration_buffer_size_ <
-        threaded_loop_restoration_buffer_size) {
-      threaded_loop_restoration_buffer_.reset(
-          new (std::nothrow) uint8_t[threaded_loop_restoration_buffer_size]);
-      if (threaded_loop_restoration_buffer_ == nullptr) {
+    if (do_cdef && !do_restoration) {
+      // TODO(chengchen): for cdef U, V planes, if there's subsampling, we can
+      // use smaller buffer.
+      threaded_window_buffer_size *= num_planes;
+    }
+    if (threaded_window_buffer_size_ < threaded_window_buffer_size) {
+      // threaded_window_buffer_ will be subdivided by PostFilter into windows
+      // of width 512 pixels. Each row in the window is filtered by a worker
+      // thread. To avoid false sharing, each 512-pixel row processed by one
+      // thread should not share a cache line with a row processed by another
+      // thread. So we align threaded_window_buffer_ to the cache line size.
+      // In addition, it is faster to memcpy from an aligned buffer.
+      //
+      // On Linux, the cache line size can be looked up with the command:
+      //   getconf LEVEL1_DCACHE_LINESIZE
+      //
+      // The cache line size should ideally be queried at run time. 64 is a
+      // common cache line size of x86 CPUs. Web searches showed the cache line
+      // size of ARM CPUs is 32 or 64 bytes. So aligning to 64-byte boundary
+      // will work for all CPUs that we care about, even though it is excessive
+      // for some ARM CPUs.
+      constexpr size_t kCacheLineSize = 64;
+      // To avoid false sharing, PostFilter's window width in bytes should also
+      // be a multiple of the cache line size. For simplicity, we check the
+      // window width in pixels.
+      assert(window_buffer_width % kCacheLineSize == 0);
+      threaded_window_buffer_ = MakeAlignedUniquePtr<uint8_t>(
+          kCacheLineSize, threaded_window_buffer_size);
+      if (threaded_window_buffer_ == nullptr) {
         LIBGAV1_DLOG(ERROR,
                      "Failed to allocate threaded loop restoration buffer.\n");
-        threaded_loop_restoration_buffer_size_ = 0;
+        threaded_window_buffer_size_ = 0;
         return kLibgav1StatusOutOfMemory;
       }
-      threaded_loop_restoration_buffer_size_ =
-          threaded_loop_restoration_buffer_size;
+      threaded_window_buffer_size_ = threaded_window_buffer_size;
     }
   }
 
@@ -445,9 +512,10 @@
       cdef_index, &loop_restoration_info, &block_parameters_holder,
       state_.current_frame->buffer(), dsp,
       threading_strategy_.post_filter_thread_pool(),
-      threaded_loop_restoration_buffer_.get(), settings_.post_filter_mask);
+      threaded_window_buffer_.get(), settings_.post_filter_mask);
   SymbolDecoderContext saved_symbol_decoder_context;
   int tile_index = 0;
+  BlockingCounterWithStatus pending_tiles(tile_count);
   for (const auto& tile_group : obu->tile_groups()) {
     size_t bytes_left = tile_group.data_size;
     size_t byte_offset = 0;
@@ -484,86 +552,91 @@
           prev_segment_ids, &post_filter, &block_parameters_holder, &cdef_index,
           &inter_transform_sizes_, dsp,
           threading_strategy_.row_thread_pool(tile_index++),
-          residual_buffer_pool_.get()));
+          residual_buffer_pool_.get(), &pending_tiles));
       if (tile == nullptr) {
         LIBGAV1_DLOG(ERROR, "Failed to allocate tile.");
         return kLibgav1StatusOutOfMemory;
       }
-      tiles.push_back(std::move(tile));
+      tiles.push_back_unchecked(std::move(tile));
 
       byte_offset += tile_size;
       bytes_left -= tile_size;
     }
   }
   assert(tiles.size() == static_cast<size_t>(tile_count));
+  bool tile_decoding_failed = false;
   if (threading_strategy_.tile_thread_pool() == nullptr) {
     for (const auto& tile_ptr : tiles) {
-      if (!tile_ptr->Decode()) {
-        LIBGAV1_DLOG(ERROR, "Error decoding tile #%d", tile_ptr->number());
-        return kLibgav1StatusUnknownError;
+      if (!tile_decoding_failed) {
+        if (!tile_ptr->Decode(/*is_main_thread=*/true)) {
+          LIBGAV1_DLOG(ERROR, "Error decoding tile #%d", tile_ptr->number());
+          tile_decoding_failed = true;
+        }
+      } else {
+        pending_tiles.Decrement(false);
       }
     }
   } else {
-    const int num_workers =
-        threading_strategy_.tile_thread_pool()->num_threads();
-    const int tiles_for_thread_pool =
-        tile_count * num_workers / (num_workers + 1);
-    // Make sure the current thread does some work.
-    assert(tiles_for_thread_pool < tile_count);
-    std::mutex mutex;
-    bool tile_decoding_failed = false;          // Guarded by |mutex|.
-    int pending_tiles = tiles_for_thread_pool;  // Guarded by |mutex|.
-    std::condition_variable pending_tiles_zero_condvar;
-    // Submit some tiles to the thread pool for decoding.
-    int i;
-    for (i = 0; i < tiles_for_thread_pool; ++i) {
-      auto* const tile = tiles[i].get();
+    const int num_workers = threading_strategy_.tile_thread_count();
+    BlockingCounterWithStatus pending_workers(num_workers);
+    std::atomic<int> tile_counter(0);
+    // Submit tile decoding jobs to the thread pool.
+    for (int i = 0; i < num_workers; ++i) {
       threading_strategy_.tile_thread_pool()->Schedule(
-          [&mutex, &tile_decoding_failed, tile, &pending_tiles,
-           &pending_tiles_zero_condvar]() {
-            const bool failed = !tile->Decode();
-            if (failed) {
-              LIBGAV1_DLOG(ERROR, "Error decoding tile #%d", tile->number());
+          [&tiles, tile_count, &tile_counter, &pending_workers,
+           &pending_tiles]() {
+            bool failed = false;
+            int index;
+            while ((index = tile_counter.fetch_add(
+                        1, std::memory_order_relaxed)) < tile_count) {
+              if (!failed) {
+                const auto& tile_ptr = tiles[index];
+                if (!tile_ptr->Decode(/*is_main_thread=*/false)) {
+                  LIBGAV1_DLOG(ERROR, "Error decoding tile #%d",
+                               tile_ptr->number());
+                  failed = true;
+                }
+              } else {
+                pending_tiles.Decrement(false);
+              }
             }
-            std::lock_guard<std::mutex> lock(mutex);
-            tile_decoding_failed |= failed;
-            if (--pending_tiles == 0) {
-              // TODO(jzern): the mutex doesn't need to be locked to signal the
-              // condition.
-              pending_tiles_zero_condvar.notify_one();
-            }
+            pending_workers.Decrement(!failed);
           });
     }
-    // Decode the rest of the tiles on the current thread.
-    bool failed = false;
-    for (; i < tile_count; ++i) {
-      const auto& tile_ptr = tiles[i];
-      if (!tile_ptr->Decode()) {
-        LIBGAV1_DLOG(ERROR, "Error decoding tile #%d", tile_ptr->number());
-        failed = true;
-        break;
+    // Have the current thread partake in tile decoding.
+    int index;
+    while ((index = tile_counter.fetch_add(1, std::memory_order_relaxed)) <
+           tile_count) {
+      if (!tile_decoding_failed) {
+        const auto& tile_ptr = tiles[index];
+        if (!tile_ptr->Decode(/*is_main_thread=*/true)) {
+          LIBGAV1_DLOG(ERROR, "Error decoding tile #%d", tile_ptr->number());
+          tile_decoding_failed = true;
+        }
+      } else {
+        pending_tiles.Decrement(false);
       }
     }
-    // Wait for the thread pool to finish decoding the tiles.
-    std::unique_lock<std::mutex> lock(mutex);
-    tile_decoding_failed |= failed;
-    while (pending_tiles != 0) {
-      pending_tiles_zero_condvar.wait(lock);
-    }
-    if (tile_decoding_failed) return kLibgav1StatusUnknownError;
+    // Wait until all the workers are done. This ensures that all the tiles have
+    // been parsed.
+    tile_decoding_failed |= !pending_workers.Wait();
   }
+  // Wait until all the tiles have been decoded.
+  tile_decoding_failed |= !pending_tiles.Wait();
+
+  // At this point, all the tiles have been parsed and decoded and the
+  // threadpool will be empty.
+  if (tile_decoding_failed) return kLibgav1StatusUnknownError;
 
   if (obu->frame_header().enable_frame_end_update_cdf) {
     symbol_decoder_context_ = saved_symbol_decoder_context;
   }
   state_.current_frame->SetFrameContext(symbol_decoder_context_);
-  if (post_filter.DoDeblock() &&
-      !loop_filter_mask_.Build(
-          obu->sequence_header(), obu->frame_header(),
-          obu->tile_groups().front().start, obu->tile_groups().back().end,
-          &block_parameters_holder, inter_transform_sizes_)) {
-    LIBGAV1_DLOG(ERROR, "Error building deblocking filter masks.");
-    return kLibgav1StatusUnknownError;
+  if (post_filter.DoDeblock()) {
+    loop_filter_mask_.Build(obu->sequence_header(), obu->frame_header(),
+                            obu->tile_groups().front().start,
+                            obu->tile_groups().back().end,
+                            block_parameters_holder, inter_transform_sizes_);
   }
   if (!post_filter.ApplyFiltering()) {
     LIBGAV1_DLOG(ERROR, "Error applying in-loop filtering.");
diff --git a/libgav1/src/decoder_impl.h b/libgav1/src/decoder_impl.h
index 0920661..c4ea526 100644
--- a/libgav1/src/decoder_impl.h
+++ b/libgav1/src/decoder_impl.h
@@ -5,7 +5,6 @@
 #include <cstddef>
 #include <cstdint>
 #include <memory>
-#include <vector>
 
 #include "src/buffer_pool.h"
 #include "src/decoder_buffer.h"
@@ -22,12 +21,17 @@
 #include "src/utils/block_parameters_holder.h"
 #include "src/utils/constants.h"
 #include "src/utils/memory.h"
+#include "src/utils/queue.h"
 #include "src/utils/segmentation_map.h"
 #include "src/utils/types.h"
 
 namespace libgav1 {
 
-struct EncodedFrame {
+struct EncodedFrame : public Allocable {
+  // The default constructor is invoked by the Queue<EncodedFrame>::Init()
+  // method. Queue<> does not use the default-constructed elements, so it is
+  // safe for the default constructor to not initialize the members.
+  EncodedFrame() = default;
   EncodedFrame(const uint8_t* data, size_t size, int64_t user_private_data)
       : data(data), size(size), user_private_data(user_private_data) {}
 
@@ -38,6 +42,8 @@
 
 struct DecoderState {
   ObuSequenceHeader sequence_header = {};
+  // If true, sequence_header is valid.
+  bool has_sequence_header = false;
   // reference_valid and reference_frame_id are used only if
   // sequence_header_.frame_id_numbers_present is true.
   // The reference_valid array is indexed by a reference picture slot number.
@@ -82,14 +88,26 @@
 class DecoderImpl : public Allocable {
  public:
   // The constructor saves a const reference to |*settings|. Therefore
-  // |*settings| must outlive the DecoderImpl object.
-  explicit DecoderImpl(const DecoderSettings* settings);
+  // |*settings| must outlive the DecoderImpl object. On success, |*output|
+  // contains a pointer to the newly-created DecoderImpl object. On failure,
+  // |*output| is not modified.
+  static StatusCode Create(const DecoderSettings* settings,
+                           std::unique_ptr<DecoderImpl>* output);
   ~DecoderImpl();
   StatusCode EnqueueFrame(const uint8_t* data, size_t size,
                           int64_t user_private_data);
   StatusCode DequeueFrame(const DecoderBuffer** out_ptr);
+  static constexpr int GetMaxBitdepth() {
+#if LIBGAV1_MAX_BITDEPTH >= 10
+    return 10;
+#else
+    return 8;
+#endif
+  }
 
  private:
+  explicit DecoderImpl(const DecoderSettings* settings);
+  StatusCode Init();
   bool AllocateCurrentFrame(const ObuFrameHeader& frame_header);
   void ReleaseOutputFrame();
   // Populates buffer_ with values from |frame|. Adds a reference to |frame|
@@ -104,7 +122,7 @@
   // state_.current_frame, based on the refresh_frame_flags bitmask.
   void UpdateReferenceFrames(int refresh_frame_flags);
 
-  std::vector<EncodedFrame> encoded_frames_;
+  Queue<EncodedFrame> encoded_frames_;
   DecoderState state_;
   ThreadingStrategy threading_strategy_;
   SymbolDecoderContext symbol_decoder_context_;
@@ -117,8 +135,8 @@
 
   BufferPool buffer_pool_;
   std::unique_ptr<ResidualBufferPool> residual_buffer_pool_;
-  std::unique_ptr<uint8_t[]> threaded_loop_restoration_buffer_;
-  size_t threaded_loop_restoration_buffer_size_ = 0;
+  AlignedUniquePtr<uint8_t> threaded_window_buffer_;
+  size_t threaded_window_buffer_size_ = 0;
   Array2D<TransformSize> inter_transform_sizes_;
 
   LoopFilterMask loop_filter_mask_;
diff --git a/libgav1/src/decoder_settings.h b/libgav1/src/decoder_settings.h
index 4a3adce..051e5bb 100644
--- a/libgav1/src/decoder_settings.h
+++ b/libgav1/src/decoder_settings.h
@@ -11,7 +11,10 @@
 
 // Applications must populate this structure before creating a decoder instance.
 struct DecoderSettings {
-  // Number of threads to use when decoding. Defaults to 1.
+  // Number of threads to use when decoding. Must be greater than 0. The
+  // library will create at most |threads|-1 new threads, the calling thread is
+  // considered part of the library's thread count. Defaults to 1 (no new
+  // threads will be created).
   int threads = 1;
   // Do frame parallel decoding.
   bool frame_parallel = false;
diff --git a/libgav1/src/dsp/arm/average_blend_neon.cc b/libgav1/src/dsp/arm/average_blend_neon.cc
new file mode 100644
index 0000000..5f9c458
--- /dev/null
+++ b/libgav1/src/dsp/arm/average_blend_neon.cc
@@ -0,0 +1,149 @@
+#include "src/dsp/average_blend.h"
+#include "src/dsp/dsp.h"
+
+#if LIBGAV1_ENABLE_NEON
+
+#include <arm_neon.h>
+
+#include <cassert>
+#include <cstddef>
+#include <cstdint>
+
+#include "src/dsp/arm/common_neon.h"
+#include "src/utils/common.h"
+
+namespace libgav1 {
+namespace dsp {
+namespace {
+
+constexpr int kBitdepth8 = 8;
+constexpr int kInterPostRoundBit = 4;
+// An offset to cancel offsets used in compound predictor generation that
+// make intermediate computations non negative.
+const int16x8_t kCompoundRoundOffset =
+    vdupq_n_s16((2 << (kBitdepth8 + 4)) + (2 << (kBitdepth8 + 3)));
+
+inline void AverageBlend4Row(const uint16_t* prediction_0,
+                             const uint16_t* prediction_1, uint8_t* dest) {
+  const int16x4_t pred0 = vreinterpret_s16_u16(vld1_u16(prediction_0));
+  const int16x4_t pred1 = vreinterpret_s16_u16(vld1_u16(prediction_1));
+  int16x4_t res = vadd_s16(pred0, pred1);
+  res = vsub_s16(res, vget_low_s16(kCompoundRoundOffset));
+  StoreLo4(dest,
+           vqrshrun_n_s16(vcombine_s16(res, res), kInterPostRoundBit + 1));
+}
+
+inline void AverageBlend8Row(const uint16_t* prediction_0,
+                             const uint16_t* prediction_1, uint8_t* dest) {
+  const int16x8_t pred0 = vreinterpretq_s16_u16(vld1q_u16(prediction_0));
+  const int16x8_t pred1 = vreinterpretq_s16_u16(vld1q_u16(prediction_1));
+  int16x8_t res = vaddq_s16(pred0, pred1);
+  res = vsubq_s16(res, kCompoundRoundOffset);
+  vst1_u8(dest, vqrshrun_n_s16(res, kInterPostRoundBit + 1));
+}
+
+inline void AverageBlendLargeRow(const uint16_t* prediction_0,
+                                 const uint16_t* prediction_1, const int width,
+                                 uint8_t* dest) {
+  int x = 0;
+  do {
+    const int16x8_t pred_00 =
+        vreinterpretq_s16_u16(vld1q_u16(&prediction_0[x]));
+    const int16x8_t pred_01 =
+        vreinterpretq_s16_u16(vld1q_u16(&prediction_1[x]));
+    int16x8_t res0 = vaddq_s16(pred_00, pred_01);
+    res0 = vsubq_s16(res0, kCompoundRoundOffset);
+    const uint8x8_t res_out0 = vqrshrun_n_s16(res0, kInterPostRoundBit + 1);
+    const int16x8_t pred_10 =
+        vreinterpretq_s16_u16(vld1q_u16(&prediction_0[x + 8]));
+    const int16x8_t pred_11 =
+        vreinterpretq_s16_u16(vld1q_u16(&prediction_1[x + 8]));
+    int16x8_t res1 = vaddq_s16(pred_10, pred_11);
+    res1 = vsubq_s16(res1, kCompoundRoundOffset);
+    const uint8x8_t res_out1 = vqrshrun_n_s16(res1, kInterPostRoundBit + 1);
+    vst1q_u8(dest + x, vcombine_u8(res_out0, res_out1));
+    x += 16;
+  } while (x < width);
+}
+
+void AverageBlend_NEON(const uint16_t* prediction_0,
+                       const ptrdiff_t prediction_stride_0,
+                       const uint16_t* prediction_1,
+                       const ptrdiff_t prediction_stride_1, const int width,
+                       const int height, void* const dest,
+                       const ptrdiff_t dest_stride) {
+  auto* dst = static_cast<uint8_t*>(dest);
+  int y = height;
+
+  if (width == 4) {
+    do {
+      AverageBlend4Row(prediction_0, prediction_1, dst);
+      dst += dest_stride;
+      prediction_0 += prediction_stride_0;
+      prediction_1 += prediction_stride_1;
+
+      AverageBlend4Row(prediction_0, prediction_1, dst);
+      dst += dest_stride;
+      prediction_0 += prediction_stride_0;
+      prediction_1 += prediction_stride_1;
+
+      y -= 2;
+    } while (y != 0);
+    return;
+  }
+
+  if (width == 8) {
+    do {
+      AverageBlend8Row(prediction_0, prediction_1, dst);
+      dst += dest_stride;
+      prediction_0 += prediction_stride_0;
+      prediction_1 += prediction_stride_1;
+
+      AverageBlend8Row(prediction_0, prediction_1, dst);
+      dst += dest_stride;
+      prediction_0 += prediction_stride_0;
+      prediction_1 += prediction_stride_1;
+
+      y -= 2;
+    } while (y != 0);
+    return;
+  }
+
+  do {
+    AverageBlendLargeRow(prediction_0, prediction_1, width, dst);
+    dst += dest_stride;
+    prediction_0 += prediction_stride_0;
+    prediction_1 += prediction_stride_1;
+
+    AverageBlendLargeRow(prediction_0, prediction_1, width, dst);
+    dst += dest_stride;
+    prediction_0 += prediction_stride_0;
+    prediction_1 += prediction_stride_1;
+
+    y -= 2;
+  } while (y != 0);
+}
+
+void Init8bpp() {
+  Dsp* const dsp = dsp_internal::GetWritableDspTable(8);
+  assert(dsp != nullptr);
+  dsp->average_blend = AverageBlend_NEON;
+}
+
+}  // namespace
+
+void AverageBlendInit_NEON() { Init8bpp(); }
+
+}  // namespace dsp
+}  // namespace libgav1
+
+#else   // !LIBGAV1_ENABLE_NEON
+
+namespace libgav1 {
+namespace dsp {
+
+void AverageBlendInit_NEON() {}
+
+}  // namespace dsp
+}  // namespace libgav1
+#endif  // LIBGAV1_ENABLE_NEON
diff --git a/libgav1/src/dsp/arm/average_blend_neon.h b/libgav1/src/dsp/arm/average_blend_neon.h
new file mode 100644
index 0000000..1225975
--- /dev/null
+++ b/libgav1/src/dsp/arm/average_blend_neon.h
@@ -0,0 +1,20 @@
+#ifndef LIBGAV1_SRC_DSP_ARM_AVERAGE_BLEND_NEON_H_
+#define LIBGAV1_SRC_DSP_ARM_AVERAGE_BLEND_NEON_H_
+
+#include "src/dsp/cpu.h"
+#include "src/dsp/dsp.h"
+
+namespace libgav1 {
+namespace dsp {
+
+// Initializes Dsp::average_blend. This function is not thread-safe.
+void AverageBlendInit_NEON();
+
+}  // namespace dsp
+}  // namespace libgav1
+
+#if LIBGAV1_ENABLE_NEON
+#define LIBGAV1_Dsp8bpp_AverageBlend LIBGAV1_DSP_NEON
+#endif  // LIBGAV1_ENABLE_NEON
+
+#endif  // LIBGAV1_SRC_DSP_ARM_AVERAGE_BLEND_NEON_H_
diff --git a/libgav1/src/dsp/arm/common_neon.h b/libgav1/src/dsp/arm/common_neon.h
index 6a28526..2ba35ed 100644
--- a/libgav1/src/dsp/arm/common_neon.h
+++ b/libgav1/src/dsp/arm/common_neon.h
@@ -10,6 +10,154 @@
 #include <cstdint>
 #include <cstring>
 
+#if 0
+#include <cstdio>
+
+constexpr bool kEnablePrintRegs = true;
+
+union DebugRegister {
+  int8_t i8[8];
+  int16_t i16[4];
+  int32_t i32[2];
+  uint8_t u8[8];
+  uint16_t u16[4];
+  uint32_t u32[2];
+};
+
+union DebugRegisterQ {
+  int8_t i8[16];
+  int16_t i16[8];
+  int32_t i32[4];
+  uint8_t u8[16];
+  uint16_t u16[8];
+  uint32_t u32[4];
+};
+
+// Quite useful macro for debugging. Left here for convenience.
+inline void PrintVect(const DebugRegister r, const char* const name, int size) {
+  int n;
+  if (kEnablePrintRegs) {
+    fprintf(stderr, "%s\t: ", name);
+    if (size == 8) {
+      for (n = 0; n < 8; ++n) fprintf(stderr, "%.2x ", r.u8[n]);
+    } else if (size == 16) {
+      for (n = 0; n < 4; ++n) fprintf(stderr, "%.4x ", r.u16[n]);
+    } else if (size == 32) {
+      for (n = 0; n < 2; ++n) fprintf(stderr, "%.8x ", r.u32[n]);
+    }
+    fprintf(stderr, "\n");
+  }
+}
+
+// Debugging macro for 128-bit types.
+inline void PrintVectQ(const DebugRegisterQ r, const char* const name,
+                       int size) {
+  int n;
+  if (kEnablePrintRegs) {
+    fprintf(stderr, "%s\t: ", name);
+    if (size == 8) {
+      for (n = 0; n < 16; ++n) fprintf(stderr, "%.2x ", r.u8[n]);
+    } else if (size == 16) {
+      for (n = 0; n < 8; ++n) fprintf(stderr, "%.4x ", r.u16[n]);
+    } else if (size == 32) {
+      for (n = 0; n < 4; ++n) fprintf(stderr, "%.8x ", r.u32[n]);
+    }
+    fprintf(stderr, "\n");
+  }
+}
+
+inline void PrintReg(const uint32x4_t val, const char* name) {
+  DebugRegisterQ r;
+  vst1q_u32(r.u32, val);
+  PrintVectQ(r, name, 32);
+}
+
+inline void PrintReg(const uint32x2_t val, const char* name) {
+  DebugRegister r;
+  vst1_u32(r.u32, val);
+  PrintVect(r, name, 32);
+}
+
+inline void PrintReg(const uint16x8_t val, const char* name) {
+  DebugRegisterQ r;
+  vst1q_u16(r.u16, val);
+  PrintVectQ(r, name, 16);
+}
+
+inline void PrintReg(const uint16x4_t val, const char* name) {
+  DebugRegister r;
+  vst1_u16(r.u16, val);
+  PrintVect(r, name, 16);
+}
+
+inline void PrintReg(const uint8x16_t val, const char* name) {
+  DebugRegisterQ r;
+  vst1q_u8(r.u8, val);
+  PrintVectQ(r, name, 8);
+}
+
+inline void PrintReg(const uint8x8_t val, const char* name) {
+  DebugRegister r;
+  vst1_u8(r.u8, val);
+  PrintVect(r, name, 8);
+}
+
+inline void PrintReg(const int32x4_t val, const char* name) {
+  DebugRegisterQ r;
+  vst1q_s32(r.i32, val);
+  PrintVectQ(r, name, 32);
+}
+
+inline void PrintReg(const int32x2_t val, const char* name) {
+  DebugRegister r;
+  vst1_s32(r.i32, val);
+  PrintVect(r, name, 32);
+}
+
+inline void PrintReg(const int16x8_t val, const char* name) {
+  DebugRegisterQ r;
+  vst1q_s16(r.i16, val);
+  PrintVectQ(r, name, 16);
+}
+
+inline void PrintReg(const int16x4_t val, const char* name) {
+  DebugRegister r;
+  vst1_s16(r.i16, val);
+  PrintVect(r, name, 16);
+}
+
+inline void PrintReg(const int8x16_t val, const char* name) {
+  DebugRegisterQ r;
+  vst1q_s8(r.i8, val);
+  PrintVectQ(r, name, 8);
+}
+
+inline void PrintReg(const int8x8_t val, const char* name) {
+  DebugRegister r;
+  vst1_s8(r.i8, val);
+  PrintVect(r, name, 8);
+}
+
+// Print an individual (non-vector) value in decimal format.
+inline void PrintReg(const int x, const char* name) {
+  if (kEnablePrintRegs) {
+    printf("%s: %d\n", name, x);
+  }
+}
+
+// Print an individual (non-vector) value in hexadecimal format.
+inline void PrintHex(const int x, const char* name) {
+  if (kEnablePrintRegs) {
+    printf("%s: %x\n", name, x);
+  }
+}
+
+#define PR(x) PrintReg(x, #x)
+#define PD(x) PrintReg(x, #x)
+#define PX(x) PrintHex(x, #x)
+
+#endif  // 0
+
 namespace libgav1 {
 namespace dsp {
 
@@ -64,9 +212,97 @@
   return vreinterpret_s8_u64(vshr_n_u64(vreinterpret_u64_s8(vector), shift));
 }
 
+// Shim vqtbl1_u8 for armv7.
+inline uint8x8_t VQTbl1U8(const uint8x16_t a, const uint8x8_t index) {
+#if defined(__aarch64__)
+  return vqtbl1_u8(a, index);
+#else
+  const uint8x8x2_t b = {vget_low_u8(a), vget_high_u8(a)};
+  return vtbl2_u8(b, index);
+#endif
+}
+
+//------------------------------------------------------------------------------
+// Interleave.
+
+// vzipN is exclusive to A64.
+inline uint8x8_t InterleaveLow8(const uint8x8_t a, const uint8x8_t b) {
+#if defined(__aarch64__)
+  return vzip1_u8(a, b);
+#else
+  // Discard |.val[1]|
+  return vzip_u8(a, b).val[0];
+#endif
+}
+
+inline uint8x8_t InterleaveLow32(const uint8x8_t a, const uint8x8_t b) {
+#if defined(__aarch64__)
+  return vreinterpret_u8_u32(
+      vzip1_u32(vreinterpret_u32_u8(a), vreinterpret_u32_u8(b)));
+#else
+  // Discard |.val[1]|
+  return vreinterpret_u8_u32(
+      vzip_u32(vreinterpret_u32_u8(a), vreinterpret_u32_u8(b)).val[0]);
+#endif
+}
+
+inline int8x8_t InterleaveLow32(const int8x8_t a, const int8x8_t b) {
+#if defined(__aarch64__)
+  return vreinterpret_s8_u32(
+      vzip1_u32(vreinterpret_u32_s8(a), vreinterpret_u32_s8(b)));
+#else
+  // Discard |.val[1]|
+  return vreinterpret_s8_u32(
+      vzip_u32(vreinterpret_u32_s8(a), vreinterpret_u32_s8(b)).val[0]);
+#endif
+}
+
+inline uint8x8_t InterleaveHigh32(const uint8x8_t a, const uint8x8_t b) {
+#if defined(__aarch64__)
+  return vreinterpret_u8_u32(
+      vzip2_u32(vreinterpret_u32_u8(a), vreinterpret_u32_u8(b)));
+#else
+  // Discard |.val[0]|
+  return vreinterpret_u8_u32(
+      vzip_u32(vreinterpret_u32_u8(a), vreinterpret_u32_u8(b)).val[1]);
+#endif
+}
+
+inline int8x8_t InterleaveHigh32(const int8x8_t a, const int8x8_t b) {
+#if defined(__aarch64__)
+  return vreinterpret_s8_u32(
+      vzip2_u32(vreinterpret_u32_s8(a), vreinterpret_u32_s8(b)));
+#else
+  // Discard |.val[0]|
+  return vreinterpret_s8_u32(
+      vzip_u32(vreinterpret_u32_s8(a), vreinterpret_u32_s8(b)).val[1]);
+#endif
+}
+
 //------------------------------------------------------------------------------
 // Transpose.
 
+// Transpose 32 bit elements such that:
+// a: 00 01
+// b: 02 03
+// returns
+// val[0]: 00 02
+// val[1]: 01 03
+inline uint8x8x2_t Interleave32(const uint8x8_t a, const uint8x8_t b) {
+  const uint32x2_t a_32 = vreinterpret_u32_u8(a);
+  const uint32x2_t b_32 = vreinterpret_u32_u8(b);
+  const uint32x2x2_t c = vtrn_u32(a_32, b_32);
+  const uint8x8x2_t d = {vreinterpret_u8_u32(c.val[0]),
+                         vreinterpret_u8_u32(c.val[1])};
+  return d;
+}
+
+// Swap high and low 32 bit elements.
+inline uint8x8_t Transpose32(const uint8x8_t a) {
+  const uint32x2_t b = vrev64_u32(vreinterpret_u32_u8(a));
+  return vreinterpret_u8_u32(b);
+}
+
 // Implement vtrnq_s64().
 // Input:
 // a0: 00 01 02 03 04 05 06 07
@@ -83,6 +319,15 @@
   return b0;
 }
 
+inline uint16x8x2_t VtrnqU64(uint32x4_t a0, uint32x4_t a1) {
+  uint16x8x2_t b0;
+  b0.val[0] = vcombine_u16(vreinterpret_u16_u32(vget_low_u32(a0)),
+                           vreinterpret_u16_u32(vget_low_u32(a1)));
+  b0.val[1] = vcombine_u16(vreinterpret_u16_u32(vget_high_u32(a0)),
+                           vreinterpret_u16_u32(vget_high_u32(a1)));
+  return b0;
+}
+
 // Input:
 // a: 00 01 02 03 10 11 12 13
 // b: 20 21 22 23 30 31 32 33
@@ -129,6 +374,102 @@
 }
 
 // Input:
+// a[0]: 00 01 02 03 04 05 06 07
+// a[1]: 10 11 12 13 14 15 16 17
+// a[2]: 20 21 22 23 24 25 26 27
+// a[3]: 30 31 32 33 34 35 36 37
+// a[4]: 40 41 42 43 44 45 46 47
+// a[5]: 50 51 52 53 54 55 56 57
+// a[6]: 60 61 62 63 64 65 66 67
+// a[7]: 70 71 72 73 74 75 76 77
+
+// Output:
+// a[0]: 00 10 20 30 40 50 60 70
+// a[1]: 01 11 21 31 41 51 61 71
+// a[2]: 02 12 22 32 42 52 62 72
+// a[3]: 03 13 23 33 43 53 63 73
+// a[4]: 04 14 24 34 44 54 64 74
+// a[5]: 05 15 25 35 45 55 65 75
+// a[6]: 06 16 26 36 46 56 66 76
+// a[7]: 07 17 27 37 47 57 67 77
+inline void Transpose8x8(int8x8_t a[8]) {
+  // Swap 8 bit elements. Goes from:
+  // a[0]: 00 01 02 03 04 05 06 07
+  // a[1]: 10 11 12 13 14 15 16 17
+  // a[2]: 20 21 22 23 24 25 26 27
+  // a[3]: 30 31 32 33 34 35 36 37
+  // a[4]: 40 41 42 43 44 45 46 47
+  // a[5]: 50 51 52 53 54 55 56 57
+  // a[6]: 60 61 62 63 64 65 66 67
+  // a[7]: 70 71 72 73 74 75 76 77
+  // to:
+  // b0.val[0]: 00 10 02 12 04 14 06 16  40 50 42 52 44 54 46 56
+  // b0.val[1]: 01 11 03 13 05 15 07 17  41 51 43 53 45 55 47 57
+  // b1.val[0]: 20 30 22 32 24 34 26 36  60 70 62 72 64 74 66 76
+  // b1.val[1]: 21 31 23 33 25 35 27 37  61 71 63 73 65 75 67 77
+  const int8x16x2_t b0 =
+      vtrnq_s8(vcombine_s8(a[0], a[4]), vcombine_s8(a[1], a[5]));
+  const int8x16x2_t b1 =
+      vtrnq_s8(vcombine_s8(a[2], a[6]), vcombine_s8(a[3], a[7]));
+
+  // Swap 16 bit elements resulting in:
+  // c0.val[0]: 00 10 20 30 04 14 24 34  40 50 60 70 44 54 64 74
+  // c0.val[1]: 02 12 22 32 06 16 26 36  42 52 62 72 46 56 66 76
+  // c1.val[0]: 01 11 21 31 05 15 25 35  41 51 61 71 45 55 65 75
+  // c1.val[1]: 03 13 23 33 07 17 27 37  43 53 63 73 47 57 67 77
+  const int16x8x2_t c0 = vtrnq_s16(vreinterpretq_s16_s8(b0.val[0]),
+                                   vreinterpretq_s16_s8(b1.val[0]));
+  const int16x8x2_t c1 = vtrnq_s16(vreinterpretq_s16_s8(b0.val[1]),
+                                   vreinterpretq_s16_s8(b1.val[1]));
+
+  // Unzip 32 bit elements resulting in:
+  // d0.val[0]: 00 10 20 30 40 50 60 70  01 11 21 31 41 51 61 71
+  // d0.val[1]: 04 14 24 34 44 54 64 74  05 15 25 35 45 55 65 75
+  // d1.val[0]: 02 12 22 32 42 52 62 72  03 13 23 33 43 53 63 73
+  // d1.val[1]: 06 16 26 36 46 56 66 76  07 17 27 37 47 57 67 77
+  const int32x4x2_t d0 = vuzpq_s32(vreinterpretq_s32_s16(c0.val[0]),
+                                   vreinterpretq_s32_s16(c1.val[0]));
+  const int32x4x2_t d1 = vuzpq_s32(vreinterpretq_s32_s16(c0.val[1]),
+                                   vreinterpretq_s32_s16(c1.val[1]));
+
+  a[0] = vreinterpret_s8_s32(vget_low_s32(d0.val[0]));
+  a[1] = vreinterpret_s8_s32(vget_high_s32(d0.val[0]));
+  a[2] = vreinterpret_s8_s32(vget_low_s32(d1.val[0]));
+  a[3] = vreinterpret_s8_s32(vget_high_s32(d1.val[0]));
+  a[4] = vreinterpret_s8_s32(vget_low_s32(d0.val[1]));
+  a[5] = vreinterpret_s8_s32(vget_high_s32(d0.val[1]));
+  a[6] = vreinterpret_s8_s32(vget_low_s32(d1.val[1]));
+  a[7] = vreinterpret_s8_s32(vget_high_s32(d1.val[1]));
+}
+
+// Unsigned.
+inline void Transpose8x8(uint8x8_t a[8]) {
+  const uint8x16x2_t b0 =
+      vtrnq_u8(vcombine_u8(a[0], a[4]), vcombine_u8(a[1], a[5]));
+  const uint8x16x2_t b1 =
+      vtrnq_u8(vcombine_u8(a[2], a[6]), vcombine_u8(a[3], a[7]));
+
+  const uint16x8x2_t c0 = vtrnq_u16(vreinterpretq_u16_u8(b0.val[0]),
+                                    vreinterpretq_u16_u8(b1.val[0]));
+  const uint16x8x2_t c1 = vtrnq_u16(vreinterpretq_u16_u8(b0.val[1]),
+                                    vreinterpretq_u16_u8(b1.val[1]));
+
+  const uint32x4x2_t d0 = vuzpq_u32(vreinterpretq_u32_u16(c0.val[0]),
+                                    vreinterpretq_u32_u16(c1.val[0]));
+  const uint32x4x2_t d1 = vuzpq_u32(vreinterpretq_u32_u16(c0.val[1]),
+                                    vreinterpretq_u32_u16(c1.val[1]));
+
+  a[0] = vreinterpret_u8_u32(vget_low_u32(d0.val[0]));
+  a[1] = vreinterpret_u8_u32(vget_high_u32(d0.val[0]));
+  a[2] = vreinterpret_u8_u32(vget_low_u32(d1.val[0]));
+  a[3] = vreinterpret_u8_u32(vget_high_u32(d1.val[0]));
+  a[4] = vreinterpret_u8_u32(vget_low_u32(d0.val[1]));
+  a[5] = vreinterpret_u8_u32(vget_high_u32(d0.val[1]));
+  a[6] = vreinterpret_u8_u32(vget_low_u32(d1.val[1]));
+  a[7] = vreinterpret_u8_u32(vget_high_u32(d1.val[1]));
+}
+
+// Input:
 // a0: 00 01 02 03 04 05 06 07
 // a1: 10 11 12 13 14 15 16 17
 // a2: 20 21 22 23 24 25 26 27
@@ -180,6 +521,87 @@
 }
 
 // Input:
+// a0: 00 01 02 03 04 05 06 07
+// a1: 10 11 12 13 14 15 16 17
+// a2: 20 21 22 23 24 25 26 27
+// a3: 30 31 32 33 34 35 36 37
+// a4: 40 41 42 43 44 45 46 47
+// a5: 50 51 52 53 54 55 56 57
+// a6: 60 61 62 63 64 65 66 67
+// a7: 70 71 72 73 74 75 76 77
+
+// Output:
+// a0: 00 10 20 30 40 50 60 70
+// a1: 01 11 21 31 41 51 61 71
+// a2: 02 12 22 32 42 52 62 72
+// a3: 03 13 23 33 43 53 63 73
+// a4: 04 14 24 34 44 54 64 74
+// a5: 05 15 25 35 45 55 65 75
+// a6: 06 16 26 36 46 56 66 76
+// a7: 07 17 27 37 47 57 67 77
+// TODO(johannkoenig): Switch users of the above transpose to this one.
+inline void Transpose8x8(int16x8_t a[8]) {
+  const int16x8x2_t b0 = vtrnq_s16(a[0], a[1]);
+  const int16x8x2_t b1 = vtrnq_s16(a[2], a[3]);
+  const int16x8x2_t b2 = vtrnq_s16(a[4], a[5]);
+  const int16x8x2_t b3 = vtrnq_s16(a[6], a[7]);
+
+  const int32x4x2_t c0 = vtrnq_s32(vreinterpretq_s32_s16(b0.val[0]),
+                                   vreinterpretq_s32_s16(b1.val[0]));
+  const int32x4x2_t c1 = vtrnq_s32(vreinterpretq_s32_s16(b0.val[1]),
+                                   vreinterpretq_s32_s16(b1.val[1]));
+  const int32x4x2_t c2 = vtrnq_s32(vreinterpretq_s32_s16(b2.val[0]),
+                                   vreinterpretq_s32_s16(b3.val[0]));
+  const int32x4x2_t c3 = vtrnq_s32(vreinterpretq_s32_s16(b2.val[1]),
+                                   vreinterpretq_s32_s16(b3.val[1]));
+
+  const int16x8x2_t d0 = VtrnqS64(c0.val[0], c2.val[0]);
+  const int16x8x2_t d1 = VtrnqS64(c1.val[0], c3.val[0]);
+  const int16x8x2_t d2 = VtrnqS64(c0.val[1], c2.val[1]);
+  const int16x8x2_t d3 = VtrnqS64(c1.val[1], c3.val[1]);
+
+  a[0] = d0.val[0];
+  a[1] = d1.val[0];
+  a[2] = d2.val[0];
+  a[3] = d3.val[0];
+  a[4] = d0.val[1];
+  a[5] = d1.val[1];
+  a[6] = d2.val[1];
+  a[7] = d3.val[1];
+}
+
+// Unsigned.
+inline void Transpose8x8(uint16x8_t a[8]) {
+  const uint16x8x2_t b0 = vtrnq_u16(a[0], a[1]);
+  const uint16x8x2_t b1 = vtrnq_u16(a[2], a[3]);
+  const uint16x8x2_t b2 = vtrnq_u16(a[4], a[5]);
+  const uint16x8x2_t b3 = vtrnq_u16(a[6], a[7]);
+
+  const uint32x4x2_t c0 = vtrnq_u32(vreinterpretq_u32_u16(b0.val[0]),
+                                    vreinterpretq_u32_u16(b1.val[0]));
+  const uint32x4x2_t c1 = vtrnq_u32(vreinterpretq_u32_u16(b0.val[1]),
+                                    vreinterpretq_u32_u16(b1.val[1]));
+  const uint32x4x2_t c2 = vtrnq_u32(vreinterpretq_u32_u16(b2.val[0]),
+                                    vreinterpretq_u32_u16(b3.val[0]));
+  const uint32x4x2_t c3 = vtrnq_u32(vreinterpretq_u32_u16(b2.val[1]),
+                                    vreinterpretq_u32_u16(b3.val[1]));
+
+  const uint16x8x2_t d0 = VtrnqU64(c0.val[0], c2.val[0]);
+  const uint16x8x2_t d1 = VtrnqU64(c1.val[0], c3.val[0]);
+  const uint16x8x2_t d2 = VtrnqU64(c0.val[1], c2.val[1]);
+  const uint16x8x2_t d3 = VtrnqU64(c1.val[1], c3.val[1]);
+
+  a[0] = d0.val[0];
+  a[1] = d1.val[0];
+  a[2] = d2.val[0];
+  a[3] = d3.val[0];
+  a[4] = d0.val[1];
+  a[5] = d1.val[1];
+  a[6] = d2.val[1];
+  a[7] = d3.val[1];
+}
+
+// Input:
 // i0: 00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f
 // i1: 10 11 12 13 14 15 16 17 18 19 1a 1b 1c 1d 1e 1f
 // i2: 20 21 22 23 24 25 26 27 28 29 2a 2b 2c 2d 2e 2f
@@ -257,6 +679,49 @@
   *o15 = vget_high_u8(vreinterpretq_u8_u32(d3.val[1]));
 }
 
+// TODO(johannkoenig): Replace usage of the above transpose with this one.
+inline void Transpose16x8(const uint8x16_t input[8], uint8x8_t output[16]) {
+  const uint8x16x2_t b0 = vtrnq_u8(input[0], input[1]);
+  const uint8x16x2_t b1 = vtrnq_u8(input[2], input[3]);
+  const uint8x16x2_t b2 = vtrnq_u8(input[4], input[5]);
+  const uint8x16x2_t b3 = vtrnq_u8(input[6], input[7]);
+
+  const uint16x8x2_t c0 = vtrnq_u16(vreinterpretq_u16_u8(b0.val[0]),
+                                    vreinterpretq_u16_u8(b1.val[0]));
+  const uint16x8x2_t c1 = vtrnq_u16(vreinterpretq_u16_u8(b0.val[1]),
+                                    vreinterpretq_u16_u8(b1.val[1]));
+  const uint16x8x2_t c2 = vtrnq_u16(vreinterpretq_u16_u8(b2.val[0]),
+                                    vreinterpretq_u16_u8(b3.val[0]));
+  const uint16x8x2_t c3 = vtrnq_u16(vreinterpretq_u16_u8(b2.val[1]),
+                                    vreinterpretq_u16_u8(b3.val[1]));
+
+  const uint32x4x2_t d0 = vtrnq_u32(vreinterpretq_u32_u16(c0.val[0]),
+                                    vreinterpretq_u32_u16(c2.val[0]));
+  const uint32x4x2_t d1 = vtrnq_u32(vreinterpretq_u32_u16(c0.val[1]),
+                                    vreinterpretq_u32_u16(c2.val[1]));
+  const uint32x4x2_t d2 = vtrnq_u32(vreinterpretq_u32_u16(c1.val[0]),
+                                    vreinterpretq_u32_u16(c3.val[0]));
+  const uint32x4x2_t d3 = vtrnq_u32(vreinterpretq_u32_u16(c1.val[1]),
+                                    vreinterpretq_u32_u16(c3.val[1]));
+
+  output[0] = vget_low_u8(vreinterpretq_u8_u32(d0.val[0]));
+  output[1] = vget_low_u8(vreinterpretq_u8_u32(d2.val[0]));
+  output[2] = vget_low_u8(vreinterpretq_u8_u32(d1.val[0]));
+  output[3] = vget_low_u8(vreinterpretq_u8_u32(d3.val[0]));
+  output[4] = vget_low_u8(vreinterpretq_u8_u32(d0.val[1]));
+  output[5] = vget_low_u8(vreinterpretq_u8_u32(d2.val[1]));
+  output[6] = vget_low_u8(vreinterpretq_u8_u32(d1.val[1]));
+  output[7] = vget_low_u8(vreinterpretq_u8_u32(d3.val[1]));
+  output[8] = vget_high_u8(vreinterpretq_u8_u32(d0.val[0]));
+  output[9] = vget_high_u8(vreinterpretq_u8_u32(d2.val[0]));
+  output[10] = vget_high_u8(vreinterpretq_u8_u32(d1.val[0]));
+  output[11] = vget_high_u8(vreinterpretq_u8_u32(d3.val[0]));
+  output[12] = vget_high_u8(vreinterpretq_u8_u32(d0.val[1]));
+  output[13] = vget_high_u8(vreinterpretq_u8_u32(d2.val[1]));
+  output[14] = vget_high_u8(vreinterpretq_u8_u32(d1.val[1]));
+  output[15] = vget_high_u8(vreinterpretq_u8_u32(d3.val[1]));
+}
+
 }  // namespace dsp
 }  // namespace libgav1
 
diff --git a/libgav1/src/dsp/arm/convolve_neon.cc b/libgav1/src/dsp/arm/convolve_neon.cc
new file mode 100644
index 0000000..4126c41
--- /dev/null
+++ b/libgav1/src/dsp/arm/convolve_neon.cc
@@ -0,0 +1,1694 @@
+#include "src/dsp/convolve.h"
+#include "src/dsp/dsp.h"
+
+#if LIBGAV1_ENABLE_NEON
+
+#include <arm_neon.h>
+
+#include <algorithm>
+#include <cassert>
+#include <cstddef>
+#include <cstdint>
+
+#include "src/dsp/arm/common_neon.h"
+#include "src/utils/common.h"
+
+namespace libgav1 {
+namespace dsp {
+namespace low_bitdepth {
+namespace {
+
+constexpr int kBitdepth8 = 8;
+constexpr int kIntermediateStride = kMaxSuperBlockSizeInPixels;
+constexpr int kSubPixelMask = (1 << kSubPixelBits) - 1;
+constexpr int kHorizontalOffset = 3;
+constexpr int kVerticalOffset = 3;
+constexpr int kInterRoundBitsVertical = 11;
+
+int GetFilterIndex(const int filter_index, const int length) {
+  if (length <= 4) {
+    if (filter_index == kInterpolationFilterEightTap ||
+        filter_index == kInterpolationFilterEightTapSharp) {
+      return 4;
+    }
+    if (filter_index == kInterpolationFilterEightTapSmooth) {
+      return 5;
+    }
+  }
+  return filter_index;
+}
+
+inline int16x8_t ZeroExtend(const uint8x8_t in) {
+  return vreinterpretq_s16_u16(vmovl_u8(in));
+}
+
+inline void Load8x8(const uint8_t* s, const ptrdiff_t p, int16x8_t* dst) {
+  dst[0] = ZeroExtend(vld1_u8(s));
+  s += p;
+  dst[1] = ZeroExtend(vld1_u8(s));
+  s += p;
+  dst[2] = ZeroExtend(vld1_u8(s));
+  s += p;
+  dst[3] = ZeroExtend(vld1_u8(s));
+  s += p;
+  dst[4] = ZeroExtend(vld1_u8(s));
+  s += p;
+  dst[5] = ZeroExtend(vld1_u8(s));
+  s += p;
+  dst[6] = ZeroExtend(vld1_u8(s));
+  s += p;
+  dst[7] = ZeroExtend(vld1_u8(s));
+}
+
+// Multiply every entry in |src[]| by the corresponding lane in |taps| and sum.
+// The sum of the entries in |taps| is always 128. In some situations negative
+// values are used. This creates a situation where the positive taps sum to more
+// than 128. An example is:
+// {-4, 10, -24, 100, 60, -20, 8, -2}
+// The negative taps never sum to < -128
+// The center taps are always positive. The remaining positive taps never sum
+// to > 128.
+// Summing these naively can overflow int16_t. This can be avoided by adding the
+// center taps last and saturating the result.
+// We do not need to expand to int32_t because later in the function the value
+// is shifted by |kFilterBits| (7) and saturated to uint8_t. This means any
+// value over 255 << 7 (32576 because of rounding) is clamped.
+template <int num_taps>
+int16x8_t SumTaps(const int16x8_t* const src, const int16x8_t taps) {
+  int16x8_t sum;
+  if (num_taps == 8) {
+    const int16x4_t taps_lo = vget_low_s16(taps);
+    const int16x4_t taps_hi = vget_high_s16(taps);
+    sum = vmulq_lane_s16(src[0], taps_lo, 0);
+    sum = vmlaq_lane_s16(sum, src[1], taps_lo, 1);
+    sum = vmlaq_lane_s16(sum, src[2], taps_lo, 2);
+    sum = vmlaq_lane_s16(sum, src[5], taps_hi, 1);
+    sum = vmlaq_lane_s16(sum, src[6], taps_hi, 2);
+    sum = vmlaq_lane_s16(sum, src[7], taps_hi, 3);
+
+    // Center taps.
+    sum = vqaddq_s16(sum, vmulq_lane_s16(src[3], taps_lo, 3));
+    sum = vqaddq_s16(sum, vmulq_lane_s16(src[4], taps_hi, 0));
+  } else if (num_taps == 6) {
+    const int16x4_t taps_lo = vget_low_s16(taps);
+    const int16x4_t taps_hi = vget_high_s16(taps);
+    sum = vmulq_lane_s16(src[0], taps_lo, 1);
+    sum = vmlaq_lane_s16(sum, src[1], taps_lo, 2);
+    sum = vmlaq_lane_s16(sum, src[4], taps_hi, 1);
+    sum = vmlaq_lane_s16(sum, src[5], taps_hi, 2);
+
+    // Center taps.
+    sum = vqaddq_s16(sum, vmulq_lane_s16(src[2], taps_lo, 3));
+    sum = vqaddq_s16(sum, vmulq_lane_s16(src[3], taps_hi, 0));
+  } else if (num_taps == 4) {
+    const int16x4_t taps_lo = vget_low_s16(taps);
+    sum = vmulq_lane_s16(src[0], taps_lo, 0);
+    sum = vmlaq_lane_s16(sum, src[3], taps_lo, 3);
+
+    // Center taps.
+    sum = vqaddq_s16(sum, vmulq_lane_s16(src[1], taps_lo, 1));
+    sum = vqaddq_s16(sum, vmulq_lane_s16(src[2], taps_lo, 2));
+  } else {
+    assert(num_taps == 2);
+    // All the taps are positive so there is no concern regarding saturation.
+    const int16x4_t taps_lo = vget_low_s16(taps);
+    sum = vmulq_lane_s16(src[0], taps_lo, 1);
+    sum = vmlaq_lane_s16(sum, src[1], taps_lo, 2);
+  }
+
+  return sum;
+}
+
+// Add an offset to ensure the sum is positive and it fits within uint16_t.
+template <int num_taps>
+uint16x8_t SumTaps8To16(const int16x8_t* const src, const int16x8_t taps) {
+  // The worst case sum of negative taps is -56. The worst case sum of positive
+  // taps is 184. With the single pass versions of the Convolve we could safely
+  // saturate to int16_t because it outranged the final shift and narrow to
+  // uint8_t. For the 2D Convolve the intermediate values are 16 bits so we
+  // don't have that option.
+  // 184 * 255 = 46920 which is greater than int16_t can hold, but not uint16_t.
+  // The minimum value we need to handle is -56 * 255 = -14280.
+  // By offsetting the sum with 1 << 14 = 16384 we ensure that the sum is never
+  // negative and that 46920 + 16384 = 63304 fits comfortably in uint16_t. This
+  // allows us to use 16 bit registers instead of 32 bit registers.
+  // When considering the bit operations it is safe to ignore signedness. Due to
+  // the magic of 2's complement and well defined rollover rules the bit
+  // representations are equivalent.
+  const int16x4_t taps_lo = vget_low_s16(taps);
+  const int16x4_t taps_hi = vget_high_s16(taps);
+  // |offset| == 1 << (bitdepth + kFilterBits - 1);
+  int16x8_t sum = vdupq_n_s16(1 << 14);
+  if (num_taps == 8) {
+    sum = vmlaq_lane_s16(sum, src[0], taps_lo, 0);
+    sum = vmlaq_lane_s16(sum, src[1], taps_lo, 1);
+    sum = vmlaq_lane_s16(sum, src[2], taps_lo, 2);
+    sum = vmlaq_lane_s16(sum, src[3], taps_lo, 3);
+    sum = vmlaq_lane_s16(sum, src[4], taps_hi, 0);
+    sum = vmlaq_lane_s16(sum, src[5], taps_hi, 1);
+    sum = vmlaq_lane_s16(sum, src[6], taps_hi, 2);
+    sum = vmlaq_lane_s16(sum, src[7], taps_hi, 3);
+  } else if (num_taps == 6) {
+    sum = vmlaq_lane_s16(sum, src[0], taps_lo, 1);
+    sum = vmlaq_lane_s16(sum, src[1], taps_lo, 2);
+    sum = vmlaq_lane_s16(sum, src[2], taps_lo, 3);
+    sum = vmlaq_lane_s16(sum, src[3], taps_hi, 0);
+    sum = vmlaq_lane_s16(sum, src[4], taps_hi, 1);
+    sum = vmlaq_lane_s16(sum, src[5], taps_hi, 2);
+  } else if (num_taps == 4) {
+    sum = vmlaq_lane_s16(sum, src[0], taps_lo, 2);
+    sum = vmlaq_lane_s16(sum, src[1], taps_lo, 3);
+    sum = vmlaq_lane_s16(sum, src[2], taps_hi, 0);
+    sum = vmlaq_lane_s16(sum, src[3], taps_hi, 1);
+  } else if (num_taps == 2) {
+    sum = vmlaq_lane_s16(sum, src[0], taps_lo, 3);
+    sum = vmlaq_lane_s16(sum, src[1], taps_hi, 0);
+  }
+
+  // This is guaranteed to be positive. Convert it for the final shift.
+  return vreinterpretq_u16_s16(sum);
+}
+
+// Process 16 bit inputs and output 32 bits.
+template <int num_taps>
+uint32x4x2_t Sum2DVerticalTaps(const int16x8_t* const src,
+                               const int16x8_t taps) {
+  // In order to get the rollover correct with the lengthening instruction we
+  // need to treat these as signed so that they sign extend properly.
+  const int16x4_t taps_lo = vget_low_s16(taps);
+  const int16x4_t taps_hi = vget_high_s16(taps);
+  // An offset to guarantee the sum is non negative. Captures 56 * -4590 =
+  // 257040 (worst case negative value from horizontal pass). It should be
+  // possible to use 1 << 18 (262144) instead of 1 << 19 but there probably
+  // isn't any benefit.
+  // |offset_bits| = bitdepth + 2 * kFilterBits - kInterRoundBitsHorizontal
+  // == 19.
+  int32x4_t sum_lo = vdupq_n_s32(1 << 19);
+  int32x4_t sum_hi = sum_lo;
+  if (num_taps == 8) {
+    sum_lo = vmlal_lane_s16(sum_lo, vget_low_s16(src[0]), taps_lo, 0);
+    sum_hi = vmlal_lane_s16(sum_hi, vget_high_s16(src[0]), taps_lo, 0);
+    sum_lo = vmlal_lane_s16(sum_lo, vget_low_s16(src[1]), taps_lo, 1);
+    sum_hi = vmlal_lane_s16(sum_hi, vget_high_s16(src[1]), taps_lo, 1);
+    sum_lo = vmlal_lane_s16(sum_lo, vget_low_s16(src[2]), taps_lo, 2);
+    sum_hi = vmlal_lane_s16(sum_hi, vget_high_s16(src[2]), taps_lo, 2);
+    sum_lo = vmlal_lane_s16(sum_lo, vget_low_s16(src[3]), taps_lo, 3);
+    sum_hi = vmlal_lane_s16(sum_hi, vget_high_s16(src[3]), taps_lo, 3);
+
+    sum_lo = vmlal_lane_s16(sum_lo, vget_low_s16(src[4]), taps_hi, 0);
+    sum_hi = vmlal_lane_s16(sum_hi, vget_high_s16(src[4]), taps_hi, 0);
+    sum_lo = vmlal_lane_s16(sum_lo, vget_low_s16(src[5]), taps_hi, 1);
+    sum_hi = vmlal_lane_s16(sum_hi, vget_high_s16(src[5]), taps_hi, 1);
+    sum_lo = vmlal_lane_s16(sum_lo, vget_low_s16(src[6]), taps_hi, 2);
+    sum_hi = vmlal_lane_s16(sum_hi, vget_high_s16(src[6]), taps_hi, 2);
+    sum_lo = vmlal_lane_s16(sum_lo, vget_low_s16(src[7]), taps_hi, 3);
+    sum_hi = vmlal_lane_s16(sum_hi, vget_high_s16(src[7]), taps_hi, 3);
+  } else if (num_taps == 6) {
+    sum_lo = vmlal_lane_s16(sum_lo, vget_low_s16(src[0]), taps_lo, 1);
+    sum_hi = vmlal_lane_s16(sum_hi, vget_high_s16(src[0]), taps_lo, 1);
+    sum_lo = vmlal_lane_s16(sum_lo, vget_low_s16(src[1]), taps_lo, 2);
+    sum_hi = vmlal_lane_s16(sum_hi, vget_high_s16(src[1]), taps_lo, 2);
+    sum_lo = vmlal_lane_s16(sum_lo, vget_low_s16(src[2]), taps_lo, 3);
+    sum_hi = vmlal_lane_s16(sum_hi, vget_high_s16(src[2]), taps_lo, 3);
+
+    sum_lo = vmlal_lane_s16(sum_lo, vget_low_s16(src[3]), taps_hi, 0);
+    sum_hi = vmlal_lane_s16(sum_hi, vget_high_s16(src[3]), taps_hi, 0);
+    sum_lo = vmlal_lane_s16(sum_lo, vget_low_s16(src[4]), taps_hi, 1);
+    sum_hi = vmlal_lane_s16(sum_hi, vget_high_s16(src[4]), taps_hi, 1);
+    sum_lo = vmlal_lane_s16(sum_lo, vget_low_s16(src[5]), taps_hi, 2);
+    sum_hi = vmlal_lane_s16(sum_hi, vget_high_s16(src[5]), taps_hi, 2);
+  } else if (num_taps == 4) {
+    sum_lo = vmlal_lane_s16(sum_lo, vget_low_s16(src[0]), taps_lo, 2);
+    sum_hi = vmlal_lane_s16(sum_hi, vget_high_s16(src[0]), taps_lo, 2);
+    sum_lo = vmlal_lane_s16(sum_lo, vget_low_s16(src[1]), taps_lo, 3);
+    sum_hi = vmlal_lane_s16(sum_hi, vget_high_s16(src[1]), taps_lo, 3);
+
+    sum_lo = vmlal_lane_s16(sum_lo, vget_low_s16(src[2]), taps_hi, 0);
+    sum_hi = vmlal_lane_s16(sum_hi, vget_high_s16(src[2]), taps_hi, 0);
+    sum_lo = vmlal_lane_s16(sum_lo, vget_low_s16(src[3]), taps_hi, 1);
+    sum_hi = vmlal_lane_s16(sum_hi, vget_high_s16(src[3]), taps_hi, 1);
+  } else if (num_taps == 2) {
+    sum_lo = vmlal_lane_s16(sum_lo, vget_low_s16(src[0]), taps_lo, 3);
+    sum_hi = vmlal_lane_s16(sum_hi, vget_high_s16(src[0]), taps_lo, 3);
+
+    sum_lo = vmlal_lane_s16(sum_lo, vget_low_s16(src[1]), taps_hi, 0);
+    sum_hi = vmlal_lane_s16(sum_hi, vget_high_s16(src[1]), taps_hi, 0);
+  }
+
+  // This is guaranteed to be positive. Convert it for the final shift.
+  const uint32x4x2_t return_val = {vreinterpretq_u32_s32(sum_lo),
+                                   vreinterpretq_u32_s32(sum_hi)};
+  return return_val;
+}
+
+template <int num_taps>
+void Filter2DVertical(const uint16_t* src, const ptrdiff_t src_stride,
+                      uint8_t* dst, const ptrdiff_t dst_stride, const int width,
+                      const int height, const int16x8_t taps) {
+  constexpr int next_row = num_taps - 1;
+
+  int x = 0;
+  do {
+    int16x8_t srcs[8];
+    srcs[0] = vreinterpretq_s16_u16(vld1q_u16(src + x));
+    if (num_taps >= 4) {
+      srcs[1] = vreinterpretq_s16_u16(vld1q_u16(src + x + src_stride));
+      srcs[2] = vreinterpretq_s16_u16(vld1q_u16(src + x + 2 * src_stride));
+      if (num_taps >= 6) {
+        srcs[3] = vreinterpretq_s16_u16(vld1q_u16(src + x + 3 * src_stride));
+        srcs[4] = vreinterpretq_s16_u16(vld1q_u16(src + x + 4 * src_stride));
+        if (num_taps == 8) {
+          srcs[5] = vreinterpretq_s16_u16(vld1q_u16(src + x + 5 * src_stride));
+          srcs[6] = vreinterpretq_s16_u16(vld1q_u16(src + x + 6 * src_stride));
+        }
+      }
+    }
+
+    int y = 0;
+    do {
+      srcs[next_row] = vreinterpretq_s16_u16(
+          vld1q_u16(src + x + (y + next_row) * src_stride));
+
+      const uint32x4x2_t sums = Sum2DVerticalTaps<num_taps>(srcs, taps);
+      const uint16x8_t first_shift =
+          vcombine_u16(vqrshrn_n_u32(sums.val[0], kInterRoundBitsVertical),
+                       vqrshrn_n_u32(sums.val[1], kInterRoundBitsVertical));
+      // |single_round_offset| == (1 << bitdepth) + (1 << (bitdepth - 1)) ==
+      // 384
+      const uint8x8_t results =
+          vqmovn_u16(vqsubq_u16(first_shift, vdupq_n_u16(384)));
+
+      vst1_u8(dst + x + y * dst_stride, results);
+
+      srcs[0] = srcs[1];
+      if (num_taps >= 4) {
+        srcs[1] = srcs[2];
+        srcs[2] = srcs[3];
+        if (num_taps >= 6) {
+          srcs[3] = srcs[4];
+          srcs[4] = srcs[5];
+          if (num_taps == 8) {
+            srcs[5] = srcs[6];
+            srcs[6] = srcs[7];
+          }
+        }
+      }
+    } while (++y < height);
+    x += 8;
+  } while (x < width);
+}
+
+void Convolve2D_NEON(const void* const reference,
+                     const ptrdiff_t reference_stride,
+                     const int horizontal_filter_index,
+                     const int vertical_filter_index,
+                     const uint8_t /*inter_round_bits_vertical*/,
+                     const int subpixel_x, const int subpixel_y,
+                     const int /*step_x*/, const int /*step_y*/,
+                     const int width, const int height, void* prediction,
+                     const ptrdiff_t pred_stride) {
+  const int horiz_filter_index = GetFilterIndex(horizontal_filter_index, width);
+  const int vert_filter_index = GetFilterIndex(vertical_filter_index, height);
+  int horizontal_taps, horizontal_taps_start, vertical_taps,
+      vertical_taps_start;
+
+  if (horiz_filter_index < 2) {
+    horizontal_taps = 6;
+    horizontal_taps_start = 1;
+  } else if (horiz_filter_index == 2) {
+    horizontal_taps = 8;
+    horizontal_taps_start = 0;
+  } else if (horiz_filter_index == 3) {
+    horizontal_taps = 2;
+    horizontal_taps_start = 3;
+  } else /* if (horiz_filter_index > 3) */ {
+    horizontal_taps = 4;
+    horizontal_taps_start = 2;
+  }
+
+  if (vert_filter_index < 2) {
+    vertical_taps = 6;
+    vertical_taps_start = 1;
+  } else if (vert_filter_index == 2) {
+    vertical_taps = 8;
+    vertical_taps_start = 0;
+  } else if (vert_filter_index == 3) {
+    vertical_taps = 2;
+    vertical_taps_start = 3;
+  } else /* if (vert_filter_index > 3) */ {
+    vertical_taps = 4;
+    vertical_taps_start = 2;
+  }
+
+  // Neon processes blocks of 8x8 for context during the horizontal pass so it
+  // still does a few more than it needs.
+  const int intermediate_height = height + vertical_taps - 1;
+  // The output of the horizontal filter is guaranteed to fit in 16 bits.
+  uint16_t intermediate_result[kMaxSuperBlockSizeInPixels *
+                               (kMaxSuperBlockSizeInPixels + kSubPixelTaps)];
+  const int intermediate_stride = width;
+  const int max_pixel_value = 255;
+
+  if (width > 4) {
+    // Horizontal filter.
+    const int horiz_filter_id = (subpixel_x >> 6) & kSubPixelMask;
+    const int16x8_t horiz_taps =
+        vld1q_s16(kSubPixelFilters[horiz_filter_index][horiz_filter_id]);
+
+    uint16_t* intermediate = intermediate_result;
+    const ptrdiff_t src_stride = reference_stride;
+    // Offset for 8 tap horizontal filter and |vertical_taps|.
+    const auto* src = static_cast<const uint8_t*>(reference) -
+                      ((vertical_taps / 2) - 1) * src_stride -
+                      kHorizontalOffset;
+    int y = 0;
+    do {
+      int x = 0;
+      do {
+        uint8x16_t temp[8];
+        uint8x8_t input[16];
+        for (int i = 0; i < 8; ++i) {
+          temp[i] = vld1q_u8(src + 0 + x + i * src_stride);
+        }
+        // TODO(johannkoenig): It should be possible to get the transpose
+        // started with vld2().
+        Transpose16x8(temp, input);
+        int16x8_t input16[16];
+        for (int i = 0; i < 16; ++i) {
+          input16[i] = ZeroExtend(input[i]);
+        }
+
+        // TODO(johannkoenig): Explore moving the branch outside the main loop.
+        uint16x8_t output[8];
+        if (horizontal_taps == 8) {
+          for (int i = 0; i < 8; ++i) {
+            const uint16x8_t neon_sums =
+                SumTaps8To16<8>(input16 + i, horiz_taps);
+            output[i] = vrshrq_n_u16(neon_sums, kInterRoundBitsHorizontal);
+          }
+        } else if (horizontal_taps == 6) {
+          for (int i = 0; i < 8; ++i) {
+            const uint16x8_t neon_sums =
+                SumTaps8To16<6>(input16 + i + 1, horiz_taps);
+            output[i] = vrshrq_n_u16(neon_sums, kInterRoundBitsHorizontal);
+          }
+        } else {  // |horizontal_taps| == 2
+          for (int i = 0; i < 8; ++i) {
+            const uint16x8_t neon_sums =
+                SumTaps8To16<2>(input16 + i + 3, horiz_taps);
+            output[i] = vrshrq_n_u16(neon_sums, kInterRoundBitsHorizontal);
+          }
+        }
+
+        Transpose8x8(output);
+        for (int i = 0; i < 8; ++i) {
+          vst1q_u16(intermediate + x + i * intermediate_stride, output[i]);
+        }
+        x += 8;
+      } while (x < width);
+      src += src_stride << 3;
+      intermediate += intermediate_stride << 3;
+      y += 8;
+    } while (y < intermediate_height);
+
+    // Vertical filter.
+    auto* dest = static_cast<uint8_t*>(prediction);
+    const ptrdiff_t dest_stride = pred_stride;
+    const int filter_id = ((subpixel_y & 1023) >> 6) & kSubPixelMask;
+    const int16x8_t taps =
+        vld1q_s16(kSubPixelFilters[vert_filter_index][filter_id]);
+
+    if (vertical_taps == 8) {
+      Filter2DVertical<8>(intermediate_result, intermediate_stride, dest,
+                          dest_stride, width, height, taps);
+    } else if (vertical_taps == 6) {
+      Filter2DVertical<6>(intermediate_result, intermediate_stride, dest,
+                          dest_stride, width, height, taps);
+    } else if (vertical_taps == 4) {
+      Filter2DVertical<4>(intermediate_result, intermediate_stride, dest,
+                          dest_stride, width, height, taps);
+    } else {  // |vertical_taps| == 2
+      Filter2DVertical<2>(intermediate_result, intermediate_stride, dest,
+                          dest_stride, width, height, taps);
+    }
+  } else {
+    // Horizontal filter.
+    // Filter types used for width <= 4 are different from those for width > 4.
+    // When width > 4, the valid filter index range is always [0, 3].
+    // When width <= 4, the valid filter index range is always [4, 5].
+    // Similarly for height.
+    uint16_t* intermediate = intermediate_result;
+    const ptrdiff_t src_stride = reference_stride;
+    const auto* src = static_cast<const uint8_t*>(reference) -
+                      ((vertical_taps / 2) - 1) * src_stride -
+                      ((horizontal_taps / 2) - 1);
+    auto* dest = static_cast<uint8_t*>(prediction);
+    const ptrdiff_t dest_stride = pred_stride;
+    int filter_id = (subpixel_x >> 6) & kSubPixelMask;
+    for (int y = 0; y < intermediate_height; ++y) {
+      for (int x = 0; x < width; ++x) {
+        // An offset to guarantee the sum is non negative.
+        int sum = 1 << 14;
+        for (int k = 0; k < horizontal_taps; ++k) {
+          const int tap = k + horizontal_taps_start;
+          sum +=
+              kSubPixelFilters[horiz_filter_index][filter_id][tap] * src[x + k];
+        }
+        intermediate[x] = static_cast<int16_t>(RightShiftWithRounding(sum, 3));
+      }
+      src += src_stride;
+      intermediate += intermediate_stride;
+    }
+    // Vertical filter.
+    intermediate = intermediate_result;
+    filter_id = ((subpixel_y & 1023) >> 6) & kSubPixelMask;
+    for (int y = 0; y < height; ++y) {
+      for (int x = 0; x < width; ++x) {
+        // An offset to guarantee the sum is non negative.
+        int sum = 1 << 19;
+        for (int k = 0; k < vertical_taps; ++k) {
+          const int tap = k + vertical_taps_start;
+          sum += kSubPixelFilters[vert_filter_index][filter_id][tap] *
+                 intermediate[k * intermediate_stride + x];
+        }
+        dest[x] = static_cast<uint8_t>(
+            Clip3(RightShiftWithRounding(sum, 11) - 384, 0, max_pixel_value));
+      }
+      dest += dest_stride;
+      intermediate += intermediate_stride;
+    }
+  }
+}
+
+template <int tap_lane0, int tap_lane1>
+inline int16x8_t CombineFilterTapsLong(const int16x8_t sum,
+                                       const int16x8_t src0, int16x8_t src1,
+                                       int16x4_t taps0, int16x4_t taps1) {
+  int32x4_t sum_lo = vmovl_s16(vget_low_s16(sum));
+  int32x4_t sum_hi = vmovl_s16(vget_high_s16(sum));
+  const int16x8_t product0 = vmulq_lane_s16(src0, taps0, tap_lane0);
+  const int16x8_t product1 = vmulq_lane_s16(src1, taps1, tap_lane1);
+  const int32x4_t center_vals_lo =
+      vaddl_s16(vget_low_s16(product0), vget_low_s16(product1));
+  const int32x4_t center_vals_hi =
+      vaddl_s16(vget_high_s16(product0), vget_high_s16(product1));
+
+  sum_lo = vaddq_s32(sum_lo, center_vals_lo);
+  sum_hi = vaddq_s32(sum_hi, center_vals_hi);
+  return vcombine_s16(vrshrn_n_s32(sum_lo, 3), vrshrn_n_s32(sum_hi, 3));
+}
+
+// TODO(b/133525024): Replace usage of this function with version that uses
+// unsigned trick, once cl/263050071 is submitted.
+template <int num_taps>
+inline int16x8_t SumTapsCompound(const int16x8_t* const src,
+                                 const int16x8_t taps) {
+  int16x8_t sum = vdupq_n_s16(1 << (kBitdepth8 + kFilterBits - 1));
+  if (num_taps == 8) {
+    const int16x4_t taps_lo = vget_low_s16(taps);
+    const int16x4_t taps_hi = vget_high_s16(taps);
+    sum = vmlaq_lane_s16(sum, src[0], taps_lo, 0);
+    sum = vmlaq_lane_s16(sum, src[1], taps_lo, 1);
+    sum = vmlaq_lane_s16(sum, src[2], taps_lo, 2);
+    sum = vmlaq_lane_s16(sum, src[5], taps_hi, 1);
+    sum = vmlaq_lane_s16(sum, src[6], taps_hi, 2);
+    sum = vmlaq_lane_s16(sum, src[7], taps_hi, 3);
+
+    // Center taps may sum to as much as 160, which pollutes the sign bit in
+    // int16 types.
+    sum = CombineFilterTapsLong<3, 0>(sum, src[3], src[4], taps_lo, taps_hi);
+  } else if (num_taps == 6) {
+    const int16x4_t taps_lo = vget_low_s16(taps);
+    const int16x4_t taps_hi = vget_high_s16(taps);
+    sum = vmlaq_lane_s16(sum, src[0], taps_lo, 0);
+    sum = vmlaq_lane_s16(sum, src[1], taps_lo, 1);
+    sum = vmlaq_lane_s16(sum, src[4], taps_hi, 0);
+    sum = vmlaq_lane_s16(sum, src[5], taps_hi, 1);
+
+    // Center taps in filter 0 may sum to as much as 148, which pollutes the
+    // sign bit in int16 types. This is not true of filter 1.
+    sum = CombineFilterTapsLong<2, 3>(sum, src[2], src[3], taps_lo, taps_lo);
+  } else if (num_taps == 4) {
+    const int16x4_t taps_lo = vget_low_s16(taps);
+    sum = vmlaq_lane_s16(sum, src[0], taps_lo, 0);
+    sum = vmlaq_lane_s16(sum, src[3], taps_lo, 3);
+
+    // Center taps.
+    sum = vqaddq_s16(sum, vmulq_lane_s16(src[1], taps_lo, 1));
+    sum = vrshrq_n_s16(vqaddq_s16(sum, vmulq_lane_s16(src[2], taps_lo, 2)),
+                       kInterRoundBitsHorizontal);
+  } else {
+    assert(num_taps == 2);
+    // All the taps are positive so there is no concern regarding saturation.
+    const int16x4_t taps_lo = vget_low_s16(taps);
+    sum = vmlaq_lane_s16(sum, src[0], taps_lo, 0);
+    sum = vrshrq_n_s16(vmlaq_lane_s16(sum, src[1], taps_lo, 1),
+                       kInterRoundBitsHorizontal);
+  }
+  return sum;
+}
+
+// |grade_x| determines an upper limit on how many whole-pixel steps will be
+// realized with 8 |step_x| increments.
+template <int filter_index, int num_taps, int grade_x>
+inline void ConvolveHorizontalScaled_NEON(const uint8_t* src,
+                                          const ptrdiff_t src_stride,
+                                          const int width, const int subpixel_x,
+                                          const int step_x,
+                                          const int intermediate_height,
+                                          int16_t* dst) {
+  const int dst_stride = kMaxSuperBlockSizeInPixels;
+  const int kernel_offset = (8 - num_taps) / 2;
+  const int ref_x = subpixel_x >> kScaleSubPixelBits;
+  int y = intermediate_height;
+  do {  // y > 0
+    int p = subpixel_x;
+    int prev_p = p;
+    int x = 0;
+    int16x8_t s[(grade_x + 1) * 8];
+    const uint8_t* src_x =
+        &src[(p >> kScaleSubPixelBits) - ref_x + kernel_offset];
+    Load8x8(src_x, src_stride, s);
+    Transpose8x8(s);
+    if (grade_x > 1) {
+      Load8x8(src_x + 8, src_stride, &s[8]);
+      Transpose8x8(&s[8]);
+    }
+
+    do {  // x < width
+      int16x8_t result[8];
+      src_x = &src[(p >> kScaleSubPixelBits) - ref_x + kernel_offset];
+      // process 8 src_x steps
+      Load8x8(src_x + 8, src_stride, &s[8]);
+      Transpose8x8(&s[8]);
+      if (grade_x > 1) {
+        Load8x8(src_x + 16, src_stride, &s[16]);
+        Transpose8x8(&s[16]);
+      }
+      // Remainder after whole index increments.
+      int pixel_offset = p & ((1 << kScaleSubPixelBits) - 1);
+      for (int z = 0; z < 8; ++z) {
+        const int16x8_t filter = vld1q_s16(
+            &kSubPixelFilters[filter_index][(p >> 6) & 0xF][kernel_offset]);
+        result[z] = SumTapsCompound<num_taps>(
+            &s[pixel_offset >> kScaleSubPixelBits], filter);
+        pixel_offset += step_x;
+        p += step_x;
+      }
+
+      // Transpose the 8x8 filtered values back to dst.
+      Transpose8x8(result);
+
+      vst1q_s16(&dst[x + 0 * dst_stride], result[0]);
+      vst1q_s16(&dst[x + 1 * dst_stride], result[1]);
+      vst1q_s16(&dst[x + 2 * dst_stride], result[2]);
+      vst1q_s16(&dst[x + 3 * dst_stride], result[3]);
+      vst1q_s16(&dst[x + 4 * dst_stride], result[4]);
+      vst1q_s16(&dst[x + 5 * dst_stride], result[5]);
+      vst1q_s16(&dst[x + 6 * dst_stride], result[6]);
+      vst1q_s16(&dst[x + 7 * dst_stride], result[7]);
+
+      for (int i = 0; i < 8; ++i) {
+        s[i] =
+            s[(p >> kScaleSubPixelBits) - (prev_p >> kScaleSubPixelBits) + i];
+        if (grade_x > 1) {
+          s[i + 8] = s[(p >> kScaleSubPixelBits) -
+                       (prev_p >> kScaleSubPixelBits) + i + 8];
+        }
+      }
+
+      prev_p = p;
+      x += 8;
+    } while (x < width);
+
+    src += src_stride * 8;
+    dst += dst_stride * 8;
+    y -= 8;
+  } while (y > 0);
+}
+
+inline uint8x16_t GetPositive2TapFilter(const int tap_index) {
+  assert(tap_index < 2);
+  constexpr uint8_t kSubPixel2TapFilterColumns[2][16] = {
+      {128, 120, 112, 104, 96, 88, 80, 72, 64, 56, 48, 40, 32, 24, 16, 8},
+      {0, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120}};
+
+  return vld1q_u8(kSubPixel2TapFilterColumns[tap_index]);
+}
+
+inline void ConvolveKernelHorizontal2Tap(const uint8_t* src,
+                                         const ptrdiff_t src_stride,
+                                         const int width, const int subpixel_x,
+                                         const int step_x,
+                                         const int intermediate_height,
+                                         int16_t* intermediate) {
+  const int kIntermediateStride = kMaxSuperBlockSizeInPixels;
+  // Account for the 0-taps that precede the 2 nonzero taps.
+  const int kernel_offset = 3;
+  const int ref_x = subpixel_x >> kScaleSubPixelBits;
+  const int step_x8 = step_x << 3;
+  const uint8x16_t filter_taps0 = GetPositive2TapFilter(0);
+  const uint8x16_t filter_taps1 = GetPositive2TapFilter(1);
+  const uint16x8_t sum = vdupq_n_u16(1 << (kBitdepth8 + kFilterBits - 1));
+  uint16x8_t index_steps = vmulq_n_u16(vmovl_u8(vcreate_u8(0x0706050403020100)),
+                                       static_cast<uint16_t>(step_x));
+
+  const uint8x8_t filter_index_mask = vdup_n_u8(kSubPixelMask);
+  for (int x = 0, p = subpixel_x; x < width; x += 8, p += step_x8) {
+    const uint8_t* src_x =
+        &src[(p >> kScaleSubPixelBits) - ref_x + kernel_offset];
+    int16_t* intermediate_x = intermediate + x;
+    // Only add steps to the 10-bit truncated p to avoid overflow.
+    const uint16x8_t p_fraction = vdupq_n_u16(p & 1023);
+    const uint16x8_t subpel_index_offsets = vaddq_u16(index_steps, p_fraction);
+    const uint8x8_t filter_indices =
+        vand_u8(vshrn_n_u16(subpel_index_offsets, 6), filter_index_mask);
+    // This is a special case. The 2-tap filter has no negative taps, so we
+    // can use unsigned values.
+    // For each x, a lane of tapsK has
+    // kSubPixelFilters[filter_index][filter_id][k], where filter_id depends
+    // on x.
+    const uint8x8_t taps0 = VQTbl1U8(filter_taps0, filter_indices);
+    const uint8x8_t taps1 = VQTbl1U8(filter_taps1, filter_indices);
+    for (int y = 0; y < intermediate_height; ++y) {
+      // Load a pool of samples to select from using stepped indices.
+      uint8x16_t src_vals = vld1q_u8(src_x);
+      const uint8x8_t src_indices =
+          vmovn_u16(vshrq_n_u16(subpel_index_offsets, kScaleSubPixelBits));
+
+      // For each x, a lane of srcK contains src_x[k].
+      const uint8x8_t src0 = VQTbl1U8(src_vals, src_indices);
+      const uint8x8_t src1 =
+          VQTbl1U8(src_vals, vadd_u8(src_indices, vdup_n_u8(1)));
+
+      const uint16x8_t product0 = vmlal_u8(sum, taps0, src0);
+      // product0 + product1
+      const uint16x8_t result = vmlal_u8(product0, taps1, src1);
+
+      vst1q_s16(intermediate_x, vreinterpretq_s16_u16(vrshrq_n_u16(result, 3)));
+      src_x += src_stride;
+      intermediate_x += kIntermediateStride;
+    }
+  }
+}
+
+inline uint8x16_t GetPositive4TapFilter(const int tap_index) {
+  assert(tap_index < 4);
+  constexpr uint8_t kSubPixel4TapPositiveFilterColumns[4][16] = {
+      {0, 30, 26, 22, 20, 18, 16, 14, 12, 12, 10, 8, 6, 4, 4, 2},
+      {128, 62, 62, 62, 60, 58, 56, 54, 52, 48, 46, 44, 42, 40, 36, 34},
+      {0, 34, 36, 40, 42, 44, 46, 48, 52, 54, 56, 58, 60, 62, 62, 62},
+      {0, 2, 4, 4, 6, 8, 10, 12, 12, 14, 16, 18, 20, 22, 26, 30}};
+
+  uint8x16_t filter_taps =
+      vld1q_u8(kSubPixel4TapPositiveFilterColumns[tap_index]);
+  return filter_taps;
+}
+
+// This filter is only possible when width <= 4.
+inline void ConvolveKernelHorizontalPositive4Tap(
+    const uint8_t* src, const ptrdiff_t src_stride, const int subpixel_x,
+    const int step_x, const int intermediate_height, int16_t* intermediate) {
+  const int kernel_offset = 2;
+  const int ref_x = subpixel_x >> kScaleSubPixelBits;
+  const uint8x8_t filter_index_mask = vdup_n_u8(kSubPixelMask);
+  const uint8x16_t filter_taps0 = GetPositive4TapFilter(0);
+  const uint8x16_t filter_taps1 = GetPositive4TapFilter(1);
+  const uint8x16_t filter_taps2 = GetPositive4TapFilter(2);
+  const uint8x16_t filter_taps3 = GetPositive4TapFilter(3);
+  uint16x8_t index_steps = vmulq_n_u16(vmovl_u8(vcreate_u8(0x0706050403020100)),
+                                       static_cast<uint16_t>(step_x));
+  int p = subpixel_x;
+  const uint16x8_t base = vdupq_n_u16(1 << (kBitdepth8 + kFilterBits - 1));
+  // First filter is special, just a 128 tap on the center.
+  const uint8_t* src_x =
+      &src[(p >> kScaleSubPixelBits) - ref_x + kernel_offset];
+  // Only add steps to the 10-bit truncated p to avoid overflow.
+  const uint16x8_t p_fraction = vdupq_n_u16(p & 1023);
+  const uint16x8_t subpel_index_offsets = vaddq_u16(index_steps, p_fraction);
+  const uint8x8_t filter_indices =
+      vand_u8(vshrn_n_u16(subpel_index_offsets, 6), filter_index_mask);
+  // Note that filter_id depends on x.
+  // For each x, tapsK has kSubPixelFilters[filter_index][filter_id][k].
+  const uint8x8_t taps0 = VQTbl1U8(filter_taps0, filter_indices);
+  const uint8x8_t taps1 = VQTbl1U8(filter_taps1, filter_indices);
+  const uint8x8_t taps2 = VQTbl1U8(filter_taps2, filter_indices);
+  const uint8x8_t taps3 = VQTbl1U8(filter_taps3, filter_indices);
+
+  const uint8x8_t src_indices =
+      vmovn_u16(vshrq_n_u16(subpel_index_offsets, kScaleSubPixelBits));
+  for (int y = 0; y < intermediate_height; ++y) {
+    // Load a pool of samples to select from using stepped index vectors.
+    uint8x16_t src_vals = vld1q_u8(src_x);
+
+    // For each x, srcK contains src_x[k] where k=1.
+    // Whereas taps come from different arrays, src pixels are drawn from the
+    // same contiguous line.
+    const uint8x8_t src0 = VQTbl1U8(src_vals, src_indices);
+    const uint8x8_t src1 =
+        VQTbl1U8(src_vals, vadd_u8(src_indices, vdup_n_u8(1)));
+    const uint8x8_t src2 =
+        VQTbl1U8(src_vals, vadd_u8(src_indices, vdup_n_u8(2)));
+    const uint8x8_t src3 =
+        VQTbl1U8(src_vals, vadd_u8(src_indices, vdup_n_u8(3)));
+
+    uint16x8_t sum = vmlal_u8(base, taps0, src0);
+    sum = vmlal_u8(sum, taps1, src1);
+    sum = vmlal_u8(sum, taps2, src2);
+    sum = vmlal_u8(sum, taps3, src3);
+
+    vst1_s16(intermediate,
+             vreinterpret_s16_u16(vrshr_n_u16(vget_low_u16(sum), 3)));
+
+    src_x += src_stride;
+    intermediate += kIntermediateStride;
+  }
+}
+
+inline uint8x16_t GetSigned4TapFilter(const int tap_index) {
+  assert(tap_index < 4);
+  // The first and fourth taps of each filter are negative. However
+  // 128 does not fit in an 8-bit signed integer. Thus we use subtraction to
+  // keep everything unsigned.
+  constexpr uint8_t kSubPixel4TapSignedFilterColumns[4][16] = {
+      {0, 4, 8, 10, 12, 12, 14, 12, 12, 10, 10, 10, 8, 6, 4, 2},
+      {128, 126, 122, 116, 110, 102, 94, 84, 76, 66, 58, 48, 38, 28, 18, 8},
+      {0, 8, 18, 28, 38, 48, 58, 66, 76, 84, 94, 102, 110, 116, 122, 126},
+      {0, 2, 4, 6, 8, 10, 10, 10, 12, 12, 14, 12, 12, 10, 8, 4}};
+
+  uint8x16_t filter_taps =
+      vld1q_u8(kSubPixel4TapSignedFilterColumns[tap_index]);
+  return filter_taps;
+}
+
+// This filter is only possible when width <= 4.
+inline void ConvolveKernelHorizontalSigned4Tap(
+    const uint8_t* src, const ptrdiff_t src_stride, const int subpixel_x,
+    const int step_x, const int intermediate_height, int16_t* intermediate) {
+  const int kernel_offset = 2;
+  const int ref_x = subpixel_x >> kScaleSubPixelBits;
+  const uint8x8_t filter_index_mask = vdup_n_u8(kSubPixelMask);
+  const uint8x16_t filter_taps0 = GetSigned4TapFilter(0);
+  const uint8x16_t filter_taps1 = GetSigned4TapFilter(1);
+  const uint8x16_t filter_taps2 = GetSigned4TapFilter(2);
+  const uint8x16_t filter_taps3 = GetSigned4TapFilter(3);
+  const uint16x8_t index_steps = vmulq_n_u16(vmovl_u8(vcreate_u8(0x03020100)),
+                                             static_cast<uint16_t>(step_x));
+
+  const uint16x8_t base = vdupq_n_u16(1 << (kBitdepth8 + kFilterBits - 1));
+  int p = subpixel_x;
+  const uint8_t* src_x =
+      &src[(p >> kScaleSubPixelBits) - ref_x + kernel_offset];
+  // Only add steps to the 10-bit truncated p to avoid overflow.
+  const uint16x8_t p_fraction = vdupq_n_u16(p & 1023);
+  const uint16x8_t subpel_index_offsets = vaddq_u16(index_steps, p_fraction);
+  const uint8x8_t filter_indices =
+      vand_u8(vshrn_n_u16(subpel_index_offsets, 6), filter_index_mask);
+  // Note that filter_id depends on x.
+  // For each x, tapsK has kSubPixelFilters[filter_index][filter_id][k].
+  const uint8x8_t taps0 = VQTbl1U8(filter_taps0, filter_indices);
+  const uint8x8_t taps1 = VQTbl1U8(filter_taps1, filter_indices);
+  const uint8x8_t taps2 = VQTbl1U8(filter_taps2, filter_indices);
+  const uint8x8_t taps3 = VQTbl1U8(filter_taps3, filter_indices);
+  for (int y = 0; y < intermediate_height; ++y) {
+    // Load a pool of samples to select from using stepped indices.
+    uint8x16_t src_vals = vld1q_u8(src_x);
+    const uint8x8_t src_indices =
+        vmovn_u16(vshrq_n_u16(subpel_index_offsets, kScaleSubPixelBits));
+
+    // For each x, srcK contains src_x[k] where k=1.
+    // Whereas taps come from different arrays, src pixels are drawn from the
+    // same contiguous line.
+    const uint8x8_t src0 = VQTbl1U8(src_vals, src_indices);
+    const uint8x8_t src1 =
+        VQTbl1U8(src_vals, vadd_u8(src_indices, vdup_n_u8(1)));
+    const uint8x8_t src2 =
+        VQTbl1U8(src_vals, vadd_u8(src_indices, vdup_n_u8(2)));
+    const uint8x8_t src3 =
+        VQTbl1U8(src_vals, vadd_u8(src_indices, vdup_n_u8(3)));
+
+    // Offsetting by base permits a guaranteed positive.
+    uint16x8_t sum = vmlsl_u8(base, taps0, src0);
+    sum = vmlal_u8(sum, taps1, src1);
+    sum = vmlal_u8(sum, taps2, src2);
+    sum = vmlsl_u8(sum, taps3, src3);
+
+    vst1_s16(intermediate,
+             vreinterpret_s16_u16(vrshr_n_u16(vget_low_u16(sum), 3)));
+    src_x += src_stride;
+    intermediate += kIntermediateStride;
+  }
+}
+
+void ConvolveCompoundScale2D_NEON(
+    const void* const reference, const ptrdiff_t reference_stride,
+    const int horizontal_filter_index, const int vertical_filter_index,
+    const uint8_t inter_round_bits_vertical, const int subpixel_x,
+    const int subpixel_y, const int step_x, const int step_y, const int width,
+    const int height, void* prediction, const ptrdiff_t pred_stride) {
+  const int intermediate_height =
+      (((height - 1) * step_y + (1 << kScaleSubPixelBits) - 1) >>
+       kScaleSubPixelBits) +
+      kSubPixelTaps;
+  // TODO(b/133525024): Decide whether it's worth branching to a special case
+  // when step_x or step_y is 1024.
+  assert(step_x <= 2048);
+  // The output of the horizontal filter, i.e. the intermediate_result, is
+  // guaranteed to fit in int16_t.
+  int16_t intermediate_result[kMaxSuperBlockSizeInPixels *
+                              (2 * kMaxSuperBlockSizeInPixels + 8)];
+
+  // Horizontal filter.
+  // Filter types used for width <= 4 are different from those for width > 4.
+  // When width > 4, the valid filter index range is always [0, 3].
+  // When width <= 4, the valid filter index range is always [3, 5].
+  // Similarly for height.
+  const int kIntermediateStride = kMaxSuperBlockSizeInPixels;
+  int filter_index = GetFilterIndex(horizontal_filter_index, width);
+  int16_t* intermediate = intermediate_result;
+  const auto* src = static_cast<const uint8_t*>(reference);
+  const ptrdiff_t src_stride = reference_stride;
+  auto* dest = static_cast<uint16_t*>(prediction);
+  switch (filter_index) {
+    case 0:
+      if (step_x < 1024) {
+        ConvolveHorizontalScaled_NEON<0, 6, 1>(
+            src, src_stride, width, subpixel_x, step_x, intermediate_height,
+            intermediate);
+      } else {
+        ConvolveHorizontalScaled_NEON<0, 6, 2>(
+            src, src_stride, width, subpixel_x, step_x, intermediate_height,
+            intermediate);
+      }
+      break;
+    case 1:
+      if (step_x < 1024) {
+        ConvolveHorizontalScaled_NEON<1, 6, 1>(
+            src, src_stride, width, subpixel_x, step_x, intermediate_height,
+            intermediate);
+      } else {
+        ConvolveHorizontalScaled_NEON<1, 6, 2>(
+            src, src_stride, width, subpixel_x, step_x, intermediate_height,
+            intermediate);
+      }
+      break;
+    case 2:
+      if (step_x <= 1024) {
+        ConvolveHorizontalScaled_NEON<2, 8, 1>(
+            src, src_stride, width, subpixel_x, step_x, intermediate_height,
+            intermediate);
+      } else {
+        ConvolveHorizontalScaled_NEON<2, 8, 2>(
+            src, src_stride, width, subpixel_x, step_x, intermediate_height,
+            intermediate);
+      }
+      break;
+    case 3:
+      ConvolveKernelHorizontal2Tap(src, src_stride, width, subpixel_x, step_x,
+                                   intermediate_height, intermediate);
+      break;
+    case 4:
+      assert(width <= 4);
+      ConvolveKernelHorizontalSigned4Tap(src, src_stride, subpixel_x, step_x,
+                                         intermediate_height, intermediate);
+      break;
+    default:
+      assert(filter_index == 5);
+      ConvolveKernelHorizontalPositive4Tap(src, src_stride, subpixel_x, step_x,
+                                           intermediate_height, intermediate);
+  }
+  // Vertical filter.
+  filter_index = GetFilterIndex(vertical_filter_index, height);
+  intermediate = intermediate_result;
+  const int offset_bits = kBitdepth8 + 2 * kFilterBits - 3;
+  for (int y = 0, p = subpixel_y & 1023; y < height; ++y, p += step_y) {
+    const int filter_id = (p >> 6) & kSubPixelMask;
+    for (int x = 0; x < width; ++x) {
+      // An offset to guarantee the sum is non negative.
+      int sum = 1 << offset_bits;
+      for (int k = 0; k < kSubPixelTaps; ++k) {
+        sum +=
+            kSubPixelFilters[filter_index][filter_id][k] *
+            intermediate[((p >> kScaleSubPixelBits) + k) * kIntermediateStride +
+                         x];
+      }
+      assert(sum >= 0 && sum < (1 << (offset_bits + 2)));
+      dest[x] = static_cast<uint16_t>(
+          RightShiftWithRounding(sum, inter_round_bits_vertical));
+    }
+    dest += pred_stride;
+  }
+}
+
+void ConvolveHorizontal_NEON(const void* const reference,
+                             const ptrdiff_t reference_stride,
+                             const int horizontal_filter_index,
+                             const int /*vertical_filter_index*/,
+                             const uint8_t /*inter_round_bits_vertical*/,
+                             const int subpixel_x, const int /*subpixel_y*/,
+                             const int /*step_x*/, const int /*step_y*/,
+                             const int width, const int height,
+                             void* prediction, const ptrdiff_t pred_stride) {
+  // For 8 (and 10) bit calculations |inter_round_bits_horizontal| is 3.
+  const int filter_index = GetFilterIndex(horizontal_filter_index, width);
+  // Set |src| to the outermost tap.
+  const auto* src = static_cast<const uint8_t*>(reference) - kHorizontalOffset;
+  const ptrdiff_t src_stride = reference_stride;
+  auto* dest = static_cast<uint8_t*>(prediction);
+  const ptrdiff_t dest_stride = pred_stride;
+  const int filter_id = (subpixel_x >> 6) & kSubPixelMask;
+  const int block_output_height = std::min(height, 8);
+  const int16x8_t four = vdupq_n_s16(4);
+
+  int16x8_t taps;
+  if (filter_index < 3) {
+    // 6 and 8 tap filters.
+    taps = vld1q_s16(kSubPixelFilters[filter_index][filter_id]);
+  } else {
+    // The 2 tap filter only uses the lower half of |taps|.
+    taps = vld1q_s16(kSubPixelFilters[filter_index][filter_id] + 2);
+  }
+
+  // TODO(johannkoenig): specialize small |height| variants so we don't
+  // overread |reference|.
+  if (width > 4 && height > 4) {
+    int y = 0;
+    do {
+      // This was intended to load and transpose 16 values before the |width|
+      // loop. At the end of the loop it would keep 8 of those values and only
+      // load and transpose 8 additional values. Unfortunately the approach did
+      // not appear to provide any benefit.
+      int x = 0;
+      do {
+        uint8x16_t temp[8];
+        uint8x8_t input[16];
+        for (int i = 0; i < 8; ++i) {
+          temp[i] = vld1q_u8(src + x + i * src_stride);
+        }
+        // TODO(johannkoenig): It should be possible to get the transpose
+        // started with vld4().
+        Transpose16x8(temp, input);
+        int16x8_t input16[16];
+        for (int i = 0; i < 16; ++i) {
+          input16[i] = ZeroExtend(input[i]);
+        }
+
+        // This does not handle |filter_index| > 3 because those 4 tap filters
+        // are only used when |width| <= 4.
+        // TODO(johannkoenig): Explore moving the branch outside the main loop.
+        uint8x8_t output[8];
+        if (filter_index == 2) {  // 8 taps.
+          for (int i = 0; i < 8; ++i) {
+            const int16x8_t neon_sums = SumTaps<8>(input16 + i, taps);
+            output[i] =
+                vqrshrun_n_s16(vqaddq_s16(neon_sums, four), kFilterBits);
+          }
+        } else if (filter_index < 2) {  // 6 taps.
+          for (int i = 0; i < 8; ++i) {
+            const int16x8_t neon_sums = SumTaps<6>(input16 + i + 1, taps);
+            output[i] =
+                vqrshrun_n_s16(vqaddq_s16(neon_sums, four), kFilterBits);
+          }
+        } else {  // |filter_index| == 3. 2 taps.
+          for (int i = 0; i < 8; ++i) {
+            const int16x8_t neon_sums = SumTaps<2>(input16 + i + 3, taps);
+            output[i] =
+                vqrshrun_n_s16(vqaddq_s16(neon_sums, four), kFilterBits);
+          }
+        }
+
+        Transpose8x8(output);
+
+        int i = 0;
+        do {
+          vst1_u8(dest + x + i * dest_stride, output[i]);
+        } while (++i < block_output_height);
+        x += 8;
+      } while (x < width);
+      y += 8;
+      src += 8 * src_stride;
+      dest += 8 * dest_stride;
+    } while (y < height);
+  } else {
+    // TODO(johannkoenig): Investigate 2xH and 4xH. During the original
+    // implementation 4x2 was slower than C, 4x4 reached parity, and 4x8
+    // was < 20% faster.
+    for (int y = 0; y < height; ++y) {
+      for (int x = 0; x < width; ++x) {
+        int sum = 0;
+        for (int k = 0; k < kSubPixelTaps; ++k) {
+          sum += kSubPixelFilters[filter_index][filter_id][k] * src[x + k];
+        }
+        // We can combine the shifts if we compensate for the skipped rounding.
+        // ((sum + 4 >> 3) + 8) >> 4 == (sum + 64 + 4) >> 7;
+        dest[x] = static_cast<uint8_t>(
+            Clip3(RightShiftWithRounding(sum + 4, kFilterBits), 0, 255));
+      }
+      src += src_stride;
+      dest += dest_stride;
+    }
+  }
+}
+
+template <int min_width, int num_taps>
+void FilterVertical(const uint8_t* src, const ptrdiff_t src_stride,
+                    uint8_t* dst, const ptrdiff_t dst_stride, const int width,
+                    const int height, const int16x8_t taps) {
+  constexpr int next_row = num_taps - 1;
+  // |src| points to the outermost tap of the first value. When doing fewer than
+  // 8 taps it needs to be adjusted.
+  if (num_taps == 6) {
+    src += src_stride;
+  } else if (num_taps == 4) {
+    src += 2 * src_stride;
+  } else if (num_taps == 2) {
+    src += 3 * src_stride;
+  }
+
+  int x = 0;
+  do {
+    int16x8_t srcs[8];
+    srcs[0] = ZeroExtend(vld1_u8(src + x));
+    if (num_taps >= 4) {
+      srcs[1] = ZeroExtend(vld1_u8(src + x + src_stride));
+      srcs[2] = ZeroExtend(vld1_u8(src + x + 2 * src_stride));
+      if (num_taps >= 6) {
+        srcs[3] = ZeroExtend(vld1_u8(src + x + 3 * src_stride));
+        srcs[4] = ZeroExtend(vld1_u8(src + x + 4 * src_stride));
+        if (num_taps == 8) {
+          srcs[5] = ZeroExtend(vld1_u8(src + x + 5 * src_stride));
+          srcs[6] = ZeroExtend(vld1_u8(src + x + 6 * src_stride));
+        }
+      }
+    }
+
+    int y = 0;
+    do {
+      srcs[next_row] =
+          ZeroExtend(vld1_u8(src + x + (y + next_row) * src_stride));
+
+      const int16x8_t sums = SumTaps<num_taps>(srcs, taps);
+      const uint8x8_t results = vqrshrun_n_s16(sums, kFilterBits);
+
+      if (min_width == 4) {
+        StoreLo4(dst + x + y * dst_stride, results);
+      } else {
+        vst1_u8(dst + x + y * dst_stride, results);
+      }
+
+      srcs[0] = srcs[1];
+      if (num_taps >= 4) {
+        srcs[1] = srcs[2];
+        srcs[2] = srcs[3];
+        if (num_taps >= 6) {
+          srcs[3] = srcs[4];
+          srcs[4] = srcs[5];
+          if (num_taps == 8) {
+            srcs[5] = srcs[6];
+            srcs[6] = srcs[7];
+          }
+        }
+      }
+    } while (++y < height);
+    x += 8;
+  } while (x < width);
+}
+
+// This function is a simplified version of Convolve2D_C.
+// It is called when it is single prediction mode, where only vertical
+// filtering is required.
+// The output is the single prediction of the block, clipped to valid pixel
+// range.
+void ConvolveVertical_NEON(const void* const reference,
+                           const ptrdiff_t reference_stride,
+                           const int /*horizontal_filter_index*/,
+                           const int vertical_filter_index,
+                           const uint8_t /*inter_round_bits_vertical*/,
+                           const int /*subpixel_x*/, const int subpixel_y,
+                           const int /*step_x*/, const int /*step_y*/,
+                           const int width, const int height, void* prediction,
+                           const ptrdiff_t pred_stride) {
+  const int filter_index = GetFilterIndex(vertical_filter_index, height);
+  const ptrdiff_t src_stride = reference_stride;
+  const auto* src =
+      static_cast<const uint8_t*>(reference) - kVerticalOffset * src_stride;
+  auto* dest = static_cast<uint8_t*>(prediction);
+  const ptrdiff_t dest_stride = pred_stride;
+  const int filter_id = (subpixel_y >> 6) & kSubPixelMask;
+  // First filter is always a copy.
+  if (filter_id == 0) {
+    // Move |src| down the actual values and not the start of the context.
+    src = static_cast<const uint8_t*>(reference);
+    int y = 0;
+    do {
+      memcpy(dest, src, width * sizeof(src[0]));
+      src += src_stride;
+      dest += dest_stride;
+    } while (++y < height);
+    return;
+  }
+
+  // Break up by # of taps
+  // |filter_index| taps  enum InterpolationFilter
+  //        0       6     kInterpolationFilterEightTap
+  //        1       6     kInterpolationFilterEightTapSmooth
+  //        2       8     kInterpolationFilterEightTapSharp
+  //        3       2     kInterpolationFilterBilinear
+  //        4       4     kInterpolationFilterSwitchable
+  //        5       4     !!! SECRET FILTER !!! only for Wx4.
+  if (width >= 4) {
+    if (filter_index == 2) {  // 8 tap.
+      const int16x8_t taps =
+          vld1q_s16(kSubPixelFilters[filter_index][filter_id]);
+      if (width == 4) {
+        FilterVertical<4, 8>(src, src_stride, dest, dest_stride, width, height,
+                             taps);
+      } else {
+        FilterVertical<8, 8>(src, src_stride, dest, dest_stride, width, height,
+                             taps);
+      }
+    } else if (filter_index < 2) {  // 6 tap.
+      const int16x8_t taps =
+          vld1q_s16(kSubPixelFilters[filter_index][filter_id]);
+      if (width == 4) {
+        FilterVertical<4, 6>(src, src_stride, dest, dest_stride, width, height,
+                             taps);
+      } else {
+        FilterVertical<8, 6>(src, src_stride, dest, dest_stride, width, height,
+                             taps);
+      }
+    } else if (filter_index > 3) {  // 4 tap.
+      // Store taps in vget_low_s16(taps).
+      const int16x8_t taps =
+          vld1q_s16(kSubPixelFilters[filter_index][filter_id] + 2);
+      if (width == 4) {
+        FilterVertical<4, 4>(src, src_stride, dest, dest_stride, width, height,
+                             taps);
+      } else {
+        FilterVertical<8, 4>(src, src_stride, dest, dest_stride, width, height,
+                             taps);
+      }
+    } else {  // 2 tap.
+      // Store taps in vget_low_s16(taps).
+      const int16x8_t taps =
+          vld1q_s16(kSubPixelFilters[filter_index][filter_id] + 2);
+      if (width == 4) {
+        FilterVertical<4, 2>(src, src_stride, dest, dest_stride, width, height,
+                             taps);
+      } else {
+        FilterVertical<8, 2>(src, src_stride, dest, dest_stride, width, height,
+                             taps);
+      }
+    }
+  } else {
+    // TODO(johannkoenig): Determine if it is worth writing a 2xH
+    // implementation.
+    assert(width == 2);
+    const int max_pixel_value = 255;
+    int y = 0;
+    do {
+      for (int x = 0; x < 2; ++x) {
+        int sum = 0;
+        for (int k = 0; k < kSubPixelTaps; ++k) {
+          sum += kSubPixelFilters[filter_index][filter_id][k] *
+                 src[k * src_stride + x];
+        }
+        dest[x] = static_cast<uint8_t>(Clip3(
+            RightShiftWithRounding(sum, kFilterBits), 0, max_pixel_value));
+      }
+      src += src_stride;
+      dest += pred_stride;
+    } while (++y < height);
+  }
+}
+
+void ConvolveCompoundCopy_NEON(
+    const void* const reference, const ptrdiff_t reference_stride,
+    const int /*horizontal_filter_index*/, const int /*vertical_filter_index*/,
+    const uint8_t /*inter_round_bits_vertical*/, const int /*subpixel_x*/,
+    const int /*subpixel_y*/, const int /*step_x*/, const int /*step_y*/,
+    const int width, const int height, void* prediction,
+    const ptrdiff_t pred_stride) {
+  const auto* src = static_cast<const uint8_t*>(reference);
+  const ptrdiff_t src_stride = reference_stride;
+  auto* dest = static_cast<uint16_t*>(prediction);
+  const int bitdepth = 8;
+  const int compound_round_offset =
+      (1 << (bitdepth + 4)) + (1 << (bitdepth + 3));
+  const uint16x8_t v_compound_round_offset = vdupq_n_u16(compound_round_offset);
+
+  if (width >= 16) {
+    int y = 0;
+    do {
+      int x = 0;
+      do {
+        const uint8x16_t v_src = vld1q_u8(&src[x]);
+        const uint16x8_t v_src_x16_lo = vshll_n_u8(vget_low_u8(v_src), 4);
+        const uint16x8_t v_src_x16_hi = vshll_n_u8(vget_high_u8(v_src), 4);
+        const uint16x8_t v_dest_lo =
+            vaddq_u16(v_src_x16_lo, v_compound_round_offset);
+        const uint16x8_t v_dest_hi =
+            vaddq_u16(v_src_x16_hi, v_compound_round_offset);
+        vst1q_u16(&dest[x], v_dest_lo);
+        x += 8;
+        vst1q_u16(&dest[x], v_dest_hi);
+        x += 8;
+      } while (x < width);
+      src += src_stride;
+      dest += pred_stride;
+    } while (++y < height);
+  } else if (width == 8) {
+    int y = 0;
+    do {
+      const uint8x8_t v_src = vld1_u8(&src[0]);
+      const uint16x8_t v_src_x16 = vshll_n_u8(v_src, 4);
+      vst1q_u16(&dest[0], vaddq_u16(v_src_x16, v_compound_round_offset));
+      src += src_stride;
+      dest += pred_stride;
+    } while (++y < height);
+  } else if (width == 4) {
+    const uint8x8_t zero = vdup_n_u8(0);
+    int y = 0;
+    do {
+      const uint8x8_t v_src = LoadLo4(&src[0], zero);
+      const uint16x8_t v_src_x16 = vshll_n_u8(v_src, 4);
+      const uint16x8_t v_dest = vaddq_u16(v_src_x16, v_compound_round_offset);
+      vst1_u16(&dest[0], vget_low_u16(v_dest));
+      src += src_stride;
+      dest += pred_stride;
+    } while (++y < height);
+  } else {  // width == 2
+    assert(width == 2);
+    int y = 0;
+    do {
+      dest[0] = (src[0] << 4) + compound_round_offset;
+      dest[1] = (src[1] << 4) + compound_round_offset;
+      src += src_stride;
+      dest += pred_stride;
+    } while (++y < height);
+  }
+}
+
+// Input 8 bits and output 16 bits.
+template <int min_width, int num_taps>
+void FilterCompoundVertical(const uint8_t* src, const ptrdiff_t src_stride,
+                            uint16_t* dst, const ptrdiff_t dst_stride,
+                            const int width, const int height,
+                            const int16x8_t taps) {
+  constexpr int next_row = num_taps - 1;
+  // |src| points to the outermost tap of the first value. When doing fewer than
+  // 8 taps it needs to be adjusted.
+  if (num_taps == 6) {
+    src += src_stride;
+  } else if (num_taps == 4) {
+    src += 2 * src_stride;
+  } else if (num_taps == 2) {
+    src += 3 * src_stride;
+  }
+
+  const uint16x8_t compound_round_offset = vdupq_n_u16(1 << 12);
+
+  int x = 0;
+  do {
+    int16x8_t srcs[8];
+    srcs[0] = ZeroExtend(vld1_u8(src + x));
+    if (num_taps >= 4) {
+      srcs[1] = ZeroExtend(vld1_u8(src + x + src_stride));
+      srcs[2] = ZeroExtend(vld1_u8(src + x + 2 * src_stride));
+      if (num_taps >= 6) {
+        srcs[3] = ZeroExtend(vld1_u8(src + x + 3 * src_stride));
+        srcs[4] = ZeroExtend(vld1_u8(src + x + 4 * src_stride));
+        if (num_taps == 8) {
+          srcs[5] = ZeroExtend(vld1_u8(src + x + 5 * src_stride));
+          srcs[6] = ZeroExtend(vld1_u8(src + x + 6 * src_stride));
+        }
+      }
+    }
+
+    int y = 0;
+    do {
+      srcs[next_row] =
+          ZeroExtend(vld1_u8(src + x + (y + next_row) * src_stride));
+
+      const uint16x8_t sums = SumTaps8To16<num_taps>(srcs, taps);
+      const uint16x8_t shifted = vrshrq_n_u16(sums, 3);
+      // In order to keep the sum in 16 bits we add an offset to the sum
+      // (1 << (bitdepth + kFilterBits - 1) == 1 << 14). This ensures that the
+      // results will never be negative.
+      // Normally ConvolveCompoundVertical would add |compound_round_offset| at
+      // the end. Instead we use that to compensate for the initial offset.
+      // (1 << (bitdepth + 4)) + (1 << (bitdepth + 3)) == (1 << 12) + (1 << 11)
+      // After taking into account the shift above:
+      // RightShiftWithRounding(LeftShift(sum, bits_shift), inter_round_bits[1])
+      // where bits_shift == kFilterBits - inter_round_bits[0] == 4
+      // and inter_round_bits[1] == 7
+      // and simplifying it to RightShiftWithRounding(sum, 3)
+      // we see that the initial offset of 1 << 14 >> 3 == 1 << 11 and
+      // |compound_round_offset| can be simplified to 1 << 12.
+      const uint16x8_t offset = vaddq_u16(shifted, compound_round_offset);
+
+      if (min_width == 4) {
+        vst1_u16(dst + x + y * dst_stride, vget_low_u16(offset));
+      } else {
+        vst1q_u16(dst + x + y * dst_stride, offset);
+      }
+
+      srcs[0] = srcs[1];
+      if (num_taps >= 4) {
+        srcs[1] = srcs[2];
+        srcs[2] = srcs[3];
+        if (num_taps >= 6) {
+          srcs[3] = srcs[4];
+          srcs[4] = srcs[5];
+          if (num_taps == 8) {
+            srcs[5] = srcs[6];
+            srcs[6] = srcs[7];
+          }
+        }
+      }
+    } while (++y < height);
+    x += 8;
+  } while (x < width);
+}
+
+void ConvolveCompoundVertical_NEON(
+    const void* const reference, const ptrdiff_t reference_stride,
+    const int /*horizontal_filter_index*/, const int vertical_filter_index,
+    const uint8_t /*inter_round_bits_vertical*/, const int /*subpixel_x*/,
+    const int subpixel_y, const int /*step_x*/, const int /*step_y*/,
+    const int width, const int height, void* prediction,
+    const ptrdiff_t pred_stride) {
+  const int filter_index = GetFilterIndex(vertical_filter_index, height);
+  const ptrdiff_t src_stride = reference_stride;
+  const auto* src =
+      static_cast<const uint8_t*>(reference) - kVerticalOffset * src_stride;
+  auto* dest = static_cast<uint16_t*>(prediction);
+  const int filter_id = (subpixel_y >> 6) & kSubPixelMask;
+  const int compound_round_offset = 1 << 12;  // Leave off + 1 << 11.
+
+  if (width >= 4) {
+    const int16x8_t taps = vld1q_s16(kSubPixelFilters[filter_index][filter_id]);
+
+    if (filter_index == 2) {  // 8 tap.
+      if (width == 4) {
+        FilterCompoundVertical<4, 8>(src, src_stride, dest, pred_stride, width,
+                                     height, taps);
+      } else {
+        FilterCompoundVertical<8, 8>(src, src_stride, dest, pred_stride, width,
+                                     height, taps);
+      }
+    } else if (filter_index < 2) {  // 6 tap.
+      if (width == 4) {
+        FilterCompoundVertical<4, 6>(src, src_stride, dest, pred_stride, width,
+                                     height, taps);
+      } else {
+        FilterCompoundVertical<8, 6>(src, src_stride, dest, pred_stride, width,
+                                     height, taps);
+      }
+    } else if (filter_index == 3) {  // 2 tap.
+      if (width == 4) {
+        FilterCompoundVertical<4, 2>(src, src_stride, dest, pred_stride, width,
+                                     height, taps);
+      } else {
+        FilterCompoundVertical<8, 2>(src, src_stride, dest, pred_stride, width,
+                                     height, taps);
+      }
+    } else if (filter_index > 3) {  // 4 tap.
+      if (width == 4) {
+        FilterCompoundVertical<4, 4>(src, src_stride, dest, pred_stride, width,
+                                     height, taps);
+      } else {
+        FilterCompoundVertical<8, 4>(src, src_stride, dest, pred_stride, width,
+                                     height, taps);
+      }
+    }
+  } else {
+    assert(width == 2);
+    for (int y = 0; y < height; ++y) {
+      for (int x = 0; x < 2; ++x) {
+        // Use an offset to avoid 32 bits.
+        int sum = 1 << 14;
+        for (int k = 0; k < kSubPixelTaps; ++k) {
+          sum += kSubPixelFilters[filter_index][filter_id][k] *
+                 src[k * src_stride + x];
+        }
+        // |compound_round_offset| has been modified to take into account the
+        // offset used above. The 1 << 11 term cancels out with 1 << 14 >> 3.
+        dest[x] = RightShiftWithRounding(sum, 3) + compound_round_offset;
+      }
+      src += src_stride;
+      dest += pred_stride;
+    }
+  }
+}
+
+template <int num_taps, int filter_index, bool negative_outside_taps = true>
+uint16x8_t SumCompoundHorizontalTaps(const uint8_t* const src,
+                                     uint8x8_t* v_tap) {
+  // Start with an offset to guarantee the sum is non negative.
+  uint16x8_t v_sum = vdupq_n_u16(1 << 14);
+  uint8x16_t v_src[8];
+  v_src[0] = vld1q_u8(&src[0]);
+  if (num_taps == 8) {
+    v_src[1] = vextq_u8(v_src[0], v_src[0], 1);
+    v_src[2] = vextq_u8(v_src[0], v_src[0], 2);
+    v_src[3] = vextq_u8(v_src[0], v_src[0], 3);
+    v_src[4] = vextq_u8(v_src[0], v_src[0], 4);
+    v_src[5] = vextq_u8(v_src[0], v_src[0], 5);
+    v_src[6] = vextq_u8(v_src[0], v_src[0], 6);
+    v_src[7] = vextq_u8(v_src[0], v_src[0], 7);
+
+    // tap signs : - + - + + - + -
+    v_sum = vmlsl_u8(v_sum, vget_low_u8(v_src[0]), v_tap[0]);
+    v_sum = vmlal_u8(v_sum, vget_low_u8(v_src[1]), v_tap[1]);
+    v_sum = vmlsl_u8(v_sum, vget_low_u8(v_src[2]), v_tap[2]);
+    v_sum = vmlal_u8(v_sum, vget_low_u8(v_src[3]), v_tap[3]);
+    v_sum = vmlal_u8(v_sum, vget_low_u8(v_src[4]), v_tap[4]);
+    v_sum = vmlsl_u8(v_sum, vget_low_u8(v_src[5]), v_tap[5]);
+    v_sum = vmlal_u8(v_sum, vget_low_u8(v_src[6]), v_tap[6]);
+    v_sum = vmlsl_u8(v_sum, vget_low_u8(v_src[7]), v_tap[7]);
+  } else if (num_taps == 6) {
+    v_src[1] = vextq_u8(v_src[0], v_src[0], 1);
+    v_src[2] = vextq_u8(v_src[0], v_src[0], 2);
+    v_src[3] = vextq_u8(v_src[0], v_src[0], 3);
+    v_src[4] = vextq_u8(v_src[0], v_src[0], 4);
+    v_src[5] = vextq_u8(v_src[0], v_src[0], 5);
+    v_src[6] = vextq_u8(v_src[0], v_src[0], 6);
+    if (filter_index == 0) {
+      // tap signs : + - + + - +
+      v_sum = vmlal_u8(v_sum, vget_low_u8(v_src[1]), v_tap[1]);
+      v_sum = vmlsl_u8(v_sum, vget_low_u8(v_src[2]), v_tap[2]);
+      v_sum = vmlal_u8(v_sum, vget_low_u8(v_src[3]), v_tap[3]);
+      v_sum = vmlal_u8(v_sum, vget_low_u8(v_src[4]), v_tap[4]);
+      v_sum = vmlsl_u8(v_sum, vget_low_u8(v_src[5]), v_tap[5]);
+      v_sum = vmlal_u8(v_sum, vget_low_u8(v_src[6]), v_tap[6]);
+    } else {
+      if (negative_outside_taps) {
+        // tap signs : - + + + + -
+        v_sum = vmlsl_u8(v_sum, vget_low_u8(v_src[1]), v_tap[1]);
+        v_sum = vmlal_u8(v_sum, vget_low_u8(v_src[2]), v_tap[2]);
+        v_sum = vmlal_u8(v_sum, vget_low_u8(v_src[3]), v_tap[3]);
+        v_sum = vmlal_u8(v_sum, vget_low_u8(v_src[4]), v_tap[4]);
+        v_sum = vmlal_u8(v_sum, vget_low_u8(v_src[5]), v_tap[5]);
+        v_sum = vmlsl_u8(v_sum, vget_low_u8(v_src[6]), v_tap[6]);
+      } else {
+        // tap signs : + + + + + +
+        v_sum = vmlal_u8(v_sum, vget_low_u8(v_src[1]), v_tap[1]);
+        v_sum = vmlal_u8(v_sum, vget_low_u8(v_src[2]), v_tap[2]);
+        v_sum = vmlal_u8(v_sum, vget_low_u8(v_src[3]), v_tap[3]);
+        v_sum = vmlal_u8(v_sum, vget_low_u8(v_src[4]), v_tap[4]);
+        v_sum = vmlal_u8(v_sum, vget_low_u8(v_src[5]), v_tap[5]);
+        v_sum = vmlal_u8(v_sum, vget_low_u8(v_src[6]), v_tap[6]);
+      }
+    }
+  } else if (num_taps == 4) {
+    v_src[2] = vextq_u8(v_src[0], v_src[0], 2);
+    v_src[3] = vextq_u8(v_src[0], v_src[0], 3);
+    v_src[4] = vextq_u8(v_src[0], v_src[0], 4);
+    v_src[5] = vextq_u8(v_src[0], v_src[0], 5);
+    if (filter_index == 4) {
+      // tap signs : - + + -
+      v_sum = vmlsl_u8(v_sum, vget_low_u8(v_src[2]), v_tap[2]);
+      v_sum = vmlal_u8(v_sum, vget_low_u8(v_src[3]), v_tap[3]);
+      v_sum = vmlal_u8(v_sum, vget_low_u8(v_src[4]), v_tap[4]);
+      v_sum = vmlsl_u8(v_sum, vget_low_u8(v_src[5]), v_tap[5]);
+    } else {
+      // tap signs : + + + +
+      v_sum = vmlal_u8(v_sum, vget_low_u8(v_src[2]), v_tap[2]);
+      v_sum = vmlal_u8(v_sum, vget_low_u8(v_src[3]), v_tap[3]);
+      v_sum = vmlal_u8(v_sum, vget_low_u8(v_src[4]), v_tap[4]);
+      v_sum = vmlal_u8(v_sum, vget_low_u8(v_src[5]), v_tap[5]);
+    }
+  } else {
+    assert(num_taps == 2);
+    v_src[3] = vextq_u8(v_src[0], v_src[0], 3);
+    v_src[4] = vextq_u8(v_src[0], v_src[0], 4);
+    // tap signs : + +
+    v_sum = vmlal_u8(v_sum, vget_low_u8(v_src[3]), v_tap[3]);
+    v_sum = vmlal_u8(v_sum, vget_low_u8(v_src[4]), v_tap[4]);
+  }
+
+  return v_sum;
+}
+
+template <int num_taps, int step, int filter_index,
+          bool negative_outside_taps = true>
+void ConvolveCompoundHorizontalBlock(const uint8_t* src, ptrdiff_t src_stride,
+                                     uint16_t* dest, ptrdiff_t pred_stride,
+                                     const int width, const int height,
+                                     uint8x8_t* v_tap,
+                                     int16x8_t v_inter_round_bits_0,
+                                     int16x8_t v_bits_shift,
+                                     uint16x8_t v_compound_round_offset) {
+  if (width > 4) {
+    int y = 0;
+    do {
+      int x = 0;
+      do {
+        uint16x8_t v_sum =
+            SumCompoundHorizontalTaps<num_taps, filter_index,
+                                      negative_outside_taps>(&src[x], v_tap);
+        v_sum = vrshlq_u16(v_sum, v_inter_round_bits_0);
+        v_sum = vshlq_u16(v_sum, v_bits_shift);
+        v_sum = vaddq_u16(v_sum, v_compound_round_offset);
+        vst1q_u16(&dest[x], v_sum);
+        x += step;
+      } while (x < width);
+      src += src_stride;
+      dest += pred_stride;
+    } while (++y < height);
+  } else {
+    int y = 0;
+    do {
+      uint16x8_t v_sum =
+          SumCompoundHorizontalTaps<num_taps, filter_index,
+                                    negative_outside_taps>(&src[0], v_tap);
+      v_sum = vrshlq_u16(v_sum, v_inter_round_bits_0);
+      v_sum = vshlq_u16(v_sum, v_bits_shift);
+      v_sum = vaddq_u16(v_sum, v_compound_round_offset);
+      vst1_u16(&dest[0], vget_low_u16(v_sum));
+      src += src_stride;
+      dest += pred_stride;
+    } while (++y < height);
+  }
+}
+
+void ConvolveCompoundHorizontal_NEON(
+    const void* const reference, const ptrdiff_t reference_stride,
+    const int horizontal_filter_index, const int /*vertical_filter_index*/,
+    const uint8_t inter_round_bits_vertical, const int subpixel_x,
+    const int /*subpixel_y*/, const int /*step_x*/, const int /*step_y*/,
+    const int width, const int height, void* prediction,
+    const ptrdiff_t pred_stride) {
+  const int filter_index = GetFilterIndex(horizontal_filter_index, width);
+  const auto* src = static_cast<const uint8_t*>(reference) - kHorizontalOffset;
+  const ptrdiff_t src_stride = reference_stride;
+  auto* dest = static_cast<uint16_t*>(prediction);
+  const int filter_id = (subpixel_x >> 6) & kSubPixelMask;
+  const int bits_shift = kFilterBits - inter_round_bits_vertical;
+
+  const int compound_round_offset =
+      (1 << (kBitdepth8 + 4)) + (1 << (kBitdepth8 + 3));
+
+  if (width >= 4) {
+    // Duplicate the absolute value for each tap.  Negative taps are corrected
+    // by using the vmlsl_u8 instruction.  Positive taps use vmlal_u8.
+    uint8x8_t v_tap[kSubPixelTaps];
+    for (int k = 0; k < kSubPixelTaps; ++k) {
+      v_tap[k] = vreinterpret_u8_s8(
+          vabs_s8(vdup_n_s8(kSubPixelFilters[filter_index][filter_id][k])));
+    }
+
+    const int16x8_t v_inter_round_bits_0 =
+        vdupq_n_s16(-kInterRoundBitsHorizontal);
+    const int16x8_t v_bits_shift = vdupq_n_s16(bits_shift);
+
+    const uint16x8_t v_compound_round_offset =
+        vdupq_n_u16(compound_round_offset - (1 << (kBitdepth8 + 3)));
+
+    if (filter_index == 2) {  // 8 tap.
+      ConvolveCompoundHorizontalBlock<8, 8, 2>(
+          src, src_stride, dest, pred_stride, width, height, v_tap,
+          v_inter_round_bits_0, v_bits_shift, v_compound_round_offset);
+    } else if (filter_index == 1) {  // 6 tap.
+      // Check if outside taps are positive.
+      if ((filter_id == 1) | (filter_id == 15)) {
+        ConvolveCompoundHorizontalBlock<6, 8, 1, false>(
+            src, src_stride, dest, pred_stride, width, height, v_tap,
+            v_inter_round_bits_0, v_bits_shift, v_compound_round_offset);
+      } else {
+        ConvolveCompoundHorizontalBlock<6, 8, 1>(
+            src, src_stride, dest, pred_stride, width, height, v_tap,
+            v_inter_round_bits_0, v_bits_shift, v_compound_round_offset);
+      }
+    } else if (filter_index == 0) {  // 6 tap.
+      ConvolveCompoundHorizontalBlock<6, 8, 0>(
+          src, src_stride, dest, pred_stride, width, height, v_tap,
+          v_inter_round_bits_0, v_bits_shift, v_compound_round_offset);
+    } else if (filter_index == 4) {  // 4 tap.
+      ConvolveCompoundHorizontalBlock<4, 8, 4>(
+          src, src_stride, dest, pred_stride, width, height, v_tap,
+          v_inter_round_bits_0, v_bits_shift, v_compound_round_offset);
+    } else if (filter_index == 5) {  // 4 tap.
+      ConvolveCompoundHorizontalBlock<4, 8, 5>(
+          src, src_stride, dest, pred_stride, width, height, v_tap,
+          v_inter_round_bits_0, v_bits_shift, v_compound_round_offset);
+    } else {  // 2 tap.
+      ConvolveCompoundHorizontalBlock<2, 8, 3>(
+          src, src_stride, dest, pred_stride, width, height, v_tap,
+          v_inter_round_bits_0, v_bits_shift, v_compound_round_offset);
+    }
+  } else {
+    // 2xH
+    int y = 0;
+    do {
+      for (int x = 0; x < 2; ++x) {
+        int sum = 0;
+        for (int k = 0; k < kSubPixelTaps; ++k) {
+          sum += kSubPixelFilters[filter_index][filter_id][k] * src[x + k];
+        }
+        sum = RightShiftWithRounding(sum, kInterRoundBitsHorizontal)
+              << bits_shift;
+        dest[x] = sum + compound_round_offset;
+      }
+      src += src_stride;
+      dest += pred_stride;
+    } while (++y < height);
+  }
+}
+
+void Init8bpp() {
+  Dsp* const dsp = dsp_internal::GetWritableDspTable(8);
+  assert(dsp != nullptr);
+  dsp->convolve[0][0][0][1] = ConvolveHorizontal_NEON;
+  dsp->convolve[0][0][1][0] = ConvolveVertical_NEON;
+  // TODO(b/139707209): reenable after segfault on android is fixed.
+  // dsp->convolve[0][0][1][1] = Convolve2D_NEON;
+  static_cast<void>(Convolve2D_NEON);
+
+  dsp->convolve[0][1][0][0] = ConvolveCompoundCopy_NEON;
+  dsp->convolve[0][1][0][1] = ConvolveCompoundHorizontal_NEON;
+  dsp->convolve[0][1][1][0] = ConvolveCompoundVertical_NEON;
+
+  // dsp->convolve_scale[1] = ConvolveCompoundScale2D_NEON;
+  static_cast<void>(ConvolveCompoundScale2D_NEON);
+}
+
+}  // namespace
+}  // namespace low_bitdepth
+
+void ConvolveInit_NEON() { low_bitdepth::Init8bpp(); }
+
+}  // namespace dsp
+}  // namespace libgav1
+
+#else   // !LIBGAV1_ENABLE_NEON
+
+namespace libgav1 {
+namespace dsp {
+
+void ConvolveInit_NEON() {}
+
+}  // namespace dsp
+}  // namespace libgav1
+#endif  // LIBGAV1_ENABLE_NEON
diff --git a/libgav1/src/dsp/arm/convolve_neon.h b/libgav1/src/dsp/arm/convolve_neon.h
new file mode 100644
index 0000000..6b5873c
--- /dev/null
+++ b/libgav1/src/dsp/arm/convolve_neon.h
@@ -0,0 +1,29 @@
+#ifndef LIBGAV1_SRC_DSP_ARM_CONVOLVE_NEON_H_
+#define LIBGAV1_SRC_DSP_ARM_CONVOLVE_NEON_H_
+
+#include "src/dsp/cpu.h"
+#include "src/dsp/dsp.h"
+
+namespace libgav1 {
+namespace dsp {
+
+// Initializes Dsp::convolve. This function is not thread-safe.
+void ConvolveInit_NEON();
+
+}  // namespace dsp
+}  // namespace libgav1
+
+#if LIBGAV1_ENABLE_NEON
+#define LIBGAV1_Dsp8bpp_ConvolveHorizontal LIBGAV1_DSP_NEON
+#define LIBGAV1_Dsp8bpp_ConvolveVertical LIBGAV1_DSP_NEON
+// TODO(b/139707209): reenable after segfault on android is fixed.
+// #define LIBGAV1_Dsp8bpp_Convolve2D LIBGAV1_DSP_NEON
+
+#define LIBGAV1_Dsp8bpp_ConvolveCompoundCopy LIBGAV1_DSP_NEON
+#define LIBGAV1_Dsp8bpp_ConvolveCompoundHorizontal LIBGAV1_DSP_NEON
+#define LIBGAV1_Dsp8bpp_ConvolveCompoundVertical LIBGAV1_DSP_NEON
+
+// #define LIBGAV1_Dsp8bpp_ConvolveCompoundScale2D LIBGAV1_DSP_NEON
+#endif  // LIBGAV1_ENABLE_NEON
+
+#endif  // LIBGAV1_SRC_DSP_ARM_CONVOLVE_NEON_H_
diff --git a/libgav1/src/dsp/arm/distance_weighted_blend_neon.cc b/libgav1/src/dsp/arm/distance_weighted_blend_neon.cc
new file mode 100644
index 0000000..d0a95a2
--- /dev/null
+++ b/libgav1/src/dsp/arm/distance_weighted_blend_neon.cc
@@ -0,0 +1,237 @@
+#include "src/dsp/distance_weighted_blend.h"
+#include "src/dsp/dsp.h"
+
+#if LIBGAV1_ENABLE_NEON
+
+#include <arm_neon.h>
+
+#include <cassert>
+#include <cstddef>
+#include <cstdint>
+
+#include "src/dsp/arm/common_neon.h"
+#include "src/utils/common.h"
+
+namespace libgav1 {
+namespace dsp {
+namespace {
+
+constexpr int kBitdepth8 = 8;
+constexpr int kInterPostRoundBit = 4;
+
+const int16x8_t kCompoundRoundOffset =
+    vdupq_n_s16((1 << kBitdepth8) + (1 << (kBitdepth8 - 1)));
+
+inline int16x8_t ComputeWeightedAverage8(const uint16x8_t pred0,
+                                         const uint16x8_t pred1,
+                                         const uint16x4_t weights[2]) {
+  const uint32x4_t wpred0_lo = vmull_u16(weights[0], vget_low_u16(pred0));
+  const uint32x4_t wpred0_hi = vmull_u16(weights[0], vget_high_u16(pred0));
+  const uint32x4_t blended_lo =
+      vmlal_u16(wpred0_lo, weights[1], vget_low_u16(pred1));
+  const uint32x4_t blended_hi =
+      vmlal_u16(wpred0_hi, weights[1], vget_high_u16(pred1));
+
+  const uint16x4_t result_lo =
+      vqrshrn_n_u32(blended_lo, kInterPostRoundBit + 4);
+  const uint16x4_t result_hi =
+      vqrshrn_n_u32(blended_hi, kInterPostRoundBit + 4);
+  return vsubq_s16(vreinterpretq_s16_u16(vcombine_u16(result_lo, result_hi)),
+                   kCompoundRoundOffset);
+}
+
+template <int height>
+inline void DistanceWeightedBlend4xH_NEON(const uint16_t* prediction_0,
+                                          const ptrdiff_t prediction_stride_0,
+                                          const uint16_t* prediction_1,
+                                          const ptrdiff_t prediction_stride_1,
+                                          const uint16x4_t weights[2],
+                                          void* const dest,
+                                          const ptrdiff_t dest_stride) {
+  auto* dst = static_cast<uint8_t*>(dest);
+  const uint16_t* pred_0 = prediction_0;
+  const uint16_t* pred_1 = prediction_1;
+
+  for (int y = 0; y < height; y += 4) {
+    const uint16x4_t src_00 = vld1_u16(pred_0);
+    const uint16x4_t src_10 = vld1_u16(pred_1);
+    pred_0 += prediction_stride_0;
+    pred_1 += prediction_stride_1;
+    const uint16x4_t src_01 = vld1_u16(pred_0);
+    const uint16x4_t src_11 = vld1_u16(pred_1);
+    pred_0 += prediction_stride_0;
+    pred_1 += prediction_stride_1;
+    const int16x8_t res01 = ComputeWeightedAverage8(
+        vcombine_u16(src_00, src_01), vcombine_u16(src_10, src_11), weights);
+
+    const uint16x4_t src_02 = vld1_u16(pred_0);
+    const uint16x4_t src_12 = vld1_u16(pred_1);
+    pred_0 += prediction_stride_0;
+    pred_1 += prediction_stride_1;
+    const uint16x4_t src_03 = vld1_u16(pred_0);
+    const uint16x4_t src_13 = vld1_u16(pred_1);
+    pred_0 += prediction_stride_0;
+    pred_1 += prediction_stride_1;
+    const int16x8_t res23 = ComputeWeightedAverage8(
+        vcombine_u16(src_02, src_03), vcombine_u16(src_12, src_13), weights);
+
+    const uint8x8_t result_01 = vqmovun_s16(res01);
+    const uint8x8_t result_23 = vqmovun_s16(res23);
+    StoreLo4(dst, result_01);
+    dst += dest_stride;
+    StoreHi4(dst, result_01);
+    dst += dest_stride;
+    StoreLo4(dst, result_23);
+    dst += dest_stride;
+    StoreHi4(dst, result_23);
+    dst += dest_stride;
+  }
+}
+
+template <int height>
+inline void DistanceWeightedBlend8xH_NEON(const uint16_t* prediction_0,
+                                          const ptrdiff_t prediction_stride_0,
+                                          const uint16_t* prediction_1,
+                                          const ptrdiff_t prediction_stride_1,
+                                          const uint16x4_t weights[2],
+                                          void* const dest,
+                                          const ptrdiff_t dest_stride) {
+  auto* dst = static_cast<uint8_t*>(dest);
+  const uint16_t* pred_0 = prediction_0;
+  const uint16_t* pred_1 = prediction_1;
+
+  for (int y = 0; y < height; y += 2) {
+    const uint16x8_t src_00 = vld1q_u16(pred_0);
+    const uint16x8_t src_10 = vld1q_u16(pred_1);
+    pred_0 += prediction_stride_0;
+    pred_1 += prediction_stride_1;
+    const int16x8_t res0 = ComputeWeightedAverage8(src_00, src_10, weights);
+
+    const uint16x8_t src_01 = vld1q_u16(pred_0);
+    const uint16x8_t src_11 = vld1q_u16(pred_1);
+    pred_0 += prediction_stride_0;
+    pred_1 += prediction_stride_1;
+    const int16x8_t res1 = ComputeWeightedAverage8(src_01, src_11, weights);
+
+    const uint8x8_t result0 = vqmovun_s16(res0);
+    const uint8x8_t result1 = vqmovun_s16(res1);
+    vst1_u8(dst, result0);
+    dst += dest_stride;
+    vst1_u8(dst, result1);
+    dst += dest_stride;
+  }
+}
+
+inline void DistanceWeightedBlendLarge_NEON(
+    const uint16_t* prediction_0, const ptrdiff_t prediction_stride_0,
+    const uint16_t* prediction_1, const ptrdiff_t prediction_stride_1,
+    const uint16x4_t weights[2], const int width, const int height,
+    void* const dest, const ptrdiff_t dest_stride) {
+  auto* dst = static_cast<uint8_t*>(dest);
+  const uint16_t* pred_0 = prediction_0;
+  const uint16_t* pred_1 = prediction_1;
+
+  int y = height;
+  do {
+    int x = 0;
+    do {
+      const uint16x8_t src0_lo = vld1q_u16(pred_0 + x);
+      const uint16x8_t src1_lo = vld1q_u16(pred_1 + x);
+      const int16x8_t res_lo =
+          ComputeWeightedAverage8(src0_lo, src1_lo, weights);
+
+      const uint16x8_t src0_hi = vld1q_u16(pred_0 + x + 8);
+      const uint16x8_t src1_hi = vld1q_u16(pred_1 + x + 8);
+      const int16x8_t res_hi =
+          ComputeWeightedAverage8(src0_hi, src1_hi, weights);
+
+      const uint8x16_t result =
+          vcombine_u8(vqmovun_s16(res_lo), vqmovun_s16(res_hi));
+      vst1q_u8(dst + x, result);
+      x += 16;
+    } while (x < width);
+    dst += dest_stride;
+    pred_0 += prediction_stride_0;
+    pred_1 += prediction_stride_1;
+  } while (--y != 0);
+}
+
+inline void DistanceWeightedBlend_NEON(
+    const uint16_t* prediction_0, const ptrdiff_t prediction_stride_0,
+    const uint16_t* prediction_1, const ptrdiff_t prediction_stride_1,
+    const uint8_t weight_0, const uint8_t weight_1, const int width,
+    const int height, void* const dest, const ptrdiff_t dest_stride) {
+  uint16x4_t weights[2] = {vdup_n_u16(weight_0), vdup_n_u16(weight_1)};
+  if (width == 4) {
+    if (height == 4) {
+      DistanceWeightedBlend4xH_NEON<4>(prediction_0, prediction_stride_0,
+                                       prediction_1, prediction_stride_1,
+                                       weights, dest, dest_stride);
+    } else if (height == 8) {
+      DistanceWeightedBlend4xH_NEON<8>(prediction_0, prediction_stride_0,
+                                       prediction_1, prediction_stride_1,
+                                       weights, dest, dest_stride);
+    } else {
+      assert(height == 16);
+      DistanceWeightedBlend4xH_NEON<16>(prediction_0, prediction_stride_0,
+                                        prediction_1, prediction_stride_1,
+                                        weights, dest, dest_stride);
+    }
+    return;
+  }
+
+  if (width == 8) {
+    switch (height) {
+      case 4:
+        DistanceWeightedBlend8xH_NEON<4>(prediction_0, prediction_stride_0,
+                                         prediction_1, prediction_stride_1,
+                                         weights, dest, dest_stride);
+        return;
+      case 8:
+        DistanceWeightedBlend8xH_NEON<8>(prediction_0, prediction_stride_0,
+                                         prediction_1, prediction_stride_1,
+                                         weights, dest, dest_stride);
+        return;
+      case 16:
+        DistanceWeightedBlend8xH_NEON<16>(prediction_0, prediction_stride_0,
+                                          prediction_1, prediction_stride_1,
+                                          weights, dest, dest_stride);
+        return;
+      default:
+        assert(height == 32);
+        DistanceWeightedBlend8xH_NEON<32>(prediction_0, prediction_stride_0,
+                                          prediction_1, prediction_stride_1,
+                                          weights, dest, dest_stride);
+
+        return;
+    }
+  }
+
+  DistanceWeightedBlendLarge_NEON(prediction_0, prediction_stride_0,
+                                  prediction_1, prediction_stride_1, weights,
+                                  width, height, dest, dest_stride);
+}
+
+void Init8bpp() {
+  Dsp* const dsp = dsp_internal::GetWritableDspTable(8);
+  assert(dsp != nullptr);
+  dsp->distance_weighted_blend = DistanceWeightedBlend_NEON;
+}
+
+}  // namespace
+
+void DistanceWeightedBlendInit_NEON() { Init8bpp(); }
+
+}  // namespace dsp
+}  // namespace libgav1
+
+#else   // !LIBGAV1_ENABLE_NEON
+
+namespace libgav1 {
+namespace dsp {
+
+void DistanceWeightedBlendInit_NEON() {}
+
+}  // namespace dsp
+}  // namespace libgav1
+#endif  // LIBGAV1_ENABLE_NEON
diff --git a/libgav1/src/dsp/arm/distance_weighted_blend_neon.h b/libgav1/src/dsp/arm/distance_weighted_blend_neon.h
new file mode 100644
index 0000000..846b7e1
--- /dev/null
+++ b/libgav1/src/dsp/arm/distance_weighted_blend_neon.h
@@ -0,0 +1,23 @@
+#ifndef LIBGAV1_SRC_DSP_ARM_DISTANCE_WEIGHTED_BLEND_NEON_H_
+#define LIBGAV1_SRC_DSP_ARM_DISTANCE_WEIGHTED_BLEND_NEON_H_
+
+#include "src/dsp/cpu.h"
+#include "src/dsp/dsp.h"
+
+namespace libgav1 {
+namespace dsp {
+
+// Initializes Dsp::distance_weighted_blend. This function is not thread-safe.
+void DistanceWeightedBlendInit_NEON();
+
+}  // namespace dsp
+}  // namespace libgav1
+
+// If NEON is enabled signal the NEON implementation should be used instead of
+// normal C.
+#if LIBGAV1_ENABLE_NEON
+#define LIBGAV1_Dsp8bpp_DistanceWeightedBlend LIBGAV1_DSP_NEON
+
+#endif  // LIBGAV1_ENABLE_NEON
+
+#endif  // LIBGAV1_SRC_DSP_ARM_DISTANCE_WEIGHTED_BLEND_NEON_H_
diff --git a/libgav1/src/dsp/arm/intra_edge_neon.cc b/libgav1/src/dsp/arm/intra_edge_neon.cc
new file mode 100644
index 0000000..3cc70b3
--- /dev/null
+++ b/libgav1/src/dsp/arm/intra_edge_neon.cc
@@ -0,0 +1,285 @@
+#include "src/dsp/dsp.h"
+#include "src/dsp/intra_edge.h"
+
+#if LIBGAV1_ENABLE_NEON
+
+#include <arm_neon.h>
+
+#include <algorithm>
+#include <cassert>
+
+#include "src/dsp/arm/common_neon.h"
+#include "src/utils/common.h"  // RightShiftWithRounding()
+
+namespace libgav1 {
+namespace dsp {
+namespace {
+
+// Simplified version of intra_edge.cc:kKernels[][]. Only |strength| 1 and 2 are
+// required.
+constexpr int kKernelsNEON[3][2] = {{4, 8}, {5, 6}};
+
+void IntraEdgeFilter_NEON(void* buffer, const int size, const int strength) {
+  assert(strength == 1 || strength == 2 || strength == 3);
+  const int kernel_index = strength - 1;
+  auto* const dst_buffer = static_cast<uint8_t*>(buffer);
+
+  // The first element is not written out (but it is input) so the number of
+  // elements written is |size| - 1.
+  if (size == 1) return;
+
+  // |strength| 1 and 2 use a 3 tap filter.
+  if (strength < 3) {
+    // The last value requires extending the buffer (duplicating
+    // |dst_buffer[size - 1]). Calculate it here to avoid extra processing in
+    // neon.
+    const uint8_t last_val = RightShiftWithRounding(
+        kKernelsNEON[kernel_index][0] * dst_buffer[size - 2] +
+            kKernelsNEON[kernel_index][1] * dst_buffer[size - 1] +
+            kKernelsNEON[kernel_index][0] * dst_buffer[size - 1],
+        4);
+
+    const uint8x8_t krn1 = vdup_n_u8(kKernelsNEON[kernel_index][1]);
+
+    // The first value we need gets overwritten by the output from the
+    // previous iteration.
+    uint8x16_t src_0 = vld1q_u8(dst_buffer);
+    int i = 1;
+
+    // Process blocks until there are less than 16 values remaining.
+    for (; i < size - 15; i += 16) {
+      // Loading these at the end of the block with |src_0| will read past the
+      // end of |top_row_data[160]|, the source of |buffer|.
+      const uint8x16_t src_1 = vld1q_u8(dst_buffer + i);
+      const uint8x16_t src_2 = vld1q_u8(dst_buffer + i + 1);
+      uint16x8_t sum_lo = vaddl_u8(vget_low_u8(src_0), vget_low_u8(src_2));
+      sum_lo = vmulq_n_u16(sum_lo, kKernelsNEON[kernel_index][0]);
+      sum_lo = vmlal_u8(sum_lo, vget_low_u8(src_1), krn1);
+      uint16x8_t sum_hi = vaddl_u8(vget_high_u8(src_0), vget_high_u8(src_2));
+      sum_hi = vmulq_n_u16(sum_hi, kKernelsNEON[kernel_index][0]);
+      sum_hi = vmlal_u8(sum_hi, vget_high_u8(src_1), krn1);
+
+      const uint8x16_t result =
+          vcombine_u8(vrshrn_n_u16(sum_lo, 4), vrshrn_n_u16(sum_hi, 4));
+
+      // Load the next row before overwriting. This loads an extra 15 values
+      // past |size| on the trailing iteration.
+      src_0 = vld1q_u8(dst_buffer + i + 15);
+
+      vst1q_u8(dst_buffer + i, result);
+    }
+
+    // The last output value |last_val| was already calculated so if
+    // |remainder| == 1 then we don't have to do anything.
+    const int remainder = (size - 1) & 0xf;
+    if (remainder > 1) {
+      uint8_t temp[16];
+      const uint8x16_t src_1 = vld1q_u8(dst_buffer + i);
+      const uint8x16_t src_2 = vld1q_u8(dst_buffer + i + 1);
+
+      uint16x8_t sum_lo = vaddl_u8(vget_low_u8(src_0), vget_low_u8(src_2));
+      sum_lo = vmulq_n_u16(sum_lo, kKernelsNEON[kernel_index][0]);
+      sum_lo = vmlal_u8(sum_lo, vget_low_u8(src_1), krn1);
+      uint16x8_t sum_hi = vaddl_u8(vget_high_u8(src_0), vget_high_u8(src_2));
+      sum_hi = vmulq_n_u16(sum_hi, kKernelsNEON[kernel_index][0]);
+      sum_hi = vmlal_u8(sum_hi, vget_high_u8(src_1), krn1);
+
+      const uint8x16_t result =
+          vcombine_u8(vrshrn_n_u16(sum_lo, 4), vrshrn_n_u16(sum_hi, 4));
+
+      vst1q_u8(temp, result);
+      memcpy(dst_buffer + i, temp, remainder);
+    }
+
+    dst_buffer[size - 1] = last_val;
+    return;
+  }
+
+  assert(strength == 3);
+  // 5 tap filter. The first element requires duplicating |buffer[0]| and the
+  // last two elements require duplicating |buffer[size - 1]|.
+  uint8_t special_vals[3];
+  special_vals[0] = RightShiftWithRounding(
+      (dst_buffer[0] << 1) + (dst_buffer[0] << 2) + (dst_buffer[1] << 2) +
+          (dst_buffer[2] << 2) + (dst_buffer[3] << 1),
+      4);
+  // Clamp index for very small |size| values.
+  const int first_index_min = std::max(size - 4, 0);
+  const int second_index_min = std::max(size - 3, 0);
+  const int third_index_min = std::max(size - 2, 0);
+  special_vals[1] = RightShiftWithRounding(
+      (dst_buffer[first_index_min] << 1) + (dst_buffer[second_index_min] << 2) +
+          (dst_buffer[third_index_min] << 2) + (dst_buffer[size - 1] << 2) +
+          (dst_buffer[size - 1] << 1),
+      4);
+  special_vals[2] = RightShiftWithRounding(
+      (dst_buffer[second_index_min] << 1) + (dst_buffer[third_index_min] << 2) +
+          // x << 2 + x << 2 == x << 3
+          (dst_buffer[size - 1] << 3) + (dst_buffer[size - 1] << 1),
+      4);
+
+  // The first two values we need get overwritten by the output from the
+  // previous iteration.
+  uint8x16_t src_0 = vld1q_u8(dst_buffer - 1);
+  uint8x16_t src_1 = vld1q_u8(dst_buffer);
+  int i = 1;
+
+  for (; i < size - 15; i += 16) {
+    // Loading these at the end of the block with |src_[01]| will read past
+    // the end of |top_row_data[160]|, the source of |buffer|.
+    const uint8x16_t src_2 = vld1q_u8(dst_buffer + i);
+    const uint8x16_t src_3 = vld1q_u8(dst_buffer + i + 1);
+    const uint8x16_t src_4 = vld1q_u8(dst_buffer + i + 2);
+
+    uint16x8_t sum_lo =
+        vshlq_n_u16(vaddl_u8(vget_low_u8(src_0), vget_low_u8(src_4)), 1);
+    const uint16x8_t sum_123_lo = vaddw_u8(
+        vaddl_u8(vget_low_u8(src_1), vget_low_u8(src_2)), vget_low_u8(src_3));
+    sum_lo = vaddq_u16(sum_lo, vshlq_n_u16(sum_123_lo, 2));
+
+    uint16x8_t sum_hi =
+        vshlq_n_u16(vaddl_u8(vget_high_u8(src_0), vget_high_u8(src_4)), 1);
+    const uint16x8_t sum_123_hi =
+        vaddw_u8(vaddl_u8(vget_high_u8(src_1), vget_high_u8(src_2)),
+                 vget_high_u8(src_3));
+    sum_hi = vaddq_u16(sum_hi, vshlq_n_u16(sum_123_hi, 2));
+
+    const uint8x16_t result =
+        vcombine_u8(vrshrn_n_u16(sum_lo, 4), vrshrn_n_u16(sum_hi, 4));
+
+    src_0 = vld1q_u8(dst_buffer + i + 14);
+    src_1 = vld1q_u8(dst_buffer + i + 15);
+
+    vst1q_u8(dst_buffer + i, result);
+  }
+
+  const int remainder = (size - 1) & 0xf;
+  // Like the 3 tap but if there are two remaining values we have already
+  // calculated them.
+  if (remainder > 2) {
+    uint8_t temp[16];
+    const uint8x16_t src_2 = vld1q_u8(dst_buffer + i);
+    const uint8x16_t src_3 = vld1q_u8(dst_buffer + i + 1);
+    const uint8x16_t src_4 = vld1q_u8(dst_buffer + i + 2);
+
+    uint16x8_t sum_lo =
+        vshlq_n_u16(vaddl_u8(vget_low_u8(src_0), vget_low_u8(src_4)), 1);
+    const uint16x8_t sum_123_lo = vaddw_u8(
+        vaddl_u8(vget_low_u8(src_1), vget_low_u8(src_2)), vget_low_u8(src_3));
+    sum_lo = vaddq_u16(sum_lo, vshlq_n_u16(sum_123_lo, 2));
+
+    uint16x8_t sum_hi =
+        vshlq_n_u16(vaddl_u8(vget_high_u8(src_0), vget_high_u8(src_4)), 1);
+    const uint16x8_t sum_123_hi =
+        vaddw_u8(vaddl_u8(vget_high_u8(src_1), vget_high_u8(src_2)),
+                 vget_high_u8(src_3));
+    sum_hi = vaddq_u16(sum_hi, vshlq_n_u16(sum_123_hi, 2));
+
+    const uint8x16_t result =
+        vcombine_u8(vrshrn_n_u16(sum_lo, 4), vrshrn_n_u16(sum_hi, 4));
+
+    vst1q_u8(temp, result);
+    memcpy(dst_buffer + i, temp, remainder);
+  }
+
+  dst_buffer[1] = special_vals[0];
+  // Avoid overwriting |dst_buffer[0]|.
+  if (size > 2) dst_buffer[size - 2] = special_vals[1];
+  dst_buffer[size - 1] = special_vals[2];
+}
+
+// (-|src0| + |src1| * 9 + |src2| * 9 - |src3|) >> 4
+uint8x8_t Upsample(const uint8x8_t src0, const uint8x8_t src1,
+                   const uint8x8_t src2, const uint8x8_t src3) {
+  const uint16x8_t middle = vmulq_n_u16(vaddl_u8(src1, src2), 9);
+  const uint16x8_t ends = vaddl_u8(src0, src3);
+  const int16x8_t sum =
+      vsubq_s16(vreinterpretq_s16_u16(middle), vreinterpretq_s16_u16(ends));
+  return vqrshrun_n_s16(sum, 4);
+}
+
+void IntraEdgeUpsampler_NEON(void* buffer, const int size) {
+  assert(size % 4 == 0 && size <= 16);
+  auto* const pixel_buffer = static_cast<uint8_t*>(buffer);
+  // This is OK because we don't read this value for |size| 4 or 8 but if we
+  // write |pixel_buffer[size]| and then vld() it, that seems to introduce
+  // some latency.
+  pixel_buffer[-2] = pixel_buffer[-1];
+  if (size == 4) {
+    // This uses one load and two vtbl() which is better than 4x Load{Lo,Hi}4().
+    const uint8x8_t src = vld1_u8(pixel_buffer - 1);
+    // The outside values are negated so put those in the same vector.
+    const uint8x8_t src03 = vtbl1_u8(src, vcreate_u8(0x0404030202010000));
+    // Reverse |src1| and |src2| so we can use |src2| for the interleave at the
+    // end.
+    const uint8x8_t src21 = vtbl1_u8(src, vcreate_u8(0x0302010004030201));
+
+    const uint16x8_t middle = vmull_u8(src21, vdup_n_u8(9));
+    const int16x8_t half_sum = vsubq_s16(
+        vreinterpretq_s16_u16(middle), vreinterpretq_s16_u16(vmovl_u8(src03)));
+    const int16x4_t sum =
+        vadd_s16(vget_low_s16(half_sum), vget_high_s16(half_sum));
+    const uint8x8_t result = vqrshrun_n_s16(vcombine_s16(sum, sum), 4);
+
+    vst1_u8(pixel_buffer - 1, InterleaveLow8(result, src21));
+    return;
+  } else if (size == 8) {
+    // Likewise, one load + multiple vtbls seems preferred to multiple loads.
+    const uint8x16_t src = vld1q_u8(pixel_buffer - 1);
+    const uint8x8_t src0 = VQTbl1U8(src, vcreate_u8(0x0605040302010000));
+    const uint8x8_t src1 = vget_low_u8(src);
+    const uint8x8_t src2 = VQTbl1U8(src, vcreate_u8(0x0807060504030201));
+    const uint8x8_t src3 = VQTbl1U8(src, vcreate_u8(0x0808070605040302));
+
+    const uint8x8x2_t output = {Upsample(src0, src1, src2, src3), src2};
+    vst2_u8(pixel_buffer - 1, output);
+    return;
+  }
+  assert(size == 12 || size == 16);
+  // Extend the input borders to avoid branching later.
+  pixel_buffer[size] = pixel_buffer[size - 1];
+  const uint8x16_t src0 = vld1q_u8(pixel_buffer - 2);
+  const uint8x16_t src1 = vld1q_u8(pixel_buffer - 1);
+  const uint8x16_t src2 = vld1q_u8(pixel_buffer);
+  const uint8x16_t src3 = vld1q_u8(pixel_buffer + 1);
+
+  const uint8x8_t result_lo = Upsample(vget_low_u8(src0), vget_low_u8(src1),
+                                       vget_low_u8(src2), vget_low_u8(src3));
+
+  const uint8x8x2_t output_lo = {result_lo, vget_low_u8(src2)};
+  vst2_u8(pixel_buffer - 1, output_lo);
+
+  const uint8x8_t result_hi = Upsample(vget_high_u8(src0), vget_high_u8(src1),
+                                       vget_high_u8(src2), vget_high_u8(src3));
+
+  if (size == 12) {
+    vst1_u8(pixel_buffer + 15, InterleaveLow8(result_hi, vget_high_u8(src2)));
+  } else /* size == 16 */ {
+    const uint8x8x2_t output_hi = {result_hi, vget_high_u8(src2)};
+    vst2_u8(pixel_buffer + 15, output_hi);
+  }
+}
+
+void Init8bpp() {
+  Dsp* const dsp = dsp_internal::GetWritableDspTable(8);
+  assert(dsp != nullptr);
+  dsp->intra_edge_filter = IntraEdgeFilter_NEON;
+  dsp->intra_edge_upsampler = IntraEdgeUpsampler_NEON;
+}
+
+}  // namespace
+
+void IntraEdgeInit_NEON() { Init8bpp(); }
+
+}  // namespace dsp
+}  // namespace libgav1
+
+#else   // !LIBGAV1_ENABLE_NEON
+namespace libgav1 {
+namespace dsp {
+
+void IntraEdgeInit_NEON() {}
+
+}  // namespace dsp
+}  // namespace libgav1
+#endif  // LIBGAV1_ENABLE_NEON
diff --git a/libgav1/src/dsp/arm/intra_edge_neon.h b/libgav1/src/dsp/arm/intra_edge_neon.h
new file mode 100644
index 0000000..5da3c24
--- /dev/null
+++ b/libgav1/src/dsp/arm/intra_edge_neon.h
@@ -0,0 +1,23 @@
+#ifndef LIBGAV1_SRC_DSP_ARM_INTRA_EDGE_NEON_H_
+#define LIBGAV1_SRC_DSP_ARM_INTRA_EDGE_NEON_H_
+
+#include "src/dsp/cpu.h"
+#include "src/dsp/dsp.h"
+
+namespace libgav1 {
+namespace dsp {
+
+// Initializes Dsp::intra_edge_filter and Dsp::intra_edge_upsampler. This
+// function is not thread-safe.
+void IntraEdgeInit_NEON();
+
+}  // namespace dsp
+}  // namespace libgav1
+
+#if LIBGAV1_ENABLE_NEON
+#define LIBGAV1_Dsp8bpp_IntraEdgeFilter LIBGAV1_DSP_NEON
+#define LIBGAV1_Dsp8bpp_IntraEdgeUpsampler LIBGAV1_DSP_NEON
+
+#endif  // LIBGAV1_ENABLE_NEON
+
+#endif  // LIBGAV1_SRC_DSP_ARM_INTRA_EDGE_NEON_H_
diff --git a/libgav1/src/dsp/arm/intrapred_cfl_neon.cc b/libgav1/src/dsp/arm/intrapred_cfl_neon.cc
index cad9f7b..7947fef 100644
--- a/libgav1/src/dsp/arm/intrapred_cfl_neon.cc
+++ b/libgav1/src/dsp/arm/intrapred_cfl_neon.cc
@@ -1,5 +1,5 @@
-#include "src/dsp/arm/intrapred_neon.h"
 #include "src/dsp/dsp.h"
+#include "src/dsp/intrapred.h"
 
 #if LIBGAV1_ENABLE_NEON
 
@@ -10,12 +10,258 @@
 #include <cstdint>
 
 #include "src/dsp/arm/common_neon.h"
+#include "src/utils/common.h"
 
 namespace libgav1 {
 namespace dsp {
 namespace low_bitdepth {
 namespace {
 
+uint8x16_t Set2ValuesQ(const uint8_t* a) {
+  uint16_t combined_values = a[0] | a[1] << 8;
+  return vreinterpretq_u8_u16(vdupq_n_u16(combined_values));
+}
+
+int SumVector(uint32x2_t a) {
+#if defined(__aarch64__)
+  return vaddv_u32(a);
+#else
+  const uint64x1_t b = vpaddl_u32(a);
+  return vget_lane_u64(b, 0);
+#endif  // defined(__aarch64__)
+}
+
+int SumVector(uint32x4_t a) {
+#if defined(__aarch64__)
+  return vaddvq_u32(a);
+#else
+  const uint64x2_t b = vpaddlq_u32(a);
+  const uint64x1_t c = vadd_u64(vget_low_u64(b), vget_high_u64(b));
+  return vget_lane_u64(c, 0);
+#endif  // defined(__aarch64__)
+}
+
+// Divide by the number of elements.
+int Average(const int sum, const int width, const int height) {
+  return RightShiftWithRounding(sum, FloorLog2(width) + FloorLog2(height));
+}
+
+// Subtract |val| from every element in |a|.
+void BlockSubtract(const int val,
+                   int16_t a[kCflLumaBufferStride][kCflLumaBufferStride],
+                   const int width, const int height) {
+  const int16x8_t val_v = vdupq_n_s16(val);
+
+  for (int y = 0; y < height; ++y) {
+    if (width == 4) {
+      const int16x4_t b = vld1_s16(a[y]);
+      vst1_s16(a[y], vsub_s16(b, vget_low_s16(val_v)));
+    } else if (width == 8) {
+      const int16x8_t b = vld1q_s16(a[y]);
+      vst1q_s16(a[y], vsubq_s16(b, val_v));
+    } else if (width == 16) {
+      const int16x8_t b = vld1q_s16(a[y]);
+      const int16x8_t c = vld1q_s16(a[y] + 8);
+      vst1q_s16(a[y], vsubq_s16(b, val_v));
+      vst1q_s16(a[y] + 8, vsubq_s16(c, val_v));
+    } else /* block_width == 32 */ {
+      const int16x8_t b = vld1q_s16(a[y]);
+      const int16x8_t c = vld1q_s16(a[y] + 8);
+      const int16x8_t d = vld1q_s16(a[y] + 16);
+      const int16x8_t e = vld1q_s16(a[y] + 24);
+      vst1q_s16(a[y], vsubq_s16(b, val_v));
+      vst1q_s16(a[y] + 8, vsubq_s16(c, val_v));
+      vst1q_s16(a[y] + 16, vsubq_s16(d, val_v));
+      vst1q_s16(a[y] + 24, vsubq_s16(e, val_v));
+    }
+  }
+}
+
+template <int block_width, int block_height>
+void CflSubsampler420_NEON(
+    int16_t luma[kCflLumaBufferStride][kCflLumaBufferStride],
+    const int max_luma_width, const int max_luma_height,
+    const void* const source, const ptrdiff_t stride) {
+  const auto* src = static_cast<const uint8_t*>(source);
+  int sum;
+  if (block_width == 4) {
+    assert(max_luma_width >= 8);
+    uint32x2_t running_sum = vdup_n_u32(0);
+
+    for (int y = 0; y < block_height; ++y) {
+      const uint8x8_t row0 = vld1_u8(src);
+      const uint8x8_t row1 = vld1_u8(src + stride);
+
+      uint16x4_t sum_row = vpadal_u8(vpaddl_u8(row0), row1);
+      sum_row = vshl_n_u16(sum_row, 1);
+      running_sum = vpadal_u16(running_sum, sum_row);
+      vst1_s16(luma[y], vreinterpret_s16_u16(sum_row));
+
+      if (y << 1 < max_luma_height - 2) {
+        // Once this threshold is reached the loop could be simplified.
+        src += stride << 1;
+      }
+    }
+
+    sum = SumVector(running_sum);
+  } else if (block_width == 8) {
+    const uint8x16_t x_index = {0, 0, 2,  2,  4,  4,  6,  6,
+                                8, 8, 10, 10, 12, 12, 14, 14};
+    const uint8x16_t x_max_index = vdupq_n_u8(max_luma_width - 2);
+    const uint8x16_t x_mask = vcltq_u8(x_index, x_max_index);
+
+    uint32x4_t running_sum = vdupq_n_u32(0);
+
+    for (int y = 0; y < block_height; ++y) {
+      const uint8x16_t x_max0 = Set2ValuesQ(src + max_luma_width - 2);
+      const uint8x16_t x_max1 = Set2ValuesQ(src + max_luma_width - 2 + stride);
+
+      uint8x16_t row0 = vld1q_u8(src);
+      row0 = vbslq_u8(x_mask, row0, x_max0);
+      uint8x16_t row1 = vld1q_u8(src + stride);
+      row1 = vbslq_u8(x_mask, row1, x_max1);
+
+      uint16x8_t sum_row = vpadalq_u8(vpaddlq_u8(row0), row1);
+      sum_row = vshlq_n_u16(sum_row, 1);
+      running_sum = vpadalq_u16(running_sum, sum_row);
+      vst1q_s16(luma[y], vreinterpretq_s16_u16(sum_row));
+
+      if (y << 1 < max_luma_height - 2) {
+        src += stride << 1;
+      }
+    }
+
+    sum = SumVector(running_sum);
+  } else /* block_width >= 16 */ {
+    const uint8x16_t x_max_index = vdupq_n_u8(max_luma_width - 2);
+    uint32x4_t running_sum = vdupq_n_u32(0);
+
+    for (int y = 0; y < block_height; ++y) {
+      uint8x16_t x_index = {0,  2,  4,  6,  8,  10, 12, 14,
+                            16, 18, 20, 22, 24, 26, 28, 30};
+      const uint8x16_t x_max00 = vdupq_n_u8(src[max_luma_width - 2]);
+      const uint8x16_t x_max01 = vdupq_n_u8(src[max_luma_width - 2 + 1]);
+      const uint8x16_t x_max10 = vdupq_n_u8(src[stride + max_luma_width - 2]);
+      const uint8x16_t x_max11 =
+          vdupq_n_u8(src[stride + max_luma_width - 2 + 1]);
+      for (int x = 0; x < block_width; x += 16) {
+        const ptrdiff_t src_x_offset = x << 1;
+        const uint8x16_t x_mask = vcltq_u8(x_index, x_max_index);
+        const uint8x16x2_t row0 = vld2q_u8(src + src_x_offset);
+        const uint8x16x2_t row1 = vld2q_u8(src + src_x_offset + stride);
+        const uint8x16_t row_masked_00 = vbslq_u8(x_mask, row0.val[0], x_max00);
+        const uint8x16_t row_masked_01 = vbslq_u8(x_mask, row0.val[1], x_max01);
+        const uint8x16_t row_masked_10 = vbslq_u8(x_mask, row1.val[0], x_max10);
+        const uint8x16_t row_masked_11 = vbslq_u8(x_mask, row1.val[1], x_max11);
+
+        uint16x8_t sum_row_lo =
+            vaddl_u8(vget_low_u8(row_masked_00), vget_low_u8(row_masked_01));
+        sum_row_lo = vaddw_u8(sum_row_lo, vget_low_u8(row_masked_10));
+        sum_row_lo = vaddw_u8(sum_row_lo, vget_low_u8(row_masked_11));
+        sum_row_lo = vshlq_n_u16(sum_row_lo, 1);
+        running_sum = vpadalq_u16(running_sum, sum_row_lo);
+        vst1q_s16(luma[y] + x, vreinterpretq_s16_u16(sum_row_lo));
+
+        uint16x8_t sum_row_hi =
+            vaddl_u8(vget_high_u8(row_masked_00), vget_high_u8(row_masked_01));
+        sum_row_hi = vaddw_u8(sum_row_hi, vget_high_u8(row_masked_10));
+        sum_row_hi = vaddw_u8(sum_row_hi, vget_high_u8(row_masked_11));
+        sum_row_hi = vshlq_n_u16(sum_row_hi, 1);
+        running_sum = vpadalq_u16(running_sum, sum_row_hi);
+        vst1q_s16(luma[y] + x + 8, vreinterpretq_s16_u16(sum_row_hi));
+
+        x_index = vaddq_u8(x_index, vdupq_n_u8(32));
+      }
+      if (y << 1 < max_luma_height - 2) {
+        src += stride << 1;
+      }
+    }
+    sum = SumVector(running_sum);
+  }
+
+  const int average = Average(sum, block_width, block_height);
+  BlockSubtract(average, luma, block_width, block_height);
+}
+
+template <int block_width, int block_height>
+void CflSubsampler444_NEON(
+    int16_t luma[kCflLumaBufferStride][kCflLumaBufferStride],
+    const int max_luma_width, const int max_luma_height,
+    const void* const source, const ptrdiff_t stride) {
+  const auto* src = static_cast<const uint8_t*>(source);
+  int sum;
+  if (block_width == 4) {
+    assert(max_luma_width >= 4);
+    uint32x4_t running_sum = vdupq_n_u32(0);
+
+    for (int y = 0; y < block_height; y += 2) {
+      uint8x8_t row = vdup_n_u8(0);
+      row = LoadLo4(src, row);
+      row = LoadHi4(src + stride, row);
+      if (y < (max_luma_height - 1)) {
+        src += stride << 1;
+      }
+
+      const uint16x8_t row_shifted = vshll_n_u8(row, 3);
+      running_sum = vpadalq_u16(running_sum, row_shifted);
+      vst1_s16(luma[y], vreinterpret_s16_u16(vget_low_u16(row_shifted)));
+      vst1_s16(luma[y + 1], vreinterpret_s16_u16(vget_high_u16(row_shifted)));
+    }
+
+    sum = SumVector(running_sum);
+  } else if (block_width == 8) {
+    const uint8x8_t x_index = {0, 1, 2, 3, 4, 5, 6, 7};
+    const uint8x8_t x_max_index = vdup_n_u8(max_luma_width - 1);
+    const uint8x8_t x_mask = vclt_u8(x_index, x_max_index);
+
+    uint32x4_t running_sum = vdupq_n_u32(0);
+
+    for (int y = 0; y < block_height; ++y) {
+      const uint8x8_t x_max = vdup_n_u8(src[max_luma_width - 1]);
+      const uint8x8_t row = vbsl_u8(x_mask, vld1_u8(src), x_max);
+
+      const uint16x8_t row_shifted = vshll_n_u8(row, 3);
+      running_sum = vpadalq_u16(running_sum, row_shifted);
+      vst1q_s16(luma[y], vreinterpretq_s16_u16(row_shifted));
+
+      if (y < max_luma_height - 1) {
+        src += stride;
+      }
+    }
+
+    sum = SumVector(running_sum);
+  } else /* block_width >= 16 */ {
+    const uint8x16_t x_max_index = vdupq_n_u8(max_luma_width - 1);
+    uint32x4_t running_sum = vdupq_n_u32(0);
+
+    for (int y = 0; y < block_height; ++y) {
+      uint8x16_t x_index = {0, 1, 2,  3,  4,  5,  6,  7,
+                            8, 9, 10, 11, 12, 13, 14, 15};
+      const uint8x16_t x_max = vdupq_n_u8(src[max_luma_width - 1]);
+      for (int x = 0; x < block_width; x += 16) {
+        const uint8x16_t x_mask = vcltq_u8(x_index, x_max_index);
+        const uint8x16_t row = vbslq_u8(x_mask, vld1q_u8(src + x), x_max);
+
+        const uint16x8_t row_shifted_low = vshll_n_u8(vget_low_u8(row), 3);
+        const uint16x8_t row_shifted_high = vshll_n_u8(vget_high_u8(row), 3);
+        running_sum = vpadalq_u16(running_sum, row_shifted_low);
+        running_sum = vpadalq_u16(running_sum, row_shifted_high);
+        vst1q_s16(luma[y] + x, vreinterpretq_s16_u16(row_shifted_low));
+        vst1q_s16(luma[y] + x + 8, vreinterpretq_s16_u16(row_shifted_high));
+
+        x_index = vaddq_u8(x_index, vdupq_n_u8(16));
+      }
+      if (y < max_luma_height - 1) {
+        src += stride;
+      }
+    }
+    sum = SumVector(running_sum);
+  }
+
+  const int average = Average(sum, block_width, block_height);
+  BlockSubtract(average, luma, block_width, block_height);
+}
+
 // Saturate |dc + ((alpha * luma) >> 6))| to uint8_t.
 inline uint8x8_t Combine8(const int16x8_t luma, const int alpha,
                           const int16x8_t dc) {
@@ -110,6 +356,70 @@
   Dsp* const dsp = dsp_internal::GetWritableDspTable(8);
   assert(dsp != nullptr);
 
+  dsp->cfl_subsamplers[kTransformSize4x4][kSubsamplingType420] =
+      CflSubsampler420_NEON<4, 4>;
+  dsp->cfl_subsamplers[kTransformSize4x8][kSubsamplingType420] =
+      CflSubsampler420_NEON<4, 8>;
+  dsp->cfl_subsamplers[kTransformSize4x16][kSubsamplingType420] =
+      CflSubsampler420_NEON<4, 16>;
+
+  dsp->cfl_subsamplers[kTransformSize8x4][kSubsamplingType420] =
+      CflSubsampler420_NEON<8, 4>;
+  dsp->cfl_subsamplers[kTransformSize8x8][kSubsamplingType420] =
+      CflSubsampler420_NEON<8, 8>;
+  dsp->cfl_subsamplers[kTransformSize8x16][kSubsamplingType420] =
+      CflSubsampler420_NEON<8, 16>;
+  dsp->cfl_subsamplers[kTransformSize8x32][kSubsamplingType420] =
+      CflSubsampler420_NEON<8, 32>;
+
+  dsp->cfl_subsamplers[kTransformSize16x4][kSubsamplingType420] =
+      CflSubsampler420_NEON<16, 4>;
+  dsp->cfl_subsamplers[kTransformSize16x8][kSubsamplingType420] =
+      CflSubsampler420_NEON<16, 8>;
+  dsp->cfl_subsamplers[kTransformSize16x16][kSubsamplingType420] =
+      CflSubsampler420_NEON<16, 16>;
+  dsp->cfl_subsamplers[kTransformSize16x32][kSubsamplingType420] =
+      CflSubsampler420_NEON<16, 32>;
+
+  dsp->cfl_subsamplers[kTransformSize32x8][kSubsamplingType420] =
+      CflSubsampler420_NEON<32, 8>;
+  dsp->cfl_subsamplers[kTransformSize32x16][kSubsamplingType420] =
+      CflSubsampler420_NEON<32, 16>;
+  dsp->cfl_subsamplers[kTransformSize32x32][kSubsamplingType420] =
+      CflSubsampler420_NEON<32, 32>;
+
+  dsp->cfl_subsamplers[kTransformSize4x4][kSubsamplingType444] =
+      CflSubsampler444_NEON<4, 4>;
+  dsp->cfl_subsamplers[kTransformSize4x8][kSubsamplingType444] =
+      CflSubsampler444_NEON<4, 8>;
+  dsp->cfl_subsamplers[kTransformSize4x16][kSubsamplingType444] =
+      CflSubsampler444_NEON<4, 16>;
+
+  dsp->cfl_subsamplers[kTransformSize8x4][kSubsamplingType444] =
+      CflSubsampler444_NEON<8, 4>;
+  dsp->cfl_subsamplers[kTransformSize8x8][kSubsamplingType444] =
+      CflSubsampler444_NEON<8, 8>;
+  dsp->cfl_subsamplers[kTransformSize8x16][kSubsamplingType444] =
+      CflSubsampler444_NEON<8, 16>;
+  dsp->cfl_subsamplers[kTransformSize8x32][kSubsamplingType444] =
+      CflSubsampler444_NEON<8, 32>;
+
+  dsp->cfl_subsamplers[kTransformSize16x4][kSubsamplingType444] =
+      CflSubsampler444_NEON<16, 4>;
+  dsp->cfl_subsamplers[kTransformSize16x8][kSubsamplingType444] =
+      CflSubsampler444_NEON<16, 8>;
+  dsp->cfl_subsamplers[kTransformSize16x16][kSubsamplingType444] =
+      CflSubsampler444_NEON<16, 16>;
+  dsp->cfl_subsamplers[kTransformSize16x32][kSubsamplingType444] =
+      CflSubsampler444_NEON<16, 32>;
+
+  dsp->cfl_subsamplers[kTransformSize32x8][kSubsamplingType444] =
+      CflSubsampler444_NEON<32, 8>;
+  dsp->cfl_subsamplers[kTransformSize32x16][kSubsamplingType444] =
+      CflSubsampler444_NEON<32, 16>;
+  dsp->cfl_subsamplers[kTransformSize32x32][kSubsamplingType444] =
+      CflSubsampler444_NEON<32, 32>;
+
   dsp->cfl_intra_predictors[kTransformSize4x4] = CflIntraPredictor4xN_NEON<4>;
   dsp->cfl_intra_predictors[kTransformSize4x8] = CflIntraPredictor4xN_NEON<8>;
   dsp->cfl_intra_predictors[kTransformSize4x16] = CflIntraPredictor4xN_NEON<16>;
diff --git a/libgav1/src/dsp/arm/intrapred_directional_neon.cc b/libgav1/src/dsp/arm/intrapred_directional_neon.cc
index e59d8ed..751ba89 100644
--- a/libgav1/src/dsp/arm/intrapred_directional_neon.cc
+++ b/libgav1/src/dsp/arm/intrapred_directional_neon.cc
@@ -1,10 +1,11 @@
-#include "src/dsp/arm/intrapred_neon.h"
 #include "src/dsp/dsp.h"
+#include "src/dsp/intrapred.h"
 
 #if LIBGAV1_ENABLE_NEON
 
 #include <arm_neon.h>
 
+#include <algorithm>  // std::min
 #include <cassert>
 #include <cstddef>
 #include <cstdint>
@@ -31,10 +32,159 @@
 // For vertical operations the weights are one constant value.
 inline uint8x8_t WeightedBlend(const uint8x8_t a, const uint8x8_t b,
                                const uint8_t weight) {
-  const uint16x8_t a_product = vmull_u8(a, vdup_n_u8(32 - weight));
-  const uint16x8_t b_product = vmull_u8(b, vdup_n_u8(weight));
+  return WeightedBlend(a, b, vdup_n_u8(32 - weight), vdup_n_u8(weight));
+}
 
-  return vrshrn_n_u16(vaddq_u16(a_product, b_product), 5);
+// Fill |left| and |right| with the appropriate values for a given |base_step|.
+inline void LoadStepwise(const uint8_t* const source, const uint8x8_t left_step,
+                         const uint8x8_t right_step, uint8x8_t* left,
+                         uint8x8_t* right) {
+  const uint8x16_t mixed = vld1q_u8(source);
+  *left = VQTbl1U8(mixed, left_step);
+  *right = VQTbl1U8(mixed, right_step);
+}
+
+// Handle signed step arguments by ignoring the sign. Negative values are
+// considered out of range and overwritten later.
+inline void LoadStepwise(const uint8_t* const source, const int8x8_t left_step,
+                         const int8x8_t right_step, uint8x8_t* left,
+                         uint8x8_t* right) {
+  LoadStepwise(source, vreinterpret_u8_s8(left_step),
+               vreinterpret_u8_s8(right_step), left, right);
+}
+
+// Process 4 or 8 |width| by any |height|.
+template <int width>
+inline void DirectionalZone1_WxH(uint8_t* dst, const ptrdiff_t stride,
+                                 const int height, const uint8_t* const top,
+                                 const int xstep, const bool upsampled) {
+  assert(width == 4 || width == 8);
+
+  const int upsample_shift = static_cast<int>(upsampled);
+  const int scale_bits = 6 - upsample_shift;
+
+  const int max_base_x = (width + height - 1) << upsample_shift;
+  const int8x8_t max_base = vdup_n_s8(max_base_x);
+  const uint8x8_t top_max_base = vdup_n_u8(top[max_base_x]);
+
+  const int8x8_t all = vcreate_s8(0x0706050403020100);
+  const int8x8_t even = vcreate_s8(0x0e0c0a0806040200);
+  const int8x8_t base_step = upsampled ? even : all;
+  const int8x8_t right_step = vadd_s8(base_step, vdup_n_s8(1));
+
+  int top_x = xstep;
+  int y = 0;
+  do {
+    const int top_base_x = top_x >> scale_bits;
+
+    if (top_base_x >= max_base_x) {
+      for (int i = y; i < height; ++i) {
+        memset(dst, top[max_base_x], 4 /* width */);
+        dst += stride;
+      }
+      return;
+    }
+
+    const uint8_t shift = ((top_x << upsample_shift) & 0x3F) >> 1;
+
+    // Zone2 uses negative values for xstep. Use signed values to compare
+    // |top_base_x| to |max_base_x|.
+    const int8x8_t base_v = vadd_s8(vdup_n_s8(top_base_x), base_step);
+
+    const uint8x8_t max_base_mask = vclt_s8(base_v, max_base);
+
+    // 4 wide subsamples the output. 8 wide subsamples the input.
+    if (width == 4) {
+      const uint8x8_t left_values = vld1_u8(top + top_base_x);
+      const uint8x8_t right_values = RightShift<8>(left_values);
+      const uint8x8_t value = WeightedBlend(left_values, right_values, shift);
+
+      // If |upsampled| is true then extract every other value for output.
+      const uint8x8_t value_stepped =
+          vtbl1_u8(value, vreinterpret_u8_s8(base_step));
+      const uint8x8_t masked_value =
+          vbsl_u8(max_base_mask, value_stepped, top_max_base);
+
+      StoreLo4(dst, masked_value);
+    } else /* width == 8 */ {
+      uint8x8_t left_values, right_values;
+      // WeightedBlend() steps up to Q registers. Downsample the input to avoid
+      // doing extra calculations.
+      LoadStepwise(top + top_base_x, base_step, right_step, &left_values,
+                   &right_values);
+
+      const uint8x8_t value = WeightedBlend(left_values, right_values, shift);
+      const uint8x8_t masked_value =
+          vbsl_u8(max_base_mask, value, top_max_base);
+
+      vst1_u8(dst, masked_value);
+    }
+    dst += stride;
+    top_x += xstep;
+  } while (++y < height);
+}
+
+// Process a multiple of 8 |width| by any |height|. Processes horizontally
+// before vertically in the hopes of being a little more cache friendly.
+inline void DirectionalZone1_WxH(uint8_t* dst, const ptrdiff_t stride,
+                                 const int width, const int height,
+                                 const uint8_t* const top, const int xstep,
+                                 const bool upsampled) {
+  assert(width % 8 == 0);
+  const int upsample_shift = static_cast<int>(upsampled);
+  const int scale_bits = 6 - upsample_shift;
+
+  const int max_base_x = (width + height - 1) << upsample_shift;
+  const int8x8_t max_base = vdup_n_s8(max_base_x);
+  const uint8x8_t top_max_base = vdup_n_u8(top[max_base_x]);
+
+  const int8x8_t all = vcreate_s8(0x0706050403020100);
+  const int8x8_t even = vcreate_s8(0x0e0c0a0806040200);
+  const int8x8_t base_step = upsampled ? even : all;
+  const int8x8_t right_step = vadd_s8(base_step, vdup_n_s8(1));
+  const int8x8_t block_step = vdup_n_s8(8 << upsample_shift);
+
+  int top_x = xstep;
+  int y = 0;
+  do {
+    const int top_base_x = top_x >> scale_bits;
+
+    if (top_base_x >= max_base_x) {
+      for (int i = y; i < height; ++i) {
+        memset(dst, top[max_base_x], 4 /* width */);
+        dst += stride;
+      }
+      return;
+    }
+
+    const uint8_t shift = ((top_x << upsample_shift) & 0x3F) >> 1;
+
+    // Zone2 uses negative values for xstep. Use signed values to compare
+    // |top_base_x| to |max_base_x|.
+    int8x8_t base_v = vadd_s8(vdup_n_s8(top_base_x), base_step);
+
+    int x = 0;
+    do {
+      const uint8x8_t max_base_mask = vclt_s8(base_v, max_base);
+
+      // Extract the input values based on |upsampled| here to avoid doing twice
+      // as many calculations.
+      uint8x8_t left_values, right_values;
+      LoadStepwise(top + top_base_x + x, base_step, right_step, &left_values,
+                   &right_values);
+
+      const uint8x8_t value = WeightedBlend(left_values, right_values, shift);
+      const uint8x8_t masked_value =
+          vbsl_u8(max_base_mask, value, top_max_base);
+
+      vst1_u8(dst + x, masked_value);
+
+      base_v = vadd_s8(base_v, block_step);
+      x += 8;
+    } while (x < width);
+    top_x += xstep;
+    dst += stride;
+  } while (++y < height);
 }
 
 void DirectionalIntraPredictorZone1_NEON(void* const dest,
@@ -49,64 +199,24 @@
   assert(xstep > 0);
 
   const int upsample_shift = static_cast<int>(upsampled_top);
-  const int scale_bits = 6 - upsample_shift;
 
   const uint8x8_t all = vcreate_u8(0x0706050403020100);
-  const uint8x8_t even = vcreate_u8(0x0e0c0a0806040200);
-  const uint8x8_t base_step = upsampled_top ? even : all;
 
   if (xstep == 64) {
     assert(!upsampled_top);
     const uint8_t* top_ptr = top + 1;
-    for (int y = 0; y < height; y += 4) {
+    int y = 0;
+    do {
       memcpy(dst, top_ptr, width);
       memcpy(dst + stride, top_ptr + 1, width);
       memcpy(dst + 2 * stride, top_ptr + 2, width);
       memcpy(dst + 3 * stride, top_ptr + 3, width);
       dst += 4 * stride;
       top_ptr += 4;
-    }
+      y += 4;
+    } while (y < height);
   } else if (width == 4) {
-    const int max_base_x = ((width + height) - 1) << upsample_shift;
-    const uint8x8_t max_base = vdup_n_u8(max_base_x);
-    const uint8x8_t top_max_base = vdup_n_u8(top[max_base_x]);
-
-    for (int y = 0, top_x = xstep; y < height;
-         ++y, dst += stride, top_x += xstep) {
-      const int top_base_x = top_x >> scale_bits;
-
-      if (top_base_x >= max_base_x) {
-        for (int i = y; i < height; ++i) {
-          memset(dst, top[max_base_x], width);
-          dst += stride;
-        }
-        return;
-      }
-
-      const uint8_t shift = ((top_x << upsample_shift) & 0x3F) >> 1;
-      const uint8x8_t shift_mul = vdup_n_u8(shift);
-      const uint8x8_t inv_shift_mul = vdup_n_u8(32 - shift);
-
-      const uint8x8_t base_v = vadd_u8(vdup_n_u8(top_base_x), base_step);
-
-      const uint8x8_t max_base_mask = vclt_u8(base_v, max_base);
-
-      // Load 8 values because we will extract the output values based on
-      // |upsampled_top| at the end.
-      const uint8x8_t left_values = vld1_u8(top + top_base_x);
-      const uint8x8_t right_values = RightShift<8>(left_values);
-
-      const uint8x8_t value =
-          WeightedBlend(left_values, right_values, inv_shift_mul, shift_mul);
-
-      // If |upsampled_top| is true then extract every other value for output.
-      const uint8x8_t value_stepped = vtbl1_u8(value, base_step);
-
-      const uint8x8_t masked_value =
-          vbsl_u8(max_base_mask, value_stepped, top_max_base);
-
-      StoreLo4(dst, masked_value);
-    }
+    DirectionalZone1_WxH<4>(dst, stride, height, top, xstep, upsampled_top);
   } else if (xstep > 51) {
     // 7.11.2.10. Intra edge upsample selection process
     // if ( d <= 0 || d >= 40 ) useUpsample = 0
@@ -120,8 +230,9 @@
     const uint8x8_t top_max_base = vdup_n_u8(top[max_base_x]);
     const uint8x8_t block_step = vdup_n_u8(8);
 
-    for (int y = 0, top_x = xstep; y < height;
-         ++y, dst += stride, top_x += xstep) {
+    int top_x = xstep;
+    int y = 0;
+    do {
       const int top_base_x = top_x >> 6;
 
       if (top_base_x >= max_base_x) {
@@ -133,88 +244,545 @@
       }
 
       const uint8_t shift = ((top_x << upsample_shift) & 0x3F) >> 1;
-      const uint8x8_t shift_mul = vdup_n_u8(shift);
-      const uint8x8_t inv_shift_mul = vdup_n_u8(32 - shift);
 
-      uint8x8_t base_v = vadd_u8(vdup_n_u8(top_base_x), base_step);
+      uint8x8_t base_v = vadd_u8(vdup_n_u8(top_base_x), all);
 
-      for (int x = 0; x < width; x += 8) {
+      int x = 0;
+      do {
         const uint8x8_t max_base_mask = vclt_u8(base_v, max_base);
 
         // Since these |xstep| values can not be upsampled the load is
         // simplified.
         const uint8x8_t left_values = vld1_u8(top + top_base_x + x);
         const uint8x8_t right_values = vld1_u8(top + top_base_x + x + 1);
-
-        const uint8x8_t value =
-            WeightedBlend(left_values, right_values, inv_shift_mul, shift_mul);
-
+        const uint8x8_t value = WeightedBlend(left_values, right_values, shift);
         const uint8x8_t masked_value =
             vbsl_u8(max_base_mask, value, top_max_base);
 
         vst1_u8(dst + x, masked_value);
 
         base_v = vadd_u8(base_v, block_step);
-      }
-    }
+        x += 8;
+      } while (x < width);
+      dst += stride;
+      top_x += xstep;
+    } while (++y < height);
   } else {
-    const int max_base_x = ((width + height) - 1) << upsample_shift;
-    const uint8x8_t max_base = vdup_n_u8(max_base_x);
-    const uint8x8_t top_max_base = vdup_n_u8(top[max_base_x]);
-    const uint8x8_t right_step = vadd_u8(base_step, vdup_n_u8(1));
-    const uint8x8_t block_step = vdup_n_u8(8 << upsample_shift);
-
-    for (int y = 0, top_x = xstep; y < height;
-         ++y, dst += stride, top_x += xstep) {
-      const int top_base_x = top_x >> scale_bits;
-
-      if (top_base_x >= max_base_x) {
-        for (int i = y; i < height; ++i) {
-          memset(dst, top[max_base_x], width);
-          dst += stride;
-        }
-        return;
-      }
-
-      const uint8_t shift = ((top_x << upsample_shift) & 0x3F) >> 1;
-      const uint8x8_t shift_mul = vdup_n_u8(shift);
-      const uint8x8_t inv_shift_mul = vdup_n_u8(32 - shift);
-
-      uint8x8_t base_v = vadd_u8(vdup_n_u8(top_base_x), base_step);
-
-      for (int x = 0; x < width; x += 8) {
-        const uint8x8_t max_base_mask = vclt_u8(base_v, max_base);
-
-        // Extract the input values based on |upsampled_top| here to avoid doing
-        // twice as many calculations.
-        const uint8x16_t mixed_values = vld1q_u8(top + top_base_x + x);
-        const uint8x8_t left_values = vtbl2_u8(
-            {vget_low_u8(mixed_values), vget_high_u8(mixed_values)}, base_step);
-        const uint8x8_t right_values =
-            vtbl2_u8({vget_low_u8(mixed_values), vget_high_u8(mixed_values)},
-                     right_step);
-
-        const uint8x8_t value =
-            WeightedBlend(left_values, right_values, inv_shift_mul, shift_mul);
-
-        const uint8x8_t masked_value =
-            vbsl_u8(max_base_mask, value, top_max_base);
-
-        vst1_u8(dst + x, masked_value);
-
-        base_v = vadd_u8(base_v, block_step);
-      }
-    }
+    DirectionalZone1_WxH(dst, stride, width, height, top, xstep, upsampled_top);
   }
 }
 
-// Fill |left| and |right| with the appropriate values for a given |base_step|.
-inline void LoadStepwise(const uint8_t* const source, const uint8x8_t left_step,
-                         const uint8x8_t right_step, uint8x8_t* left,
-                         uint8x8_t* right) {
-  const uint8x16_t mixed = vld1q_u8(source);
-  *left = vtbl2_u8({vget_low_u8(mixed), vget_high_u8(mixed)}, left_step);
-  *right = vtbl2_u8({vget_low_u8(mixed), vget_high_u8(mixed)}, right_step);
+// Process 4 or 8 |width| by 4 or 8 |height|.
+template <int width>
+inline void DirectionalZone3_WxH(uint8_t* dest, const ptrdiff_t stride,
+                                 const int height,
+                                 const uint8_t* const left_column,
+                                 const int base_left_y, const int ystep,
+                                 const int upsample_shift) {
+  assert(width == 4 || width == 8);
+  assert(height == 4 || height == 8);
+  const int scale_bits = 6 - upsample_shift;
+
+  // Zone3 never runs out of left_column values.
+  assert((width + height - 1) << upsample_shift >  // max_base_y
+         ((ystep * width) >> scale_bits) +
+             (/* base_step */ 1 << upsample_shift) *
+                 (height - 1));  // left_base_y
+
+  // Limited improvement for 8x8. ~20% faster for 64x64.
+  const uint8x8_t all = vcreate_u8(0x0706050403020100);
+  const uint8x8_t even = vcreate_u8(0x0e0c0a0806040200);
+  const uint8x8_t base_step = upsample_shift ? even : all;
+  const uint8x8_t right_step = vadd_u8(base_step, vdup_n_u8(1));
+
+  uint8_t* dst = dest;
+  uint8x8_t left_v[8], right_v[8], value_v[8];
+  const uint8_t* const left = left_column;
+
+  const int index_0 = base_left_y;
+  LoadStepwise(left + (index_0 >> scale_bits), base_step, right_step,
+               &left_v[0], &right_v[0]);
+  value_v[0] = WeightedBlend(left_v[0], right_v[0],
+                             ((index_0 << upsample_shift) & 0x3F) >> 1);
+
+  const int index_1 = base_left_y + ystep;
+  LoadStepwise(left + (index_1 >> scale_bits), base_step, right_step,
+               &left_v[1], &right_v[1]);
+  value_v[1] = WeightedBlend(left_v[1], right_v[1],
+                             ((index_1 << upsample_shift) & 0x3F) >> 1);
+
+  const int index_2 = base_left_y + ystep * 2;
+  LoadStepwise(left + (index_2 >> scale_bits), base_step, right_step,
+               &left_v[2], &right_v[2]);
+  value_v[2] = WeightedBlend(left_v[2], right_v[2],
+                             ((index_2 << upsample_shift) & 0x3F) >> 1);
+
+  const int index_3 = base_left_y + ystep * 3;
+  LoadStepwise(left + (index_3 >> scale_bits), base_step, right_step,
+               &left_v[3], &right_v[3]);
+  value_v[3] = WeightedBlend(left_v[3], right_v[3],
+                             ((index_3 << upsample_shift) & 0x3F) >> 1);
+
+  const int index_4 = base_left_y + ystep * 4;
+  LoadStepwise(left + (index_4 >> scale_bits), base_step, right_step,
+               &left_v[4], &right_v[4]);
+  value_v[4] = WeightedBlend(left_v[4], right_v[4],
+                             ((index_4 << upsample_shift) & 0x3F) >> 1);
+
+  const int index_5 = base_left_y + ystep * 5;
+  LoadStepwise(left + (index_5 >> scale_bits), base_step, right_step,
+               &left_v[5], &right_v[5]);
+  value_v[5] = WeightedBlend(left_v[5], right_v[5],
+                             ((index_5 << upsample_shift) & 0x3F) >> 1);
+
+  const int index_6 = base_left_y + ystep * 6;
+  LoadStepwise(left + (index_6 >> scale_bits), base_step, right_step,
+               &left_v[6], &right_v[6]);
+  value_v[6] = WeightedBlend(left_v[6], right_v[6],
+                             ((index_6 << upsample_shift) & 0x3F) >> 1);
+
+  const int index_7 = base_left_y + ystep * 7;
+  LoadStepwise(left + (index_7 >> scale_bits), base_step, right_step,
+               &left_v[7], &right_v[7]);
+  value_v[7] = WeightedBlend(left_v[7], right_v[7],
+                             ((index_7 << upsample_shift) & 0x3F) >> 1);
+
+  // 8x8 transpose.
+  const uint8x16x2_t b0 = vtrnq_u8(vcombine_u8(value_v[0], value_v[4]),
+                                   vcombine_u8(value_v[1], value_v[5]));
+  const uint8x16x2_t b1 = vtrnq_u8(vcombine_u8(value_v[2], value_v[6]),
+                                   vcombine_u8(value_v[3], value_v[7]));
+
+  const uint16x8x2_t c0 = vtrnq_u16(vreinterpretq_u16_u8(b0.val[0]),
+                                    vreinterpretq_u16_u8(b1.val[0]));
+  const uint16x8x2_t c1 = vtrnq_u16(vreinterpretq_u16_u8(b0.val[1]),
+                                    vreinterpretq_u16_u8(b1.val[1]));
+
+  const uint32x4x2_t d0 = vuzpq_u32(vreinterpretq_u32_u16(c0.val[0]),
+                                    vreinterpretq_u32_u16(c1.val[0]));
+  const uint32x4x2_t d1 = vuzpq_u32(vreinterpretq_u32_u16(c0.val[1]),
+                                    vreinterpretq_u32_u16(c1.val[1]));
+
+  if (width == 4) {
+    StoreLo4(dst, vreinterpret_u8_u32(vget_low_u32(d0.val[0])));
+    dst += stride;
+    StoreLo4(dst, vreinterpret_u8_u32(vget_high_u32(d0.val[0])));
+    dst += stride;
+    StoreLo4(dst, vreinterpret_u8_u32(vget_low_u32(d1.val[0])));
+    dst += stride;
+    StoreLo4(dst, vreinterpret_u8_u32(vget_high_u32(d1.val[0])));
+    if (height == 4) return;
+    dst += stride;
+    StoreLo4(dst, vreinterpret_u8_u32(vget_low_u32(d0.val[1])));
+    dst += stride;
+    StoreLo4(dst, vreinterpret_u8_u32(vget_high_u32(d0.val[1])));
+    dst += stride;
+    StoreLo4(dst, vreinterpret_u8_u32(vget_low_u32(d1.val[1])));
+    dst += stride;
+    StoreLo4(dst, vreinterpret_u8_u32(vget_high_u32(d1.val[1])));
+  } else {
+    vst1_u8(dst, vreinterpret_u8_u32(vget_low_u32(d0.val[0])));
+    dst += stride;
+    vst1_u8(dst, vreinterpret_u8_u32(vget_high_u32(d0.val[0])));
+    dst += stride;
+    vst1_u8(dst, vreinterpret_u8_u32(vget_low_u32(d1.val[0])));
+    dst += stride;
+    vst1_u8(dst, vreinterpret_u8_u32(vget_high_u32(d1.val[0])));
+    if (height == 4) return;
+    dst += stride;
+    vst1_u8(dst, vreinterpret_u8_u32(vget_low_u32(d0.val[1])));
+    dst += stride;
+    vst1_u8(dst, vreinterpret_u8_u32(vget_high_u32(d0.val[1])));
+    dst += stride;
+    vst1_u8(dst, vreinterpret_u8_u32(vget_low_u32(d1.val[1])));
+    dst += stride;
+    vst1_u8(dst, vreinterpret_u8_u32(vget_high_u32(d1.val[1])));
+  }
+}
+
+// Because the source values "move backwards" as the row index increases, the
+// indices derived from ystep are generally negative. This is accommodated by
+// making sure the relative indices are within [-15, 0] when the function is
+// called, and sliding them into the inclusive range [0, 15], relative to a
+// lower base address.
+constexpr int kPositiveIndexOffset = 15;
+
+// Process 4 or 8 |width| by any |height|.
+template <int width>
+inline void DirectionalZone2FromLeftCol_WxH(uint8_t* dst,
+                                            const ptrdiff_t stride,
+                                            const int height,
+                                            const uint8_t* const left_column,
+                                            const int16x8_t left_y,
+                                            const int upsample_shift) {
+  assert(width == 4 || width == 8);
+
+  // The shift argument must be a constant.
+  int16x8_t offset_y, shift_upsampled = left_y;
+  if (upsample_shift) {
+    offset_y = vshrq_n_s16(left_y, 5);
+    shift_upsampled = vshlq_n_s16(shift_upsampled, 1);
+  } else {
+    offset_y = vshrq_n_s16(left_y, 6);
+  }
+
+  // Select values to the left of the starting point.
+  // The 15th element (and 16th) will be all the way at the end, to the right.
+  // With a negative ystep everything else will be "left" of them.
+  // This supports cumulative steps up to 15. We could support up to 16 by doing
+  // separate loads for |left_values| and |right_values|. vtbl supports 2 Q
+  // registers as input which would allow for cumulative offsets of 32.
+  const int16x8_t sampler =
+      vaddq_s16(offset_y, vdupq_n_s16(kPositiveIndexOffset));
+  const uint8x8_t left_values = vqmovun_s16(sampler);
+  const uint8x8_t right_values = vadd_u8(left_values, vdup_n_u8(1));
+
+  const int16x8_t shift_masked = vandq_s16(shift_upsampled, vdupq_n_s16(0x3f));
+  const uint8x8_t shift_mul = vreinterpret_u8_s8(vshrn_n_s16(shift_masked, 1));
+  const uint8x8_t inv_shift_mul = vsub_u8(vdup_n_u8(32), shift_mul);
+
+  int y = 0;
+  do {
+    uint8x8_t src_left, src_right;
+    LoadStepwise(left_column - kPositiveIndexOffset + (y << upsample_shift),
+                 left_values, right_values, &src_left, &src_right);
+    const uint8x8_t val =
+        WeightedBlend(src_left, src_right, inv_shift_mul, shift_mul);
+
+    if (width == 4) {
+      StoreLo4(dst, val);
+    } else {
+      vst1_u8(dst, val);
+    }
+    dst += stride;
+  } while (++y < height);
+}
+
+// Process 4 or 8 |width| by any |height|.
+template <int width>
+inline void DirectionalZone1Blend_WxH(uint8_t* dest, const ptrdiff_t stride,
+                                      const int height,
+                                      const uint8_t* const top_row,
+                                      int zone_bounds, int top_x,
+                                      const int xstep,
+                                      const int upsample_shift) {
+  assert(width == 4 || width == 8);
+
+  const int scale_bits_x = 6 - upsample_shift;
+
+  const uint8x8_t all = vcreate_u8(0x0706050403020100);
+  const uint8x8_t even = vcreate_u8(0x0e0c0a0806040200);
+  const uint8x8_t base_step = upsample_shift ? even : all;
+  const uint8x8_t right_step = vadd_u8(base_step, vdup_n_u8(1));
+
+  int y = 0;
+  do {
+    const uint8_t* const src = top_row + (top_x >> scale_bits_x);
+    uint8x8_t left, right;
+    LoadStepwise(src, base_step, right_step, &left, &right);
+
+    const uint8_t shift = ((top_x << upsample_shift) & 0x3f) >> 1;
+    const uint8x8_t val = WeightedBlend(left, right, shift);
+
+    uint8x8_t dst_blend = vld1_u8(dest);
+    // |zone_bounds| values can be negative.
+    uint8x8_t blend =
+        vcge_s8(vreinterpret_s8_u8(all), vdup_n_s8((zone_bounds >> 6)));
+    uint8x8_t output = vbsl_u8(blend, val, dst_blend);
+
+    if (width == 4) {
+      StoreLo4(dest, output);
+    } else {
+      vst1_u8(dest, output);
+    }
+    dest += stride;
+    zone_bounds += xstep;
+    top_x -= xstep;
+  } while (++y < height);
+}
+
+// The height at which a load of 16 bytes will not contain enough source pixels
+// from |left_column| to supply an accurate row when computing 8 pixels at a
+// time. The values are found by inspection. By coincidence, all angles that
+// satisfy (ystep >> 6) == 2 map to the same value, so it is enough to look up
+// by ystep >> 6. The largest index for this lookup is 1023 >> 6 == 15.
+constexpr int kDirectionalZone2ShuffleInvalidHeight[16] = {
+    1024, 1024, 16, 16, 16, 16, 0, 0, 18, 0, 0, 0, 0, 0, 0, 40};
+
+// 7.11.2.4 (8) 90 < angle > 180
+// The strategy for these functions (4xH and 8+xH) is to know how many blocks
+// can be processed with just pixels from |top_ptr|, then handle mixed blocks,
+// then handle only blocks that take from |left_ptr|. Additionally, a fast
+// index-shuffle approach is used for pred values from |left_column| in sections
+// that permit it.
+inline void DirectionalZone2_4xH(uint8_t* dst, const ptrdiff_t stride,
+                                 const uint8_t* const top_row,
+                                 const uint8_t* const left_column,
+                                 const int height, const int xstep,
+                                 const int ystep, const bool upsampled_top,
+                                 const bool upsampled_left) {
+  const int upsample_left_shift = static_cast<int>(upsampled_left);
+  const int upsample_top_shift = static_cast<int>(upsampled_top);
+
+  // Helper vector.
+  const int16x8_t zero_to_seven = {0, 1, 2, 3, 4, 5, 6, 7};
+
+  // Loop incrementers for moving by block (4xN). Vertical still steps by 8. If
+  // it's only 4, it will be finished in the first iteration.
+  const ptrdiff_t stride8 = stride << 3;
+  const int xstep8 = xstep << 3;
+
+  const int min_height = (height == 4) ? 4 : 8;
+
+  // All columns from |min_top_only_x| to the right will only need |top_row| to
+  // compute and can therefore call the Zone1 functions. This assumes |xstep| is
+  // at least 3.
+  assert(xstep >= 3);
+  const int min_top_only_x = std::min((height * xstep) >> 6, /* width */ 4);
+
+  // For steep angles, the source pixels from |left_column| may not fit in a
+  // 16-byte load for shuffling.
+  // TODO(petersonab): Find a more precise formula for this subject to x.
+  // TODO(johannkoenig): Revisit this for |width| == 4.
+  const int max_shuffle_height =
+      std::min(kDirectionalZone2ShuffleInvalidHeight[ystep >> 6], height);
+
+  // Offsets the original zone bound value to simplify x < (y+1)*xstep/64 -1
+  int xstep_bounds_base = (xstep == 64) ? 0 : xstep - 1;
+
+  const int left_base_increment = ystep >> 6;
+  const int ystep_remainder = ystep & 0x3F;
+
+  // If the 64 scaling is regarded as a decimal point, the first value of the
+  // left_y vector omits the portion which is covered under the left_column
+  // offset. The following values need the full ystep as a relative offset.
+  int16x8_t left_y = vmulq_n_s16(zero_to_seven, -ystep);
+  left_y = vaddq_s16(left_y, vdupq_n_s16(-ystep_remainder));
+
+  // This loop treats each set of 4 columns in 3 stages with y-value boundaries.
+  // The first stage, before the first y-loop, covers blocks that are only
+  // computed from the top row. The second stage, comprising two y-loops, covers
+  // blocks that have a mixture of values computed from top or left. The final
+  // stage covers blocks that are only computed from the left.
+  if (min_top_only_x > 0) {
+    // Round down to the nearest multiple of 8.
+    // TODO(johannkoenig): This never hits for Wx4 blocks but maybe it should.
+    const int max_top_only_y = std::min((1 << 6) / xstep, height) & ~7;
+    DirectionalZone1_WxH<4>(dst, stride, max_top_only_y, top_row, -xstep,
+                            upsampled_top);
+
+    if (max_top_only_y == height) return;
+
+    int y = max_top_only_y;
+    dst += stride * y;
+    const int xstep_y = xstep * y;
+
+    // All rows from |min_left_only_y| down for this set of columns only need
+    // |left_column| to compute.
+    const int min_left_only_y = std::min((4 << 6) / xstep, height);
+    // At high angles such that min_left_only_y < 8, ystep is low and xstep is
+    // high. This means that max_shuffle_height is unbounded and xstep_bounds
+    // will overflow in 16 bits. This is prevented by stopping the first
+    // blending loop at min_left_only_y for such cases, which means we skip over
+    // the second blending loop as well.
+    const int left_shuffle_stop_y =
+        std::min(max_shuffle_height, min_left_only_y);
+    int xstep_bounds = xstep_bounds_base + xstep_y;
+    int top_x = -xstep - xstep_y;
+
+    // +8 increment is OK because if height is 4 this only goes once.
+    for (; y < left_shuffle_stop_y;
+         y += 8, dst += stride8, xstep_bounds += xstep8, top_x -= xstep8) {
+      DirectionalZone2FromLeftCol_WxH<4>(
+          dst, stride, min_height,
+          left_column + ((y - left_base_increment) << upsample_left_shift),
+          left_y, upsample_left_shift);
+
+      DirectionalZone1Blend_WxH<4>(dst, stride, min_height, top_row,
+                                   xstep_bounds, top_x, xstep,
+                                   upsample_top_shift);
+    }
+
+    // Pick up from the last y-value, using the slower but secure method for
+    // left prediction.
+    const int16_t base_left_y = vgetq_lane_s16(left_y, 0);
+    for (; y < min_left_only_y;
+         y += 8, dst += stride8, xstep_bounds += xstep8, top_x -= xstep8) {
+      DirectionalZone3_WxH<4>(
+          dst, stride, min_height,
+          left_column + ((y - left_base_increment) << upsample_left_shift),
+          base_left_y, -ystep, upsample_left_shift);
+
+      DirectionalZone1Blend_WxH<4>(dst, stride, min_height, top_row,
+                                   xstep_bounds, top_x, xstep,
+                                   upsample_top_shift);
+    }
+    // Loop over y for left_only rows.
+    for (; y < height; y += 8, dst += stride8) {
+      DirectionalZone3_WxH<4>(
+          dst, stride, min_height,
+          left_column + ((y - left_base_increment) << upsample_left_shift),
+          base_left_y, -ystep, upsample_left_shift);
+    }
+  } else {
+    DirectionalZone1_WxH<4>(dst, stride, height, top_row, -xstep,
+                            upsampled_top);
+  }
+}
+
+// Process a multiple of 8 |width|.
+inline void DirectionalZone2_8(uint8_t* const dst, const ptrdiff_t stride,
+                               const uint8_t* const top_row,
+                               const uint8_t* const left_column,
+                               const int width, const int height,
+                               const int xstep, const int ystep,
+                               const bool upsampled_top,
+                               const bool upsampled_left) {
+  const int upsample_left_shift = static_cast<int>(upsampled_left);
+  const int upsample_top_shift = static_cast<int>(upsampled_top);
+
+  // Helper vector.
+  const int16x8_t zero_to_seven = {0, 1, 2, 3, 4, 5, 6, 7};
+
+  // Loop incrementers for moving by block (8x8). This function handles blocks
+  // with height 4 as well. They are calculated in one pass so these variables
+  // do not get used.
+  const ptrdiff_t stride8 = stride << 3;
+  const int xstep8 = xstep << 3;
+  const int ystep8 = ystep << 3;
+
+  // Process Wx4 blocks.
+  const int min_height = (height == 4) ? 4 : 8;
+
+  // All columns from |min_top_only_x| to the right will only need |top_row| to
+  // compute and can therefore call the Zone1 functions. This assumes |xstep| is
+  // at least 3.
+  assert(xstep >= 3);
+  const int min_top_only_x = std::min((height * xstep) >> 6, width);
+
+  // For steep angles, the source pixels from |left_column| may not fit in a
+  // 16-byte load for shuffling.
+  // TODO(petersonab): Find a more precise formula for this subject to x.
+  const int max_shuffle_height =
+      std::min(kDirectionalZone2ShuffleInvalidHeight[ystep >> 6], height);
+
+  // Offsets the original zone bound value to simplify x < (y+1)*xstep/64 -1
+  int xstep_bounds_base = (xstep == 64) ? 0 : xstep - 1;
+
+  const int left_base_increment = ystep >> 6;
+  const int ystep_remainder = ystep & 0x3F;
+
+  const int left_base_increment8 = ystep8 >> 6;
+  const int ystep_remainder8 = ystep8 & 0x3F;
+  const int16x8_t increment_left8 = vdupq_n_s16(ystep_remainder8);
+
+  // If the 64 scaling is regarded as a decimal point, the first value of the
+  // left_y vector omits the portion which is covered under the left_column
+  // offset. Following values need the full ystep as a relative offset.
+  int16x8_t left_y = vmulq_n_s16(zero_to_seven, -ystep);
+  left_y = vaddq_s16(left_y, vdupq_n_s16(-ystep_remainder));
+
+  // This loop treats each set of 4 columns in 3 stages with y-value boundaries.
+  // The first stage, before the first y-loop, covers blocks that are only
+  // computed from the top row. The second stage, comprising two y-loops, covers
+  // blocks that have a mixture of values computed from top or left. The final
+  // stage covers blocks that are only computed from the left.
+  int x = 0;
+  for (int left_offset = -left_base_increment; x < min_top_only_x; x += 8,
+           xstep_bounds_base -= (8 << 6),
+           left_y = vsubq_s16(left_y, increment_left8),
+           left_offset -= left_base_increment8) {
+    uint8_t* dst_x = dst + x;
+
+    // Round down to the nearest multiple of 8.
+    const int max_top_only_y = std::min(((x + 1) << 6) / xstep, height) & ~7;
+    DirectionalZone1_WxH<8>(dst_x, stride, max_top_only_y,
+                            top_row + (x << upsample_top_shift), -xstep,
+                            upsampled_top);
+
+    if (max_top_only_y == height) continue;
+
+    int y = max_top_only_y;
+    dst_x += stride * y;
+    const int xstep_y = xstep * y;
+
+    // All rows from |min_left_only_y| down for this set of columns only need
+    // |left_column| to compute.
+    const int min_left_only_y = std::min(((x + 8) << 6) / xstep, height);
+    // At high angles such that min_left_only_y < 8, ystep is low and xstep is
+    // high. This means that max_shuffle_height is unbounded and xstep_bounds
+    // will overflow in 16 bits. This is prevented by stopping the first
+    // blending loop at min_left_only_y for such cases, which means we skip over
+    // the second blending loop as well.
+    const int left_shuffle_stop_y =
+        std::min(max_shuffle_height, min_left_only_y);
+    int xstep_bounds = xstep_bounds_base + xstep_y;
+    int top_x = -xstep - xstep_y;
+
+    for (; y < left_shuffle_stop_y;
+         y += 8, dst_x += stride8, xstep_bounds += xstep8, top_x -= xstep8) {
+      DirectionalZone2FromLeftCol_WxH<8>(
+          dst_x, stride, min_height,
+          left_column + ((left_offset + y) << upsample_left_shift), left_y,
+          upsample_left_shift);
+
+      DirectionalZone1Blend_WxH<8>(
+          dst_x, stride, min_height, top_row + (x << upsample_top_shift),
+          xstep_bounds, top_x, xstep, upsample_top_shift);
+    }
+
+    // Pick up from the last y-value, using the slower but secure method for
+    // left prediction.
+    const int16_t base_left_y = vgetq_lane_s16(left_y, 0);
+    for (; y < min_left_only_y;
+         y += 8, dst_x += stride8, xstep_bounds += xstep8, top_x -= xstep8) {
+      DirectionalZone3_WxH<8>(
+          dst_x, stride, min_height,
+          left_column + ((left_offset + y) << upsample_left_shift), base_left_y,
+          -ystep, upsample_left_shift);
+
+      DirectionalZone1Blend_WxH<8>(
+          dst_x, stride, min_height, top_row + (x << upsample_top_shift),
+          xstep_bounds, top_x, xstep, upsample_top_shift);
+    }
+    // Loop over y for left_only rows.
+    for (; y < height; y += 8, dst_x += stride8) {
+      DirectionalZone3_WxH<8>(
+          dst_x, stride, min_height,
+          left_column + ((left_offset + y) << upsample_left_shift), base_left_y,
+          -ystep, upsample_left_shift);
+    }
+  }
+  // TODO(johannkoenig): May be able to remove this branch.
+  if (x < width) {
+    DirectionalZone1_WxH(dst + x, stride, width - x, height,
+                         top_row + (x << upsample_top_shift), -xstep,
+                         upsampled_top);
+  }
+}
+
+void DirectionalIntraPredictorZone2_NEON(
+    void* const dest, const ptrdiff_t stride, const void* const top_row,
+    const void* const left_column, const int width, const int height,
+    const int xstep, const int ystep, const bool upsampled_top,
+    const bool upsampled_left) {
+  // Increasing the negative buffer for this function allows more rows to be
+  // processed at a time without branching in an inner loop to check the base.
+  uint8_t top_buffer[288];
+  uint8_t left_buffer[288];
+  memcpy(top_buffer + 128, static_cast<const uint8_t*>(top_row) - 16, 160);
+  memcpy(left_buffer + 128, static_cast<const uint8_t*>(left_column) - 16, 160);
+  const uint8_t* top_ptr = top_buffer + 144;
+  const uint8_t* left_ptr = left_buffer + 144;
+  auto* dst = static_cast<uint8_t*>(dest);
+
+  if (width == 4) {
+    DirectionalZone2_4xH(dst, stride, top_ptr, left_ptr, height, xstep, ystep,
+                         upsampled_top, upsampled_left);
+  } else {
+    DirectionalZone2_8(dst, stride, top_ptr, left_ptr, width, height, xstep,
+                       ystep, upsampled_top, upsampled_left);
+  }
 }
 
 void DirectionalIntraPredictorZone3_NEON(void* const dest,
@@ -239,8 +807,10 @@
     const uint8x8_t base_step_v = upsampled_left ? even : all;
     const uint8x8_t right_step = vadd_u8(base_step_v, vdup_n_u8(1));
 
-    for (int y = 0; y < height; y += 8) {
-      for (int x = 0; x < width; x += 4) {
+    int y = 0;
+    do {
+      int x = 0;
+      do {
         uint8_t* dst = static_cast<uint8_t*>(dest);
         dst += y * stride + x;
         uint8x8_t left_v[4], right_v[4], value_v[4];
@@ -298,104 +868,26 @@
           dst += stride;
           StoreHi4(dst, vreinterpret_u8_u16(c1.val[1]));
         }
-      }
-    }
+        x += 4;
+      } while (x < width);
+      y += 8;
+    } while (y < height);
   } else {  // 8x8 at a time.
     // Limited improvement for 8x8. ~20% faster for 64x64.
-    const uint8x8_t all = vcreate_u8(0x0706050403020100);
-    const uint8x8_t even = vcreate_u8(0x0e0c0a0806040200);
-    const uint8x8_t base_step_v = upsampled_left ? even : all;
-    const uint8x8_t right_step = vadd_u8(base_step_v, vdup_n_u8(1));
-
-    for (int y = 0; y < height; y += 8) {
-      for (int x = 0; x < width; x += 8) {
+    int y = 0;
+    do {
+      int x = 0;
+      do {
         uint8_t* dst = static_cast<uint8_t*>(dest);
         dst += y * stride + x;
-        uint8x8_t left_v[8], right_v[8], value_v[8];
-        const int ystep_base = ystep * x;
-        const int offset = y * base_step;
+        const int ystep_base = ystep * (x + 1);
 
-        const int index_0 = ystep_base + ystep * 1;
-        LoadStepwise(left + offset + (index_0 >> scale_bits), base_step_v,
-                     right_step, &left_v[0], &right_v[0]);
-        value_v[0] = WeightedBlend(left_v[0], right_v[0],
-                                   ((index_0 << upsample_shift) & 0x3F) >> 1);
-
-        const int index_1 = ystep_base + ystep * 2;
-        LoadStepwise(left + offset + (index_1 >> scale_bits), base_step_v,
-                     right_step, &left_v[1], &right_v[1]);
-        value_v[1] = WeightedBlend(left_v[1], right_v[1],
-                                   ((index_1 << upsample_shift) & 0x3F) >> 1);
-
-        const int index_2 = ystep_base + ystep * 3;
-        LoadStepwise(left + offset + (index_2 >> scale_bits), base_step_v,
-                     right_step, &left_v[2], &right_v[2]);
-        value_v[2] = WeightedBlend(left_v[2], right_v[2],
-                                   ((index_2 << upsample_shift) & 0x3F) >> 1);
-
-        const int index_3 = ystep_base + ystep * 4;
-        LoadStepwise(left + offset + (index_3 >> scale_bits), base_step_v,
-                     right_step, &left_v[3], &right_v[3]);
-        value_v[3] = WeightedBlend(left_v[3], right_v[3],
-                                   ((index_3 << upsample_shift) & 0x3F) >> 1);
-
-        const int index_4 = ystep_base + ystep * 5;
-        LoadStepwise(left + offset + (index_4 >> scale_bits), base_step_v,
-                     right_step, &left_v[4], &right_v[4]);
-        value_v[4] = WeightedBlend(left_v[4], right_v[4],
-                                   ((index_4 << upsample_shift) & 0x3F) >> 1);
-
-        const int index_5 = ystep_base + ystep * 6;
-        LoadStepwise(left + offset + (index_5 >> scale_bits), base_step_v,
-                     right_step, &left_v[5], &right_v[5]);
-        value_v[5] = WeightedBlend(left_v[5], right_v[5],
-                                   ((index_5 << upsample_shift) & 0x3F) >> 1);
-
-        const int index_6 = ystep_base + ystep * 7;
-        LoadStepwise(left + offset + (index_6 >> scale_bits), base_step_v,
-                     right_step, &left_v[6], &right_v[6]);
-        value_v[6] = WeightedBlend(left_v[6], right_v[6],
-                                   ((index_6 << upsample_shift) & 0x3F) >> 1);
-
-        const int index_7 = ystep_base + ystep * 8;
-        LoadStepwise(left + offset + (index_7 >> scale_bits), base_step_v,
-                     right_step, &left_v[7], &right_v[7]);
-        value_v[7] = WeightedBlend(left_v[7], right_v[7],
-                                   ((index_7 << upsample_shift) & 0x3F) >> 1);
-
-        // 8x8 transpose.
-        const uint8x16x2_t b0 = vtrnq_u8(vcombine_u8(value_v[0], value_v[4]),
-                                         vcombine_u8(value_v[1], value_v[5]));
-        const uint8x16x2_t b1 = vtrnq_u8(vcombine_u8(value_v[2], value_v[6]),
-                                         vcombine_u8(value_v[3], value_v[7]));
-
-        const uint16x8x2_t c0 = vtrnq_u16(vreinterpretq_u16_u8(b0.val[0]),
-                                          vreinterpretq_u16_u8(b1.val[0]));
-        const uint16x8x2_t c1 = vtrnq_u16(vreinterpretq_u16_u8(b0.val[1]),
-                                          vreinterpretq_u16_u8(b1.val[1]));
-
-        const uint32x4x2_t d0 = vuzpq_u32(vreinterpretq_u32_u16(c0.val[0]),
-                                          vreinterpretq_u32_u16(c1.val[0]));
-        const uint32x4x2_t d1 = vuzpq_u32(vreinterpretq_u32_u16(c0.val[1]),
-                                          vreinterpretq_u32_u16(c1.val[1]));
-
-        vst1_u8(dst, vreinterpret_u8_u32(vget_low_u32(d0.val[0])));
-        dst += stride;
-        vst1_u8(dst, vreinterpret_u8_u32(vget_high_u32(d0.val[0])));
-        dst += stride;
-        vst1_u8(dst, vreinterpret_u8_u32(vget_low_u32(d1.val[0])));
-        dst += stride;
-        vst1_u8(dst, vreinterpret_u8_u32(vget_high_u32(d1.val[0])));
-        dst += stride;
-        vst1_u8(dst, vreinterpret_u8_u32(vget_low_u32(d0.val[1])));
-        dst += stride;
-        vst1_u8(dst, vreinterpret_u8_u32(vget_high_u32(d0.val[1])));
-        dst += stride;
-        vst1_u8(dst, vreinterpret_u8_u32(vget_low_u32(d1.val[1])));
-        dst += stride;
-        vst1_u8(dst, vreinterpret_u8_u32(vget_high_u32(d1.val[1])));
-      }
-    }
+        DirectionalZone3_WxH<8>(dst, stride, 8, left + (y << upsample_shift),
+                                ystep_base, ystep, upsample_shift);
+        x += 8;
+      } while (x < width);
+      y += 8;
+    } while (y < height);
   }
 }
 
@@ -403,6 +895,7 @@
   Dsp* const dsp = dsp_internal::GetWritableDspTable(8);
   assert(dsp != nullptr);
   dsp->directional_intra_predictor_zone1 = DirectionalIntraPredictorZone1_NEON;
+  dsp->directional_intra_predictor_zone2 = DirectionalIntraPredictorZone2_NEON;
   dsp->directional_intra_predictor_zone3 = DirectionalIntraPredictorZone3_NEON;
 }
 
diff --git a/libgav1/src/dsp/arm/intrapred_filter_intra_neon.cc b/libgav1/src/dsp/arm/intrapred_filter_intra_neon.cc
index 9d023d3..ed7287f 100644
--- a/libgav1/src/dsp/arm/intrapred_filter_intra_neon.cc
+++ b/libgav1/src/dsp/arm/intrapred_filter_intra_neon.cc
@@ -1,5 +1,5 @@
-#include "src/dsp/arm/intrapred_neon.h"
 #include "src/dsp/dsp.h"
+#include "src/dsp/intrapred.h"
 
 #if LIBGAV1_ENABLE_NEON
 
@@ -90,9 +90,11 @@
   const uint8_t* relative_top = top;
   uint8_t relative_left[2] = {left[0], left[1]};
 
-  for (int y = 0; y < height; y += 2) {
+  int y = 0;
+  do {
     uint8_t* row_dst = dst;
-    for (int x = 0; x < width; x += 4) {
+    int x = 0;
+    do {
       uint16x8_t sum = vdupq_n_u16(0);
       const uint16x8_t subtrahend =
           vmull_u8(transposed_taps[0], vdup_n_u8(relative_top_left));
@@ -119,7 +121,8 @@
       relative_left[0] = row_dst[3];
       relative_left[1] = row_dst[3 + stride];
       row_dst += 4;
-    }
+      x += 4;
+    } while (x < width);
 
     // Progress down.
     relative_top_left = left[y + 1];
@@ -128,7 +131,8 @@
     relative_left[1] = left[y + 3];
 
     dst += 2 * stride;
-  }
+    y += 2;
+  } while (y < height);
 }
 
 void Init8bpp() {
diff --git a/libgav1/src/dsp/arm/intrapred_neon.cc b/libgav1/src/dsp/arm/intrapred_neon.cc
index d7cea3c..dad050b 100644
--- a/libgav1/src/dsp/arm/intrapred_neon.cc
+++ b/libgav1/src/dsp/arm/intrapred_neon.cc
@@ -1,6 +1,5 @@
-#include "src/dsp/arm/intrapred_neon.h"
-
 #include "src/dsp/dsp.h"
+#include "src/dsp/intrapred.h"
 
 #if LIBGAV1_ENABLE_NEON
 
diff --git a/libgav1/src/dsp/arm/intrapred_neon.h b/libgav1/src/dsp/arm/intrapred_neon.h
index 925587a..190f1a8 100644
--- a/libgav1/src/dsp/arm/intrapred_neon.h
+++ b/libgav1/src/dsp/arm/intrapred_neon.h
@@ -3,13 +3,14 @@
 
 #include "src/dsp/cpu.h"
 #include "src/dsp/dsp.h"
-#include "src/dsp/intrapred.h"
 
 namespace libgav1 {
 namespace dsp {
 
-// Initializes Dsp::intra_predictors with neon implementations. This function is
-// not thread-safe.
+// Initializes Dsp::intra_predictors, Dsp::directional_intra_predictor_zone*,
+// Dsp::cfl_intra_predictors, Dsp::cfl_subsamplers and
+// Dsp::filter_intra_predictor, see the defines below for specifics. These
+// functions are not thread-safe.
 void IntraPredCflInit_NEON();
 void IntraPredDirectionalInit_NEON();
 void IntraPredFilterIntraInit_NEON();
@@ -24,6 +25,7 @@
 #define LIBGAV1_Dsp8bpp_FilterIntraPredictor LIBGAV1_DSP_NEON
 
 #define LIBGAV1_Dsp8bpp_DirectionalIntraPredictorZone1 LIBGAV1_DSP_NEON
+#define LIBGAV1_Dsp8bpp_DirectionalIntraPredictorZone2 LIBGAV1_DSP_NEON
 #define LIBGAV1_Dsp8bpp_DirectionalIntraPredictorZone3 LIBGAV1_DSP_NEON
 
 // 4x4
@@ -38,6 +40,8 @@
   LIBGAV1_DSP_NEON
 
 #define LIBGAV1_Dsp8bpp_TransformSize4x4_CflIntraPredictor LIBGAV1_DSP_NEON
+#define LIBGAV1_Dsp8bpp_TransformSize4x4_CflSubsampler420 LIBGAV1_DSP_NEON
+#define LIBGAV1_Dsp8bpp_TransformSize4x4_CflSubsampler444 LIBGAV1_DSP_NEON
 
 // 4x8
 #define LIBGAV1_Dsp8bpp_TransformSize4x8_IntraPredictorDcTop LIBGAV1_DSP_NEON
@@ -51,6 +55,8 @@
   LIBGAV1_DSP_NEON
 
 #define LIBGAV1_Dsp8bpp_TransformSize4x8_CflIntraPredictor LIBGAV1_DSP_NEON
+#define LIBGAV1_Dsp8bpp_TransformSize4x8_CflSubsampler420 LIBGAV1_DSP_NEON
+#define LIBGAV1_Dsp8bpp_TransformSize4x8_CflSubsampler444 LIBGAV1_DSP_NEON
 
 // 4x16
 #define LIBGAV1_Dsp8bpp_TransformSize4x16_IntraPredictorDcTop LIBGAV1_DSP_NEON
@@ -64,6 +70,8 @@
   LIBGAV1_DSP_NEON
 
 #define LIBGAV1_Dsp8bpp_TransformSize4x16_CflIntraPredictor LIBGAV1_DSP_NEON
+#define LIBGAV1_Dsp8bpp_TransformSize4x16_CflSubsampler420 LIBGAV1_DSP_NEON
+#define LIBGAV1_Dsp8bpp_TransformSize4x16_CflSubsampler444 LIBGAV1_DSP_NEON
 
 // 8x4
 #define LIBGAV1_Dsp8bpp_TransformSize8x4_IntraPredictorDcTop LIBGAV1_DSP_NEON
@@ -77,6 +85,8 @@
   LIBGAV1_DSP_NEON
 
 #define LIBGAV1_Dsp8bpp_TransformSize8x4_CflIntraPredictor LIBGAV1_DSP_NEON
+#define LIBGAV1_Dsp8bpp_TransformSize8x4_CflSubsampler420 LIBGAV1_DSP_NEON
+#define LIBGAV1_Dsp8bpp_TransformSize8x4_CflSubsampler444 LIBGAV1_DSP_NEON
 
 // 8x8
 #define LIBGAV1_Dsp8bpp_TransformSize8x8_IntraPredictorDcTop LIBGAV1_DSP_NEON
@@ -90,6 +100,8 @@
   LIBGAV1_DSP_NEON
 
 #define LIBGAV1_Dsp8bpp_TransformSize8x8_CflIntraPredictor LIBGAV1_DSP_NEON
+#define LIBGAV1_Dsp8bpp_TransformSize8x8_CflSubsampler420 LIBGAV1_DSP_NEON
+#define LIBGAV1_Dsp8bpp_TransformSize8x8_CflSubsampler444 LIBGAV1_DSP_NEON
 
 // 8x16
 #define LIBGAV1_Dsp8bpp_TransformSize8x16_IntraPredictorDcTop LIBGAV1_DSP_NEON
@@ -103,6 +115,8 @@
   LIBGAV1_DSP_NEON
 
 #define LIBGAV1_Dsp8bpp_TransformSize8x16_CflIntraPredictor LIBGAV1_DSP_NEON
+#define LIBGAV1_Dsp8bpp_TransformSize8x16_CflSubsampler420 LIBGAV1_DSP_NEON
+#define LIBGAV1_Dsp8bpp_TransformSize8x16_CflSubsampler444 LIBGAV1_DSP_NEON
 
 // 8x32
 #define LIBGAV1_Dsp8bpp_TransformSize8x32_IntraPredictorDcTop LIBGAV1_DSP_NEON
@@ -116,6 +130,8 @@
   LIBGAV1_DSP_NEON
 
 #define LIBGAV1_Dsp8bpp_TransformSize8x32_CflIntraPredictor LIBGAV1_DSP_NEON
+#define LIBGAV1_Dsp8bpp_TransformSize8x32_CflSubsampler420 LIBGAV1_DSP_NEON
+#define LIBGAV1_Dsp8bpp_TransformSize8x32_CflSubsampler444 LIBGAV1_DSP_NEON
 
 // 16x4
 #define LIBGAV1_Dsp8bpp_TransformSize16x4_IntraPredictorDcTop LIBGAV1_DSP_NEON
@@ -129,6 +145,8 @@
   LIBGAV1_DSP_NEON
 
 #define LIBGAV1_Dsp8bpp_TransformSize16x4_CflIntraPredictor LIBGAV1_DSP_NEON
+#define LIBGAV1_Dsp8bpp_TransformSize16x4_CflSubsampler420 LIBGAV1_DSP_NEON
+#define LIBGAV1_Dsp8bpp_TransformSize16x4_CflSubsampler444 LIBGAV1_DSP_NEON
 
 // 16x8
 #define LIBGAV1_Dsp8bpp_TransformSize16x8_IntraPredictorDcTop LIBGAV1_DSP_NEON
@@ -142,6 +160,8 @@
   LIBGAV1_DSP_NEON
 
 #define LIBGAV1_Dsp8bpp_TransformSize16x8_CflIntraPredictor LIBGAV1_DSP_NEON
+#define LIBGAV1_Dsp8bpp_TransformSize16x8_CflSubsampler420 LIBGAV1_DSP_NEON
+#define LIBGAV1_Dsp8bpp_TransformSize16x8_CflSubsampler444 LIBGAV1_DSP_NEON
 
 // 16x16
 #define LIBGAV1_Dsp8bpp_TransformSize16x16_IntraPredictorDcTop LIBGAV1_DSP_NEON
@@ -155,6 +175,8 @@
   LIBGAV1_DSP_NEON
 
 #define LIBGAV1_Dsp8bpp_TransformSize16x16_CflIntraPredictor LIBGAV1_DSP_NEON
+#define LIBGAV1_Dsp8bpp_TransformSize16x16_CflSubsampler420 LIBGAV1_DSP_NEON
+#define LIBGAV1_Dsp8bpp_TransformSize16x16_CflSubsampler444 LIBGAV1_DSP_NEON
 
 // 16x32
 #define LIBGAV1_Dsp8bpp_TransformSize16x32_IntraPredictorDcTop LIBGAV1_DSP_NEON
@@ -168,6 +190,8 @@
   LIBGAV1_DSP_NEON
 
 #define LIBGAV1_Dsp8bpp_TransformSize16x32_CflIntraPredictor LIBGAV1_DSP_NEON
+#define LIBGAV1_Dsp8bpp_TransformSize16x32_CflSubsampler420 LIBGAV1_DSP_NEON
+#define LIBGAV1_Dsp8bpp_TransformSize16x32_CflSubsampler444 LIBGAV1_DSP_NEON
 
 // 16x64
 #define LIBGAV1_Dsp8bpp_TransformSize16x64_IntraPredictorDcTop LIBGAV1_DSP_NEON
@@ -192,6 +216,8 @@
   LIBGAV1_DSP_NEON
 
 #define LIBGAV1_Dsp8bpp_TransformSize32x8_CflIntraPredictor LIBGAV1_DSP_NEON
+#define LIBGAV1_Dsp8bpp_TransformSize32x8_CflSubsampler420 LIBGAV1_DSP_NEON
+#define LIBGAV1_Dsp8bpp_TransformSize32x8_CflSubsampler444 LIBGAV1_DSP_NEON
 
 // 32x16
 #define LIBGAV1_Dsp8bpp_TransformSize32x16_IntraPredictorDcTop LIBGAV1_DSP_NEON
@@ -205,6 +231,8 @@
   LIBGAV1_DSP_NEON
 
 #define LIBGAV1_Dsp8bpp_TransformSize32x16_CflIntraPredictor LIBGAV1_DSP_NEON
+#define LIBGAV1_Dsp8bpp_TransformSize32x16_CflSubsampler420 LIBGAV1_DSP_NEON
+#define LIBGAV1_Dsp8bpp_TransformSize32x16_CflSubsampler444 LIBGAV1_DSP_NEON
 
 // 32x32
 #define LIBGAV1_Dsp8bpp_TransformSize32x32_IntraPredictorDcTop LIBGAV1_DSP_NEON
@@ -218,6 +246,8 @@
   LIBGAV1_DSP_NEON
 
 #define LIBGAV1_Dsp8bpp_TransformSize32x32_CflIntraPredictor LIBGAV1_DSP_NEON
+#define LIBGAV1_Dsp8bpp_TransformSize32x32_CflSubsampler420 LIBGAV1_DSP_NEON
+#define LIBGAV1_Dsp8bpp_TransformSize32x32_CflSubsampler444 LIBGAV1_DSP_NEON
 
 // 32x64
 #define LIBGAV1_Dsp8bpp_TransformSize32x64_IntraPredictorDcTop LIBGAV1_DSP_NEON
diff --git a/libgav1/src/dsp/arm/intrapred_smooth_neon.cc b/libgav1/src/dsp/arm/intrapred_smooth_neon.cc
index e094174..b251fa5 100644
--- a/libgav1/src/dsp/arm/intrapred_smooth_neon.cc
+++ b/libgav1/src/dsp/arm/intrapred_smooth_neon.cc
@@ -1,5 +1,5 @@
-#include "src/dsp/arm/intrapred_neon.h"
 #include "src/dsp/dsp.h"
+#include "src/dsp/intrapred.h"
 
 #if LIBGAV1_ENABLE_NEON
 
diff --git a/libgav1/src/dsp/arm/inverse_transform_neon.cc b/libgav1/src/dsp/arm/inverse_transform_neon.cc
new file mode 100644
index 0000000..9e57365
--- /dev/null
+++ b/libgav1/src/dsp/arm/inverse_transform_neon.cc
@@ -0,0 +1,2627 @@
+#include "src/dsp/dsp.h"
+#include "src/dsp/inverse_transform.h"
+
+#if LIBGAV1_ENABLE_NEON
+
+#include <arm_neon.h>
+
+#include <algorithm>
+#include <cassert>
+#include <cstdint>
+
+#include "src/dsp/arm/common_neon.h"
+#include "src/utils/array_2d.h"
+#include "src/utils/common.h"
+#include "src/utils/compiler_attributes.h"
+#include "src/utils/constants.h"
+
+namespace libgav1 {
+namespace dsp {
+namespace low_bitdepth {
+namespace {
+
+// Include the constants and utility functions inside the anonymous namespace.
+#include "src/dsp/inverse_transform.inc"
+
+//------------------------------------------------------------------------------
+
+// TODO(slavarnway): Move transpose functions to transpose_neon.h or
+// common_neon.h.
+
+LIBGAV1_ALWAYS_INLINE void Transpose4x4(const int16x8_t in[4],
+                                        int16x8_t out[4]) {
+  // Swap 16 bit elements. Goes from:
+  // a0: 00 01 02 03
+  // a1: 10 11 12 13
+  // a2: 20 21 22 23
+  // a3: 30 31 32 33
+  // to:
+  // b0.val[0]: 00 10 02 12
+  // b0.val[1]: 01 11 03 13
+  // b1.val[0]: 20 30 22 32
+  // b1.val[1]: 21 31 23 33
+  const int16x4_t a0 = vget_low_s16(in[0]);
+  const int16x4_t a1 = vget_low_s16(in[1]);
+  const int16x4_t a2 = vget_low_s16(in[2]);
+  const int16x4_t a3 = vget_low_s16(in[3]);
+
+  const int16x4x2_t b0 = vtrn_s16(a0, a1);
+  const int16x4x2_t b1 = vtrn_s16(a2, a3);
+
+  // Swap 32 bit elements resulting in:
+  // c0.val[0]: 00 10 20 30 04 14 24 34
+  // c0.val[1]: 02 12 22 32 06 16 26 36
+  // c1.val[0]: 01 11 21 31 05 15 25 35
+  // c1.val[1]: 03 13 23 33 07 17 27 37
+  const int32x2x2_t c0 = vtrn_s32(vreinterpret_s32_s16(b0.val[0]),
+                                  vreinterpret_s32_s16(b1.val[0]));
+  const int32x2x2_t c1 = vtrn_s32(vreinterpret_s32_s16(b0.val[1]),
+                                  vreinterpret_s32_s16(b1.val[1]));
+
+  const int16x4_t d0 = vreinterpret_s16_s32(c0.val[0]);
+  const int16x4_t d1 = vreinterpret_s16_s32(c1.val[0]);
+  const int16x4_t d2 = vreinterpret_s16_s32(c0.val[1]);
+  const int16x4_t d3 = vreinterpret_s16_s32(c1.val[1]);
+
+  out[0] = vcombine_s16(d0, d0);
+  out[1] = vcombine_s16(d1, d1);
+  out[2] = vcombine_s16(d2, d2);
+  out[3] = vcombine_s16(d3, d3);
+}
+
+LIBGAV1_ALWAYS_INLINE void Transpose8x8(const int16x8_t in[8],
+                                        int16x8_t out[8]) {
+  // Swap 16 bit elements. Goes from:
+  // a0: 00 01 02 03 04 05 06 07
+  // a1: 10 11 12 13 14 15 16 17
+  // a2: 20 21 22 23 24 25 26 27
+  // a3: 30 31 32 33 34 35 36 37
+  // a4: 40 41 42 43 44 45 46 47
+  // a5: 50 51 52 53 54 55 56 57
+  // a6: 60 61 62 63 64 65 66 67
+  // a7: 70 71 72 73 74 75 76 77
+  // to:
+  // b0.val[0]: 00 10 02 12 04 14 06 16
+  // b0.val[1]: 01 11 03 13 05 15 07 17
+  // b1.val[0]: 20 30 22 32 24 34 26 36
+  // b1.val[1]: 21 31 23 33 25 35 27 37
+  // b2.val[0]: 40 50 42 52 44 54 46 56
+  // b2.val[1]: 41 51 43 53 45 55 47 57
+  // b3.val[0]: 60 70 62 72 64 74 66 76
+  // b3.val[1]: 61 71 63 73 65 75 67 77
+
+  const int16x8x2_t b0 = vtrnq_s16(in[0], in[1]);
+  const int16x8x2_t b1 = vtrnq_s16(in[2], in[3]);
+  const int16x8x2_t b2 = vtrnq_s16(in[4], in[5]);
+  const int16x8x2_t b3 = vtrnq_s16(in[6], in[7]);
+
+  // Swap 32 bit elements resulting in:
+  // c0.val[0]: 00 10 20 30 04 14 24 34
+  // c0.val[1]: 02 12 22 32 06 16 26 36
+  // c1.val[0]: 01 11 21 31 05 15 25 35
+  // c1.val[1]: 03 13 23 33 07 17 27 37
+  // c2.val[0]: 40 50 60 70 44 54 64 74
+  // c2.val[1]: 42 52 62 72 46 56 66 76
+  // c3.val[0]: 41 51 61 71 45 55 65 75
+  // c3.val[1]: 43 53 63 73 47 57 67 77
+
+  const int32x4x2_t c0 = vtrnq_s32(vreinterpretq_s32_s16(b0.val[0]),
+                                   vreinterpretq_s32_s16(b1.val[0]));
+  const int32x4x2_t c1 = vtrnq_s32(vreinterpretq_s32_s16(b0.val[1]),
+                                   vreinterpretq_s32_s16(b1.val[1]));
+  const int32x4x2_t c2 = vtrnq_s32(vreinterpretq_s32_s16(b2.val[0]),
+                                   vreinterpretq_s32_s16(b3.val[0]));
+  const int32x4x2_t c3 = vtrnq_s32(vreinterpretq_s32_s16(b2.val[1]),
+                                   vreinterpretq_s32_s16(b3.val[1]));
+
+  // Swap 64 bit elements resulting in:
+  // d0.val[0]: 00 10 20 30 40 50 60 70
+  // d0.val[1]: 04 14 24 34 44 54 64 74
+  // d1.val[0]: 01 11 21 31 41 51 61 71
+  // d1.val[1]: 05 15 25 35 45 55 65 75
+  // d2.val[0]: 02 12 22 32 42 52 62 72
+  // d2.val[1]: 06 16 26 36 46 56 66 76
+  // d3.val[0]: 03 13 23 33 43 53 63 73
+  // d3.val[1]: 07 17 27 37 47 57 67 77
+  const int16x8x2_t d0 = VtrnqS64(c0.val[0], c2.val[0]);
+  const int16x8x2_t d1 = VtrnqS64(c1.val[0], c3.val[0]);
+  const int16x8x2_t d2 = VtrnqS64(c0.val[1], c2.val[1]);
+  const int16x8x2_t d3 = VtrnqS64(c1.val[1], c3.val[1]);
+
+  out[0] = d0.val[0];
+  out[1] = d1.val[0];
+  out[2] = d2.val[0];
+  out[3] = d3.val[0];
+  out[4] = d0.val[1];
+  out[5] = d1.val[1];
+  out[6] = d2.val[1];
+  out[7] = d3.val[1];
+}
+
+LIBGAV1_ALWAYS_INLINE void Transpose4x8To8x4(const uint16x8_t in[8],
+                                             uint16x8_t out[4]) {
+  // Swap 16 bit elements. Goes from:
+  // a0: 00 01 02 03
+  // a1: 10 11 12 13
+  // a2: 20 21 22 23
+  // a3: 30 31 32 33
+  // a4: 40 41 42 43
+  // a5: 50 51 52 53
+  // a6: 60 61 62 63
+  // a7: 70 71 72 73
+  // to:
+  // b0.val[0]: 00 10 02 12
+  // b0.val[1]: 01 11 03 13
+  // b1.val[0]: 20 30 22 32
+  // b1.val[1]: 21 31 23 33
+  // b2.val[0]: 40 50 42 52
+  // b2.val[1]: 41 51 43 53
+  // b3.val[0]: 60 70 62 72
+  // b3.val[1]: 61 71 63 73
+
+  uint16x4x2_t b0 = vtrn_u16(vget_low_u16(in[0]), vget_low_u16(in[1]));
+  uint16x4x2_t b1 = vtrn_u16(vget_low_u16(in[2]), vget_low_u16(in[3]));
+  uint16x4x2_t b2 = vtrn_u16(vget_low_u16(in[4]), vget_low_u16(in[5]));
+  uint16x4x2_t b3 = vtrn_u16(vget_low_u16(in[6]), vget_low_u16(in[7]));
+
+  // Swap 32 bit elements resulting in:
+  // c0.val[0]: 00 10 20 30
+  // c0.val[1]: 02 12 22 32
+  // c1.val[0]: 01 11 21 31
+  // c1.val[1]: 03 13 23 33
+  // c2.val[0]: 40 50 60 70
+  // c2.val[1]: 42 52 62 72
+  // c3.val[0]: 41 51 61 71
+  // c3.val[1]: 43 53 63 73
+
+  uint32x2x2_t c0 = vtrn_u32(vreinterpret_u32_u16(b0.val[0]),
+                             vreinterpret_u32_u16(b1.val[0]));
+  uint32x2x2_t c1 = vtrn_u32(vreinterpret_u32_u16(b0.val[1]),
+                             vreinterpret_u32_u16(b1.val[1]));
+  uint32x2x2_t c2 = vtrn_u32(vreinterpret_u32_u16(b2.val[0]),
+                             vreinterpret_u32_u16(b3.val[0]));
+  uint32x2x2_t c3 = vtrn_u32(vreinterpret_u32_u16(b2.val[1]),
+                             vreinterpret_u32_u16(b3.val[1]));
+
+  // Swap 64 bit elements resulting in:
+  // o0: 00 10 20 30 40 50 60 70
+  // o1: 01 11 21 31 41 51 61 71
+  // o2: 02 12 22 32 42 52 62 72
+  // o3: 03 13 23 33 43 53 63 73
+
+  out[0] = vcombine_u16(vreinterpret_u16_u32(c0.val[0]),
+                        vreinterpret_u16_u32(c2.val[0]));
+  out[1] = vcombine_u16(vreinterpret_u16_u32(c1.val[0]),
+                        vreinterpret_u16_u32(c3.val[0]));
+  out[2] = vcombine_u16(vreinterpret_u16_u32(c0.val[1]),
+                        vreinterpret_u16_u32(c2.val[1]));
+  out[3] = vcombine_u16(vreinterpret_u16_u32(c1.val[1]),
+                        vreinterpret_u16_u32(c3.val[1]));
+}
+
+LIBGAV1_ALWAYS_INLINE void Transpose4x8To8x4(const int16x8_t in[8],
+                                             int16x8_t out[4]) {
+  Transpose4x8To8x4(reinterpret_cast<const uint16x8_t*>(in),
+                    reinterpret_cast<uint16x8_t*>(out));
+}
+
+LIBGAV1_ALWAYS_INLINE void Transpose8x4To4x8(const int16x8_t in[4],
+                                             int16x8_t out[8]) {
+  // Swap 16 bit elements. Goes from:
+  // a0: 00 01 02 03 04 05 06 07
+  // a1: 10 11 12 13 14 15 16 17
+  // a2: 20 21 22 23 24 25 26 27
+  // a3: 30 31 32 33 34 35 36 37
+  // to:
+  // b0.val[0]: 00 10 02 12 04 14 06 16
+  // b0.val[1]: 01 11 03 13 05 15 07 17
+  // b1.val[0]: 20 30 22 32 24 34 26 36
+  // b1.val[1]: 21 31 23 33 25 35 27 37
+  const int16x8x2_t b0 = vtrnq_s16(in[0], in[1]);
+  const int16x8x2_t b1 = vtrnq_s16(in[2], in[3]);
+
+  // Swap 32 bit elements resulting in:
+  // c0.val[0]: 00 10 20 30 04 14 24 34
+  // c0.val[1]: 02 12 22 32 06 16 26 36
+  // c1.val[0]: 01 11 21 31 05 15 25 35
+  // c1.val[1]: 03 13 23 33 07 17 27 37
+  const int32x4x2_t c0 = vtrnq_s32(vreinterpretq_s32_s16(b0.val[0]),
+                                   vreinterpretq_s32_s16(b1.val[0]));
+  const int32x4x2_t c1 = vtrnq_s32(vreinterpretq_s32_s16(b0.val[1]),
+                                   vreinterpretq_s32_s16(b1.val[1]));
+
+  // The upper 8 bytes are don't cares.
+  // out[0]: 00 10 20 30 04 14 24 34
+  // out[1]: 01 11 21 31 05 15 25 35
+  // out[2]: 02 12 22 32 06 16 26 36
+  // out[3]: 03 13 23 33 07 17 27 37
+  // out[4]: 04 14 24 34 04 14 24 34
+  // out[5]: 05 15 25 35 05 15 25 35
+  // out[6]: 06 16 26 36 06 16 26 36
+  // out[7]: 07 17 27 37 07 17 27 37
+  out[0] = vreinterpretq_s16_s32(c0.val[0]);
+  out[1] = vreinterpretq_s16_s32(c1.val[0]);
+  out[2] = vreinterpretq_s16_s32(c0.val[1]);
+  out[3] = vreinterpretq_s16_s32(c1.val[1]);
+  out[4] = vreinterpretq_s16_s32(
+      vcombine_s32(vget_high_s32(c0.val[0]), vget_high_s32(c0.val[0])));
+  out[5] = vreinterpretq_s16_s32(
+      vcombine_s32(vget_high_s32(c1.val[0]), vget_high_s32(c1.val[0])));
+  out[6] = vreinterpretq_s16_s32(
+      vcombine_s32(vget_high_s32(c0.val[1]), vget_high_s32(c0.val[1])));
+  out[7] = vreinterpretq_s16_s32(
+      vcombine_s32(vget_high_s32(c1.val[1]), vget_high_s32(c1.val[1])));
+}
+
+//------------------------------------------------------------------------------
+template <int store_width, int store_count>
+LIBGAV1_ALWAYS_INLINE void StoreDst(int16_t* dst, int32_t stride, int32_t idx,
+                                    const int16x8_t* const s) {
+  assert(store_count % 4 == 0);
+  assert(store_width == 8 || store_width == 16);
+  // NOTE: It is expected that the compiler will unroll these loops.
+  if (store_width == 16) {
+    for (int i = 0; i < store_count; i += 4) {
+      vst1q_s16(&dst[i * stride + idx], (s[i]));
+      vst1q_s16(&dst[(i + 1) * stride + idx], (s[i + 1]));
+      vst1q_s16(&dst[(i + 2) * stride + idx], (s[i + 2]));
+      vst1q_s16(&dst[(i + 3) * stride + idx], (s[i + 3]));
+    }
+  } else {
+    // store_width == 8
+    for (int i = 0; i < store_count; i += 4) {
+      vst1_s16(&dst[i * stride + idx], vget_low_s16(s[i]));
+      vst1_s16(&dst[(i + 1) * stride + idx], vget_low_s16(s[i + 1]));
+      vst1_s16(&dst[(i + 2) * stride + idx], vget_low_s16(s[i + 2]));
+      vst1_s16(&dst[(i + 3) * stride + idx], vget_low_s16(s[i + 3]));
+    }
+  }
+}
+
+template <int load_width, int load_count>
+LIBGAV1_ALWAYS_INLINE void LoadSrc(const int16_t* src, int32_t stride,
+                                   int32_t idx, int16x8_t* x) {
+  assert(load_count % 4 == 0);
+  assert(load_width == 8 || load_width == 16);
+  // NOTE: It is expected that the compiler will unroll these loops.
+  if (load_width == 16) {
+    for (int i = 0; i < load_count; i += 4) {
+      x[i] = vld1q_s16(&src[i * stride + idx]);
+      x[i + 1] = vld1q_s16(&src[(i + 1) * stride + idx]);
+      x[i + 2] = vld1q_s16(&src[(i + 2) * stride + idx]);
+      x[i + 3] = vld1q_s16(&src[(i + 3) * stride + idx]);
+    }
+  } else {
+    // load_width == 8
+    const int64x2_t zero = vdupq_n_s64(0);
+    for (int i = 0; i < load_count; i += 4) {
+      // The src buffer is aligned to 32 bytes.  Each load will always be 8
+      // byte aligned.
+      x[i] = vreinterpretq_s16_s64(vld1q_lane_s64(
+          reinterpret_cast<const int64_t*>(&src[i * stride + idx]), zero, 0));
+      x[i + 1] = vreinterpretq_s16_s64(vld1q_lane_s64(
+          reinterpret_cast<const int64_t*>(&src[(i + 1) * stride + idx]), zero,
+          0));
+      x[i + 2] = vreinterpretq_s16_s64(vld1q_lane_s64(
+          reinterpret_cast<const int64_t*>(&src[(i + 2) * stride + idx]), zero,
+          0));
+      x[i + 3] = vreinterpretq_s16_s64(vld1q_lane_s64(
+          reinterpret_cast<const int64_t*>(&src[(i + 3) * stride + idx]), zero,
+          0));
+    }
+  }
+}
+
+// Butterfly rotate 4 values.
+LIBGAV1_ALWAYS_INLINE void ButterflyRotation_4(int16x8_t* a, int16x8_t* b,
+                                               const int angle,
+                                               const bool flip) {
+  const int16_t cos128 = Cos128(angle);
+  const int16_t sin128 = Sin128(angle);
+  const int32x4_t acc_x = vmull_n_s16(vget_low_s16(*a), cos128);
+  const int32x4_t acc_y = vmull_n_s16(vget_low_s16(*a), sin128);
+  const int32x4_t x0 = vmlsl_n_s16(acc_x, vget_low_s16(*b), sin128);
+  const int32x4_t y0 = vmlal_n_s16(acc_y, vget_low_s16(*b), cos128);
+  const int16x4_t x1 = vqrshrn_n_s32(x0, 12);
+  const int16x4_t y1 = vqrshrn_n_s32(y0, 12);
+  const int16x8_t x = vcombine_s16(x1, x1);
+  const int16x8_t y = vcombine_s16(y1, y1);
+  if (flip) {
+    *a = y;
+    *b = x;
+  } else {
+    *a = x;
+    *b = y;
+  }
+}
+
+// Butterfly rotate 8 values.
+LIBGAV1_ALWAYS_INLINE void ButterflyRotation_8(int16x8_t* a, int16x8_t* b,
+                                               const int angle,
+                                               const bool flip) {
+  const int16_t cos128 = Cos128(angle);
+  const int16_t sin128 = Sin128(angle);
+  const int32x4_t acc_x = vmull_n_s16(vget_low_s16(*a), cos128);
+  const int32x4_t acc_y = vmull_n_s16(vget_low_s16(*a), sin128);
+  const int32x4_t x0 = vmlsl_n_s16(acc_x, vget_low_s16(*b), sin128);
+  const int32x4_t y0 = vmlal_n_s16(acc_y, vget_low_s16(*b), cos128);
+  const int16x4_t x1 = vqrshrn_n_s32(x0, 12);
+  const int16x4_t y1 = vqrshrn_n_s32(y0, 12);
+
+  const int32x4_t acc_x_hi = vmull_n_s16(vget_high_s16(*a), cos128);
+  const int32x4_t acc_y_hi = vmull_n_s16(vget_high_s16(*a), sin128);
+  const int32x4_t x0_hi = vmlsl_n_s16(acc_x_hi, vget_high_s16(*b), sin128);
+  const int32x4_t y0_hi = vmlal_n_s16(acc_y_hi, vget_high_s16(*b), cos128);
+  const int16x4_t x1_hi = vqrshrn_n_s32(x0_hi, 12);
+  const int16x4_t y1_hi = vqrshrn_n_s32(y0_hi, 12);
+
+  const int16x8_t x = vcombine_s16(x1, x1_hi);
+  const int16x8_t y = vcombine_s16(y1, y1_hi);
+  if (flip) {
+    *a = y;
+    *b = x;
+  } else {
+    *a = x;
+    *b = y;
+  }
+}
+
+LIBGAV1_ALWAYS_INLINE void ButterflyRotation_FirstIsZero(int16x8_t* a,
+                                                         int16x8_t* b,
+                                                         const int angle,
+                                                         const bool flip) {
+  const int16_t cos128 = Cos128(angle);
+  const int16_t sin128 = Sin128(angle);
+  const int32x4_t x0 = vmlsl_n_s16(vdupq_n_s32(0), vget_low_s16(*b), sin128);
+  const int32x4_t x0_hi =
+      vmlsl_n_s16(vdupq_n_s32(0), vget_high_s16(*b), sin128);
+  const int16x4_t x1 = vqrshrn_n_s32(x0, 12);
+  const int16x4_t x1_hi = vqrshrn_n_s32(x0_hi, 12);
+  const int16x8_t x = vcombine_s16(x1, x1_hi);
+  const int16x8_t y = vqrdmulhq_s16(*b, vdupq_n_s16(cos128 << 3));
+  if (flip) {
+    *a = y;
+    *b = x;
+  } else {
+    *a = x;
+    *b = y;
+  }
+}
+
+LIBGAV1_ALWAYS_INLINE void ButterflyRotation_SecondIsZero(int16x8_t* a,
+                                                          int16x8_t* b,
+                                                          const int angle,
+                                                          const bool flip) {
+  const int16_t cos128 = Cos128(angle);
+  const int16_t sin128 = Sin128(angle);
+  const int16x8_t x = vqrdmulhq_s16(*a, vdupq_n_s16(cos128 << 3));
+  const int16x8_t y = vqrdmulhq_s16(*a, vdupq_n_s16(sin128 << 3));
+  if (flip) {
+    *a = y;
+    *b = x;
+  } else {
+    *a = x;
+    *b = y;
+  }
+}
+
+LIBGAV1_ALWAYS_INLINE void HadamardRotation(int16x8_t* a, int16x8_t* b,
+                                            bool flip) {
+  int16x8_t x, y;
+  if (flip) {
+    y = vqaddq_s16(*b, *a);
+    x = vqsubq_s16(*b, *a);
+  } else {
+    x = vqaddq_s16(*a, *b);
+    y = vqsubq_s16(*a, *b);
+  }
+  *a = x;
+  *b = y;
+}
+
+using ButterflyRotationFunc = void (*)(int16x8_t* a, int16x8_t* b, int angle,
+                                       bool flip);
+
+//------------------------------------------------------------------------------
+// Discrete Cosine Transforms (DCT).
+
+template <ButterflyRotationFunc bufferfly_rotation,
+          bool is_fast_bufferfly = false>
+LIBGAV1_ALWAYS_INLINE void Dct4Stages(int16x8_t* s) {
+  // stage 12.
+  if (is_fast_bufferfly) {
+    ButterflyRotation_SecondIsZero(&s[0], &s[1], 32, true);
+    ButterflyRotation_SecondIsZero(&s[2], &s[3], 48, false);
+  } else {
+    bufferfly_rotation(&s[0], &s[1], 32, true);
+    bufferfly_rotation(&s[2], &s[3], 48, false);
+  }
+
+  // stage 17.
+  HadamardRotation(&s[0], &s[3], false);
+  HadamardRotation(&s[1], &s[2], false);
+}
+
+template <ButterflyRotationFunc bufferfly_rotation, bool stage_is_rectangular>
+LIBGAV1_ALWAYS_INLINE void Dct4_NEON(void* dest, const void* source,
+                                     int32_t step, bool transpose) {
+  auto* const dst = static_cast<int16_t*>(dest);
+  const auto* const src = static_cast<const int16_t*>(source);
+  int16x8_t s[4], x[4];
+
+  if (stage_is_rectangular) {
+    if (transpose) {
+      int16x8_t input[8];
+      LoadSrc<8, 8>(src, step, 0, input);
+      Transpose4x8To8x4(input, x);
+    } else {
+      LoadSrc<16, 4>(src, step, 0, x);
+    }
+  } else {
+    LoadSrc<8, 4>(src, step, 0, x);
+    if (transpose) {
+      Transpose4x4(x, x);
+    }
+  }
+
+  // stage 1.
+  // kBitReverseLookup 0, 2, 1, 3
+  s[0] = x[0];
+  s[1] = x[2];
+  s[2] = x[1];
+  s[3] = x[3];
+
+  Dct4Stages<bufferfly_rotation>(s);
+
+  if (stage_is_rectangular) {
+    if (transpose) {
+      int16x8_t output[8];
+      Transpose8x4To4x8(s, output);
+      StoreDst<8, 8>(dst, step, 0, output);
+    } else {
+      StoreDst<16, 4>(dst, step, 0, s);
+    }
+  } else {
+    if (transpose) {
+      Transpose4x4(s, s);
+    }
+    StoreDst<8, 4>(dst, step, 0, s);
+  }
+}
+
+template <ButterflyRotationFunc bufferfly_rotation,
+          bool is_fast_bufferfly = false>
+LIBGAV1_ALWAYS_INLINE void Dct8Stages(int16x8_t* s) {
+  // stage 8.
+  if (is_fast_bufferfly) {
+    ButterflyRotation_SecondIsZero(&s[4], &s[7], 56, false);
+    ButterflyRotation_FirstIsZero(&s[5], &s[6], 24, false);
+  } else {
+    bufferfly_rotation(&s[4], &s[7], 56, false);
+    bufferfly_rotation(&s[5], &s[6], 24, false);
+  }
+
+  // stage 13.
+  HadamardRotation(&s[4], &s[5], false);
+  HadamardRotation(&s[6], &s[7], true);
+
+  // stage 18.
+  bufferfly_rotation(&s[6], &s[5], 32, true);
+
+  // stage 22.
+  HadamardRotation(&s[0], &s[7], false);
+  HadamardRotation(&s[1], &s[6], false);
+  HadamardRotation(&s[2], &s[5], false);
+  HadamardRotation(&s[3], &s[4], false);
+}
+
+// Process dct8 rows or columns, depending on the transpose flag.
+template <ButterflyRotationFunc bufferfly_rotation, bool stage_is_rectangular>
+LIBGAV1_ALWAYS_INLINE void Dct8_NEON(void* dest, const void* source,
+                                     int32_t step, bool transpose) {
+  auto* const dst = static_cast<int16_t*>(dest);
+  const auto* const src = static_cast<const int16_t*>(source);
+  int16x8_t s[8], x[8];
+
+  if (stage_is_rectangular) {
+    if (transpose) {
+      int16x8_t input[4];
+      LoadSrc<16, 4>(src, step, 0, input);
+      Transpose8x4To4x8(input, x);
+    } else {
+      LoadSrc<8, 8>(src, step, 0, x);
+    }
+  } else if (transpose) {
+    int16x8_t input[8];
+    LoadSrc<16, 8>(src, step, 0, input);
+    Transpose8x8(input, x);
+  } else {
+    LoadSrc<16, 8>(src, step, 0, x);
+  }
+
+  // stage 1.
+  // kBitReverseLookup 0, 4, 2, 6, 1, 5, 3, 7,
+  s[0] = x[0];
+  s[1] = x[4];
+  s[2] = x[2];
+  s[3] = x[6];
+  s[4] = x[1];
+  s[5] = x[5];
+  s[6] = x[3];
+  s[7] = x[7];
+
+  Dct4Stages<bufferfly_rotation>(s);
+  Dct8Stages<bufferfly_rotation>(s);
+
+  if (stage_is_rectangular) {
+    if (transpose) {
+      int16x8_t output[4];
+      Transpose4x8To8x4(s, output);
+      StoreDst<16, 4>(dst, step, 0, output);
+    } else {
+      StoreDst<8, 8>(dst, step, 0, s);
+    }
+  } else if (transpose) {
+    int16x8_t output[8];
+    Transpose8x8(s, output);
+    StoreDst<16, 8>(dst, step, 0, output);
+  } else {
+    StoreDst<16, 8>(dst, step, 0, s);
+  }
+}
+
+template <ButterflyRotationFunc bufferfly_rotation,
+          bool is_fast_bufferfly = false>
+LIBGAV1_ALWAYS_INLINE void Dct16Stages(int16x8_t* s) {
+  // stage 5.
+  if (is_fast_bufferfly) {
+    ButterflyRotation_SecondIsZero(&s[8], &s[15], 60, false);
+    ButterflyRotation_FirstIsZero(&s[9], &s[14], 28, false);
+    ButterflyRotation_SecondIsZero(&s[10], &s[13], 44, false);
+    ButterflyRotation_FirstIsZero(&s[11], &s[12], 12, false);
+  } else {
+    bufferfly_rotation(&s[8], &s[15], 60, false);
+    bufferfly_rotation(&s[9], &s[14], 28, false);
+    bufferfly_rotation(&s[10], &s[13], 44, false);
+    bufferfly_rotation(&s[11], &s[12], 12, false);
+  }
+
+  // stage 9.
+  HadamardRotation(&s[8], &s[9], false);
+  HadamardRotation(&s[10], &s[11], true);
+  HadamardRotation(&s[12], &s[13], false);
+  HadamardRotation(&s[14], &s[15], true);
+
+  // stage 14.
+  bufferfly_rotation(&s[14], &s[9], 48, true);
+  bufferfly_rotation(&s[13], &s[10], 112, true);
+
+  // stage 19.
+  HadamardRotation(&s[8], &s[11], false);
+  HadamardRotation(&s[9], &s[10], false);
+  HadamardRotation(&s[12], &s[15], true);
+  HadamardRotation(&s[13], &s[14], true);
+
+  // stage 23.
+  bufferfly_rotation(&s[13], &s[10], 32, true);
+  bufferfly_rotation(&s[12], &s[11], 32, true);
+
+  // stage 26.
+  HadamardRotation(&s[0], &s[15], false);
+  HadamardRotation(&s[1], &s[14], false);
+  HadamardRotation(&s[2], &s[13], false);
+  HadamardRotation(&s[3], &s[12], false);
+  HadamardRotation(&s[4], &s[11], false);
+  HadamardRotation(&s[5], &s[10], false);
+  HadamardRotation(&s[6], &s[9], false);
+  HadamardRotation(&s[7], &s[8], false);
+}
+
+// Process dct16 rows or columns, depending on the transpose flag.
+template <ButterflyRotationFunc bufferfly_rotation, bool stage_is_rectangular>
+LIBGAV1_ALWAYS_INLINE void Dct16_NEON(void* dest, const void* source,
+                                      int32_t step, bool transpose) {
+  auto* const dst = static_cast<int16_t*>(dest);
+  const auto* const src = static_cast<const int16_t*>(source);
+  int16x8_t s[16], x[16];
+
+  if (stage_is_rectangular) {
+    if (transpose) {
+      int16x8_t input[4];
+      LoadSrc<16, 4>(src, step, 0, input);
+      Transpose8x4To4x8(input, x);
+      LoadSrc<16, 4>(src, step, 8, input);
+      Transpose8x4To4x8(input, &x[8]);
+    } else {
+      LoadSrc<8, 16>(src, step, 0, x);
+    }
+  } else if (transpose) {
+    for (int idx = 0; idx < 16; idx += 8) {
+      int16x8_t input[8];
+      LoadSrc<16, 8>(src, step, idx, input);
+      Transpose8x8(input, &x[idx]);
+    }
+  } else {
+    LoadSrc<16, 16>(src, step, 0, x);
+  }
+
+  // stage 1
+  // kBitReverseLookup 0, 8, 4, 12, 2, 10, 6, 14, 1, 9, 5, 13, 3, 11, 7, 15,
+  s[0] = x[0];
+  s[1] = x[8];
+  s[2] = x[4];
+  s[3] = x[12];
+  s[4] = x[2];
+  s[5] = x[10];
+  s[6] = x[6];
+  s[7] = x[14];
+  s[8] = x[1];
+  s[9] = x[9];
+  s[10] = x[5];
+  s[11] = x[13];
+  s[12] = x[3];
+  s[13] = x[11];
+  s[14] = x[7];
+  s[15] = x[15];
+
+  Dct4Stages<bufferfly_rotation>(s);
+  Dct8Stages<bufferfly_rotation>(s);
+  Dct16Stages<bufferfly_rotation>(s);
+
+  if (stage_is_rectangular) {
+    if (transpose) {
+      int16x8_t output[4];
+      Transpose4x8To8x4(s, output);
+      StoreDst<16, 4>(dst, step, 0, output);
+      Transpose4x8To8x4(&s[8], output);
+      StoreDst<16, 4>(dst, step, 8, output);
+    } else {
+      StoreDst<8, 16>(dst, step, 0, s);
+    }
+  } else if (transpose) {
+    for (int idx = 0; idx < 16; idx += 8) {
+      int16x8_t output[8];
+      Transpose8x8(&s[idx], output);
+      StoreDst<16, 8>(dst, step, idx, output);
+    }
+  } else {
+    StoreDst<16, 16>(dst, step, 0, s);
+  }
+}
+
+template <ButterflyRotationFunc bufferfly_rotation,
+          bool is_fast_butterfly = false>
+LIBGAV1_ALWAYS_INLINE void Dct32Stages(int16x8_t* s) {
+  // stage 3
+  if (is_fast_butterfly) {
+    ButterflyRotation_SecondIsZero(&s[16], &s[31], 62, false);
+    ButterflyRotation_FirstIsZero(&s[17], &s[30], 30, false);
+    ButterflyRotation_SecondIsZero(&s[18], &s[29], 46, false);
+    ButterflyRotation_FirstIsZero(&s[19], &s[28], 14, false);
+    ButterflyRotation_SecondIsZero(&s[20], &s[27], 54, false);
+    ButterflyRotation_FirstIsZero(&s[21], &s[26], 22, false);
+    ButterflyRotation_SecondIsZero(&s[22], &s[25], 38, false);
+    ButterflyRotation_FirstIsZero(&s[23], &s[24], 6, false);
+  } else {
+    bufferfly_rotation(&s[16], &s[31], 62, false);
+    bufferfly_rotation(&s[17], &s[30], 30, false);
+    bufferfly_rotation(&s[18], &s[29], 46, false);
+    bufferfly_rotation(&s[19], &s[28], 14, false);
+    bufferfly_rotation(&s[20], &s[27], 54, false);
+    bufferfly_rotation(&s[21], &s[26], 22, false);
+    bufferfly_rotation(&s[22], &s[25], 38, false);
+    bufferfly_rotation(&s[23], &s[24], 6, false);
+  }
+  // stage 6.
+  HadamardRotation(&s[16], &s[17], false);
+  HadamardRotation(&s[18], &s[19], true);
+  HadamardRotation(&s[20], &s[21], false);
+  HadamardRotation(&s[22], &s[23], true);
+  HadamardRotation(&s[24], &s[25], false);
+  HadamardRotation(&s[26], &s[27], true);
+  HadamardRotation(&s[28], &s[29], false);
+  HadamardRotation(&s[30], &s[31], true);
+
+  // stage 10.
+  bufferfly_rotation(&s[30], &s[17], 24 + 32, true);
+  bufferfly_rotation(&s[29], &s[18], 24 + 64 + 32, true);
+  bufferfly_rotation(&s[26], &s[21], 24, true);
+  bufferfly_rotation(&s[25], &s[22], 24 + 64, true);
+
+  // stage 15.
+  HadamardRotation(&s[16], &s[19], false);
+  HadamardRotation(&s[17], &s[18], false);
+  HadamardRotation(&s[20], &s[23], true);
+  HadamardRotation(&s[21], &s[22], true);
+  HadamardRotation(&s[24], &s[27], false);
+  HadamardRotation(&s[25], &s[26], false);
+  HadamardRotation(&s[28], &s[31], true);
+  HadamardRotation(&s[29], &s[30], true);
+
+  // stage 20.
+  bufferfly_rotation(&s[29], &s[18], 48, true);
+  bufferfly_rotation(&s[28], &s[19], 48, true);
+  bufferfly_rotation(&s[27], &s[20], 48 + 64, true);
+  bufferfly_rotation(&s[26], &s[21], 48 + 64, true);
+
+  // stage 24.
+  HadamardRotation(&s[16], &s[23], false);
+  HadamardRotation(&s[17], &s[22], false);
+  HadamardRotation(&s[18], &s[21], false);
+  HadamardRotation(&s[19], &s[20], false);
+  HadamardRotation(&s[24], &s[31], true);
+  HadamardRotation(&s[25], &s[30], true);
+  HadamardRotation(&s[26], &s[29], true);
+  HadamardRotation(&s[27], &s[28], true);
+
+  // stage 27.
+  bufferfly_rotation(&s[27], &s[20], 32, true);
+  bufferfly_rotation(&s[26], &s[21], 32, true);
+  bufferfly_rotation(&s[25], &s[22], 32, true);
+  bufferfly_rotation(&s[24], &s[23], 32, true);
+
+  // stage 29.
+  HadamardRotation(&s[0], &s[31], false);
+  HadamardRotation(&s[1], &s[30], false);
+  HadamardRotation(&s[2], &s[29], false);
+  HadamardRotation(&s[3], &s[28], false);
+  HadamardRotation(&s[4], &s[27], false);
+  HadamardRotation(&s[5], &s[26], false);
+  HadamardRotation(&s[6], &s[25], false);
+  HadamardRotation(&s[7], &s[24], false);
+  HadamardRotation(&s[8], &s[23], false);
+  HadamardRotation(&s[9], &s[22], false);
+  HadamardRotation(&s[10], &s[21], false);
+  HadamardRotation(&s[11], &s[20], false);
+  HadamardRotation(&s[12], &s[19], false);
+  HadamardRotation(&s[13], &s[18], false);
+  HadamardRotation(&s[14], &s[17], false);
+  HadamardRotation(&s[15], &s[16], false);
+}
+
+// Process dct32 rows or columns, depending on the transpose flag.
+LIBGAV1_ALWAYS_INLINE void Dct32_NEON(void* dest, const void* source,
+                                      const int32_t step,
+                                      const bool transpose) {
+  auto* const dst = static_cast<int16_t*>(dest);
+  const auto* const src = static_cast<const int16_t*>(source);
+  int16x8_t s[32], x[32];
+
+  if (transpose) {
+    for (int idx = 0; idx < 32; idx += 8) {
+      int16x8_t input[8];
+      LoadSrc<16, 8>(src, step, idx, input);
+      Transpose8x8(input, &x[idx]);
+    }
+  } else {
+    LoadSrc<16, 32>(src, step, 0, x);
+  }
+
+  // stage 1
+  // kBitReverseLookup
+  // 0, 16, 8, 24, 4, 20, 12, 28, 2, 18, 10, 26, 6, 22, 14, 30,
+  s[0] = x[0];
+  s[1] = x[16];
+  s[2] = x[8];
+  s[3] = x[24];
+  s[4] = x[4];
+  s[5] = x[20];
+  s[6] = x[12];
+  s[7] = x[28];
+  s[8] = x[2];
+  s[9] = x[18];
+  s[10] = x[10];
+  s[11] = x[26];
+  s[12] = x[6];
+  s[13] = x[22];
+  s[14] = x[14];
+  s[15] = x[30];
+
+  // 1, 17, 9, 25, 5, 21, 13, 29, 3, 19, 11, 27, 7, 23, 15, 31,
+  s[16] = x[1];
+  s[17] = x[17];
+  s[18] = x[9];
+  s[19] = x[25];
+  s[20] = x[5];
+  s[21] = x[21];
+  s[22] = x[13];
+  s[23] = x[29];
+  s[24] = x[3];
+  s[25] = x[19];
+  s[26] = x[11];
+  s[27] = x[27];
+  s[28] = x[7];
+  s[29] = x[23];
+  s[30] = x[15];
+  s[31] = x[31];
+
+  Dct4Stages<ButterflyRotation_8>(s);
+  Dct8Stages<ButterflyRotation_8>(s);
+  Dct16Stages<ButterflyRotation_8>(s);
+  Dct32Stages<ButterflyRotation_8>(s);
+
+  if (transpose) {
+    for (int idx = 0; idx < 32; idx += 8) {
+      int16x8_t output[8];
+      Transpose8x8(&s[idx], output);
+      StoreDst<16, 8>(dst, step, idx, output);
+    }
+  } else {
+    StoreDst<16, 32>(dst, step, 0, s);
+  }
+}
+
+// Allow the compiler to call this function instead of force inlining. Tests
+// show the performance is slightly faster.
+void Dct64_NEON(void* dest, const void* source, int32_t step, bool transpose) {
+  auto* const dst = static_cast<int16_t*>(dest);
+  const auto* const src = static_cast<const int16_t*>(source);
+  int16x8_t s[64], x[32];
+
+  if (transpose) {
+    // The last 32 values of every row are always zero if the |tx_width| is
+    // 64.
+    for (int idx = 0; idx < 32; idx += 8) {
+      int16x8_t input[8];
+      LoadSrc<16, 8>(src, step, idx, input);
+      Transpose8x8(input, &x[idx]);
+    }
+  } else {
+    // The last 32 values of every column are always zero if the |tx_height| is
+    // 64.
+    LoadSrc<16, 32>(src, step, 0, x);
+  }
+
+  // stage 1
+  // kBitReverseLookup
+  // 0, 32, 16, 48, 8, 40, 24, 56, 4, 36, 20, 52, 12, 44, 28, 60,
+  s[0] = x[0];
+  s[2] = x[16];
+  s[4] = x[8];
+  s[6] = x[24];
+  s[8] = x[4];
+  s[10] = x[20];
+  s[12] = x[12];
+  s[14] = x[28];
+
+  // 2, 34, 18, 50, 10, 42, 26, 58, 6, 38, 22, 54, 14, 46, 30, 62,
+  s[16] = x[2];
+  s[18] = x[18];
+  s[20] = x[10];
+  s[22] = x[26];
+  s[24] = x[6];
+  s[26] = x[22];
+  s[28] = x[14];
+  s[30] = x[30];
+
+  // 1, 33, 17, 49, 9, 41, 25, 57, 5, 37, 21, 53, 13, 45, 29, 61,
+  s[32] = x[1];
+  s[34] = x[17];
+  s[36] = x[9];
+  s[38] = x[25];
+  s[40] = x[5];
+  s[42] = x[21];
+  s[44] = x[13];
+  s[46] = x[29];
+
+  // 3, 35, 19, 51, 11, 43, 27, 59, 7, 39, 23, 55, 15, 47, 31, 63
+  s[48] = x[3];
+  s[50] = x[19];
+  s[52] = x[11];
+  s[54] = x[27];
+  s[56] = x[7];
+  s[58] = x[23];
+  s[60] = x[15];
+  s[62] = x[31];
+
+  Dct4Stages<ButterflyRotation_8, /*is_fast_butterfly=*/true>(s);
+  Dct8Stages<ButterflyRotation_8, /*is_fast_butterfly=*/true>(s);
+  Dct16Stages<ButterflyRotation_8, /*is_fast_butterfly=*/true>(s);
+  Dct32Stages<ButterflyRotation_8, /*is_fast_butterfly=*/true>(s);
+
+  //-- start dct 64 stages
+  // stage 2.
+  ButterflyRotation_SecondIsZero(&s[32], &s[63], 63 - 0, false);
+  ButterflyRotation_FirstIsZero(&s[33], &s[62], 63 - 32, false);
+  ButterflyRotation_SecondIsZero(&s[34], &s[61], 63 - 16, false);
+  ButterflyRotation_FirstIsZero(&s[35], &s[60], 63 - 48, false);
+  ButterflyRotation_SecondIsZero(&s[36], &s[59], 63 - 8, false);
+  ButterflyRotation_FirstIsZero(&s[37], &s[58], 63 - 40, false);
+  ButterflyRotation_SecondIsZero(&s[38], &s[57], 63 - 24, false);
+  ButterflyRotation_FirstIsZero(&s[39], &s[56], 63 - 56, false);
+  ButterflyRotation_SecondIsZero(&s[40], &s[55], 63 - 4, false);
+  ButterflyRotation_FirstIsZero(&s[41], &s[54], 63 - 36, false);
+  ButterflyRotation_SecondIsZero(&s[42], &s[53], 63 - 20, false);
+  ButterflyRotation_FirstIsZero(&s[43], &s[52], 63 - 52, false);
+  ButterflyRotation_SecondIsZero(&s[44], &s[51], 63 - 12, false);
+  ButterflyRotation_FirstIsZero(&s[45], &s[50], 63 - 44, false);
+  ButterflyRotation_SecondIsZero(&s[46], &s[49], 63 - 28, false);
+  ButterflyRotation_FirstIsZero(&s[47], &s[48], 63 - 60, false);
+
+  // stage 4.
+  HadamardRotation(&s[32], &s[33], false);
+  HadamardRotation(&s[34], &s[35], true);
+  HadamardRotation(&s[36], &s[37], false);
+  HadamardRotation(&s[38], &s[39], true);
+  HadamardRotation(&s[40], &s[41], false);
+  HadamardRotation(&s[42], &s[43], true);
+  HadamardRotation(&s[44], &s[45], false);
+  HadamardRotation(&s[46], &s[47], true);
+  HadamardRotation(&s[48], &s[49], false);
+  HadamardRotation(&s[50], &s[51], true);
+  HadamardRotation(&s[52], &s[53], false);
+  HadamardRotation(&s[54], &s[55], true);
+  HadamardRotation(&s[56], &s[57], false);
+  HadamardRotation(&s[58], &s[59], true);
+  HadamardRotation(&s[60], &s[61], false);
+  HadamardRotation(&s[62], &s[63], true);
+
+  // stage 7.
+  ButterflyRotation_8(&s[62], &s[33], 60 - 0, true);
+  ButterflyRotation_8(&s[61], &s[34], 60 - 0 + 64, true);
+  ButterflyRotation_8(&s[58], &s[37], 60 - 32, true);
+  ButterflyRotation_8(&s[57], &s[38], 60 - 32 + 64, true);
+  ButterflyRotation_8(&s[54], &s[41], 60 - 16, true);
+  ButterflyRotation_8(&s[53], &s[42], 60 - 16 + 64, true);
+  ButterflyRotation_8(&s[50], &s[45], 60 - 48, true);
+  ButterflyRotation_8(&s[49], &s[46], 60 - 48 + 64, true);
+
+  // stage 11.
+  HadamardRotation(&s[32], &s[35], false);
+  HadamardRotation(&s[33], &s[34], false);
+  HadamardRotation(&s[36], &s[39], true);
+  HadamardRotation(&s[37], &s[38], true);
+  HadamardRotation(&s[40], &s[43], false);
+  HadamardRotation(&s[41], &s[42], false);
+  HadamardRotation(&s[44], &s[47], true);
+  HadamardRotation(&s[45], &s[46], true);
+  HadamardRotation(&s[48], &s[51], false);
+  HadamardRotation(&s[49], &s[50], false);
+  HadamardRotation(&s[52], &s[55], true);
+  HadamardRotation(&s[53], &s[54], true);
+  HadamardRotation(&s[56], &s[59], false);
+  HadamardRotation(&s[57], &s[58], false);
+  HadamardRotation(&s[60], &s[63], true);
+  HadamardRotation(&s[61], &s[62], true);
+
+  // stage 16.
+  ButterflyRotation_8(&s[61], &s[34], 56, true);
+  ButterflyRotation_8(&s[60], &s[35], 56, true);
+  ButterflyRotation_8(&s[59], &s[36], 56 + 64, true);
+  ButterflyRotation_8(&s[58], &s[37], 56 + 64, true);
+  ButterflyRotation_8(&s[53], &s[42], 56 - 32, true);
+  ButterflyRotation_8(&s[52], &s[43], 56 - 32, true);
+  ButterflyRotation_8(&s[51], &s[44], 56 - 32 + 64, true);
+  ButterflyRotation_8(&s[50], &s[45], 56 - 32 + 64, true);
+
+  // stage 21.
+  HadamardRotation(&s[32], &s[39], false);
+  HadamardRotation(&s[33], &s[38], false);
+  HadamardRotation(&s[34], &s[37], false);
+  HadamardRotation(&s[35], &s[36], false);
+  HadamardRotation(&s[40], &s[47], true);
+  HadamardRotation(&s[41], &s[46], true);
+  HadamardRotation(&s[42], &s[45], true);
+  HadamardRotation(&s[43], &s[44], true);
+  HadamardRotation(&s[48], &s[55], false);
+  HadamardRotation(&s[49], &s[54], false);
+  HadamardRotation(&s[50], &s[53], false);
+  HadamardRotation(&s[51], &s[52], false);
+  HadamardRotation(&s[56], &s[63], true);
+  HadamardRotation(&s[57], &s[62], true);
+  HadamardRotation(&s[58], &s[61], true);
+  HadamardRotation(&s[59], &s[60], true);
+
+  // stage 25.
+  ButterflyRotation_8(&s[59], &s[36], 48, true);
+  ButterflyRotation_8(&s[58], &s[37], 48, true);
+  ButterflyRotation_8(&s[57], &s[38], 48, true);
+  ButterflyRotation_8(&s[56], &s[39], 48, true);
+  ButterflyRotation_8(&s[55], &s[40], 112, true);
+  ButterflyRotation_8(&s[54], &s[41], 112, true);
+  ButterflyRotation_8(&s[53], &s[42], 112, true);
+  ButterflyRotation_8(&s[52], &s[43], 112, true);
+
+  // stage 28.
+  HadamardRotation(&s[32], &s[47], false);
+  HadamardRotation(&s[33], &s[46], false);
+  HadamardRotation(&s[34], &s[45], false);
+  HadamardRotation(&s[35], &s[44], false);
+  HadamardRotation(&s[36], &s[43], false);
+  HadamardRotation(&s[37], &s[42], false);
+  HadamardRotation(&s[38], &s[41], false);
+  HadamardRotation(&s[39], &s[40], false);
+  HadamardRotation(&s[48], &s[63], true);
+  HadamardRotation(&s[49], &s[62], true);
+  HadamardRotation(&s[50], &s[61], true);
+  HadamardRotation(&s[51], &s[60], true);
+  HadamardRotation(&s[52], &s[59], true);
+  HadamardRotation(&s[53], &s[58], true);
+  HadamardRotation(&s[54], &s[57], true);
+  HadamardRotation(&s[55], &s[56], true);
+
+  // stage 30.
+  ButterflyRotation_8(&s[55], &s[40], 32, true);
+  ButterflyRotation_8(&s[54], &s[41], 32, true);
+  ButterflyRotation_8(&s[53], &s[42], 32, true);
+  ButterflyRotation_8(&s[52], &s[43], 32, true);
+  ButterflyRotation_8(&s[51], &s[44], 32, true);
+  ButterflyRotation_8(&s[50], &s[45], 32, true);
+  ButterflyRotation_8(&s[49], &s[46], 32, true);
+  ButterflyRotation_8(&s[48], &s[47], 32, true);
+
+  // stage 31.
+  for (int i = 0; i < 32; i += 4) {
+    HadamardRotation(&s[i], &s[63 - i], false);
+    HadamardRotation(&s[i + 1], &s[63 - i - 1], false);
+    HadamardRotation(&s[i + 2], &s[63 - i - 2], false);
+    HadamardRotation(&s[i + 3], &s[63 - i - 3], false);
+  }
+  //-- end dct 64 stages
+
+  if (transpose) {
+    for (int idx = 0; idx < 64; idx += 8) {
+      int16x8_t output[8];
+      Transpose8x8(&s[idx], output);
+      StoreDst<16, 8>(dst, step, idx, output);
+    }
+  } else {
+    StoreDst<16, 64>(dst, step, 0, s);
+  }
+}
+
+//------------------------------------------------------------------------------
+// Asymmetric Discrete Sine Transforms (ADST).
+template <bool stage_is_rectangular>
+LIBGAV1_ALWAYS_INLINE void Adst4_NEON(void* dest, const void* source,
+                                      int32_t step, bool transpose) {
+  auto* const dst = static_cast<int16_t*>(dest);
+  const auto* const src = static_cast<const int16_t*>(source);
+  int32x4_t s[8];
+  int16x8_t x[4];
+
+  if (stage_is_rectangular) {
+    if (transpose) {
+      int16x8_t input[8];
+      LoadSrc<8, 8>(src, step, 0, input);
+      Transpose4x8To8x4(input, x);
+    } else {
+      LoadSrc<16, 4>(src, step, 0, x);
+    }
+  } else {
+    LoadSrc<8, 4>(src, step, 0, x);
+    if (transpose) {
+      Transpose4x4(x, x);
+    }
+  }
+
+  const int16x4_t kAdst4Multiplier_0 = vdup_n_s16(kAdst4Multiplier[0]);
+  const int16x4_t kAdst4Multiplier_1 = vdup_n_s16(kAdst4Multiplier[1]);
+  const int16x4_t kAdst4Multiplier_2 = vdup_n_s16(kAdst4Multiplier[2]);
+  const int16x4_t kAdst4Multiplier_3 = vdup_n_s16(kAdst4Multiplier[3]);
+
+  // stage 1.
+  s[5] = vmull_s16(kAdst4Multiplier_1, vget_low_s16(x[3]));
+  s[6] = vmull_s16(kAdst4Multiplier_3, vget_low_s16(x[3]));
+
+  // stage 2.
+  const int32x4_t a7 = vsubl_s16(vget_low_s16(x[0]), vget_low_s16(x[2]));
+  const int32x4_t b7 = vaddw_s16(a7, vget_low_s16(x[3]));
+
+  // stage 3.
+  s[0] = vmull_s16(kAdst4Multiplier_0, vget_low_s16(x[0]));
+  s[1] = vmull_s16(kAdst4Multiplier_1, vget_low_s16(x[0]));
+  // s[0] = s[0] + s[3]
+  s[0] = vmlal_s16(s[0], kAdst4Multiplier_3, vget_low_s16(x[2]));
+  // s[1] = s[1] - s[4]
+  s[1] = vmlsl_s16(s[1], kAdst4Multiplier_0, vget_low_s16(x[2]));
+
+  s[3] = vmull_s16(kAdst4Multiplier_2, vget_low_s16(x[1]));
+  s[2] = vmulq_s32(vmovl_s16(kAdst4Multiplier_2), b7);
+
+  // stage 4.
+  s[0] = vaddq_s32(s[0], s[5]);
+  s[1] = vsubq_s32(s[1], s[6]);
+
+  // stages 5 and 6.
+  const int32x4_t x0 = vaddq_s32(s[0], s[3]);
+  const int32x4_t x1 = vaddq_s32(s[1], s[3]);
+  const int32x4_t x3_a = vaddq_s32(s[0], s[1]);
+  const int32x4_t x3 = vsubq_s32(x3_a, s[3]);
+  const int16x4_t dst_0 = vqrshrn_n_s32(x0, 12);
+  const int16x4_t dst_1 = vqrshrn_n_s32(x1, 12);
+  const int16x4_t dst_2 = vqrshrn_n_s32(s[2], 12);
+  const int16x4_t dst_3 = vqrshrn_n_s32(x3, 12);
+
+  x[0] = vcombine_s16(dst_0, dst_0);
+  x[1] = vcombine_s16(dst_1, dst_1);
+  x[2] = vcombine_s16(dst_2, dst_2);
+  x[3] = vcombine_s16(dst_3, dst_3);
+
+  if (stage_is_rectangular) {
+    if (transpose) {
+      int16x8_t output[8];
+      Transpose8x4To4x8(x, output);
+      StoreDst<8, 8>(dst, step, 0, output);
+    } else {
+      StoreDst<16, 4>(dst, step, 0, x);
+    }
+  } else {
+    if (transpose) {
+      Transpose4x4(x, x);
+    }
+    StoreDst<8, 4>(dst, step, 0, x);
+  }
+}
+
+template <ButterflyRotationFunc bufferfly_rotation, bool stage_is_rectangular>
+LIBGAV1_ALWAYS_INLINE void Adst8_NEON(void* dest, const void* source,
+                                      int32_t step, bool transpose) {
+  auto* const dst = static_cast<int16_t*>(dest);
+  const auto* const src = static_cast<const int16_t*>(source);
+  int16x8_t s[8], x[8];
+
+  if (stage_is_rectangular) {
+    if (transpose) {
+      int16x8_t input[4];
+      LoadSrc<16, 4>(src, step, 0, input);
+      Transpose8x4To4x8(input, x);
+    } else {
+      LoadSrc<8, 8>(src, step, 0, x);
+    }
+  } else {
+    if (transpose) {
+      int16x8_t input[8];
+      LoadSrc<16, 8>(src, step, 0, input);
+      Transpose8x8(input, x);
+    } else {
+      LoadSrc<16, 8>(src, step, 0, x);
+    }
+  }
+
+  // stage 1.
+  s[0] = x[7];
+  s[1] = x[0];
+  s[2] = x[5];
+  s[3] = x[2];
+  s[4] = x[3];
+  s[5] = x[4];
+  s[6] = x[1];
+  s[7] = x[6];
+
+  // stage 2.
+  bufferfly_rotation(&s[0], &s[1], 60 - 0, true);
+  bufferfly_rotation(&s[2], &s[3], 60 - 16, true);
+  bufferfly_rotation(&s[4], &s[5], 60 - 32, true);
+  bufferfly_rotation(&s[6], &s[7], 60 - 48, true);
+
+  // stage 3.
+  HadamardRotation(&s[0], &s[4], false);
+  HadamardRotation(&s[1], &s[5], false);
+  HadamardRotation(&s[2], &s[6], false);
+  HadamardRotation(&s[3], &s[7], false);
+
+  // stage 4.
+  bufferfly_rotation(&s[4], &s[5], 48 - 0, true);
+  bufferfly_rotation(&s[7], &s[6], 48 - 32, true);
+
+  // stage 5.
+  HadamardRotation(&s[0], &s[2], false);
+  HadamardRotation(&s[4], &s[6], false);
+  HadamardRotation(&s[1], &s[3], false);
+  HadamardRotation(&s[5], &s[7], false);
+
+  // stage 6.
+  bufferfly_rotation(&s[2], &s[3], 32, true);
+  bufferfly_rotation(&s[6], &s[7], 32, true);
+
+  // stage 7.
+  x[0] = s[0];
+  x[1] = vqnegq_s16(s[4]);
+  x[2] = s[6];
+  x[3] = vqnegq_s16(s[2]);
+  x[4] = s[3];
+  x[5] = vqnegq_s16(s[7]);
+  x[6] = s[5];
+  x[7] = vqnegq_s16(s[1]);
+
+  if (stage_is_rectangular) {
+    if (transpose) {
+      int16x8_t output[4];
+      Transpose4x8To8x4(x, output);
+      StoreDst<16, 4>(dst, step, 0, output);
+    } else {
+      StoreDst<8, 8>(dst, step, 0, x);
+    }
+  } else {
+    if (transpose) {
+      int16x8_t output[8];
+      Transpose8x8(x, output);
+      StoreDst<16, 8>(dst, step, 0, output);
+    } else {
+      StoreDst<16, 8>(dst, step, 0, x);
+    }
+  }
+}
+
+template <ButterflyRotationFunc bufferfly_rotation, bool stage_is_rectangular>
+LIBGAV1_ALWAYS_INLINE void Adst16_NEON(void* dest, const void* source,
+                                       int32_t step, bool transpose) {
+  auto* const dst = static_cast<int16_t*>(dest);
+  const auto* const src = static_cast<const int16_t*>(source);
+  int16x8_t s[16], x[16];
+
+  if (stage_is_rectangular) {
+    if (transpose) {
+      int16x8_t input[4];
+      LoadSrc<16, 4>(src, step, 0, input);
+      Transpose8x4To4x8(input, x);
+      LoadSrc<16, 4>(src, step, 8, input);
+      Transpose8x4To4x8(input, &x[8]);
+    } else {
+      LoadSrc<8, 16>(src, step, 0, x);
+    }
+  } else {
+    if (transpose) {
+      for (int idx = 0; idx < 16; idx += 8) {
+        int16x8_t input[8];
+        LoadSrc<16, 8>(src, step, idx, input);
+        Transpose8x8(input, &x[idx]);
+      }
+    } else {
+      LoadSrc<16, 16>(src, step, 0, x);
+    }
+  }
+
+  // stage 1.
+  s[0] = x[15];
+  s[1] = x[0];
+  s[2] = x[13];
+  s[3] = x[2];
+  s[4] = x[11];
+  s[5] = x[4];
+  s[6] = x[9];
+  s[7] = x[6];
+  s[8] = x[7];
+  s[9] = x[8];
+  s[10] = x[5];
+  s[11] = x[10];
+  s[12] = x[3];
+  s[13] = x[12];
+  s[14] = x[1];
+  s[15] = x[14];
+
+  // stage 2.
+  bufferfly_rotation(&s[0], &s[1], 62 - 0, true);
+  bufferfly_rotation(&s[2], &s[3], 62 - 8, true);
+  bufferfly_rotation(&s[4], &s[5], 62 - 16, true);
+  bufferfly_rotation(&s[6], &s[7], 62 - 24, true);
+  bufferfly_rotation(&s[8], &s[9], 62 - 32, true);
+  bufferfly_rotation(&s[10], &s[11], 62 - 40, true);
+  bufferfly_rotation(&s[12], &s[13], 62 - 48, true);
+  bufferfly_rotation(&s[14], &s[15], 62 - 56, true);
+
+  // stage 3.
+  HadamardRotation(&s[0], &s[8], false);
+  HadamardRotation(&s[1], &s[9], false);
+  HadamardRotation(&s[2], &s[10], false);
+  HadamardRotation(&s[3], &s[11], false);
+  HadamardRotation(&s[4], &s[12], false);
+  HadamardRotation(&s[5], &s[13], false);
+  HadamardRotation(&s[6], &s[14], false);
+  HadamardRotation(&s[7], &s[15], false);
+
+  // stage 4.
+  bufferfly_rotation(&s[8], &s[9], 56 - 0, true);
+  bufferfly_rotation(&s[13], &s[12], 8 + 0, true);
+  bufferfly_rotation(&s[10], &s[11], 56 - 32, true);
+  bufferfly_rotation(&s[15], &s[14], 8 + 32, true);
+
+  // stage 5.
+  HadamardRotation(&s[0], &s[4], false);
+  HadamardRotation(&s[8], &s[12], false);
+  HadamardRotation(&s[1], &s[5], false);
+  HadamardRotation(&s[9], &s[13], false);
+  HadamardRotation(&s[2], &s[6], false);
+  HadamardRotation(&s[10], &s[14], false);
+  HadamardRotation(&s[3], &s[7], false);
+  HadamardRotation(&s[11], &s[15], false);
+
+  // stage 6.
+  bufferfly_rotation(&s[4], &s[5], 48 - 0, true);
+  bufferfly_rotation(&s[12], &s[13], 48 - 0, true);
+  bufferfly_rotation(&s[7], &s[6], 48 - 32, true);
+  bufferfly_rotation(&s[15], &s[14], 48 - 32, true);
+
+  // stage 7.
+  HadamardRotation(&s[0], &s[2], false);
+  HadamardRotation(&s[4], &s[6], false);
+  HadamardRotation(&s[8], &s[10], false);
+  HadamardRotation(&s[12], &s[14], false);
+  HadamardRotation(&s[1], &s[3], false);
+  HadamardRotation(&s[5], &s[7], false);
+  HadamardRotation(&s[9], &s[11], false);
+  HadamardRotation(&s[13], &s[15], false);
+
+  // stage 8.
+  bufferfly_rotation(&s[2], &s[3], 32, true);
+  bufferfly_rotation(&s[6], &s[7], 32, true);
+  bufferfly_rotation(&s[10], &s[11], 32, true);
+  bufferfly_rotation(&s[14], &s[15], 32, true);
+
+  // stage 9.
+  x[0] = s[0];
+  x[1] = vqnegq_s16(s[8]);
+  x[2] = s[12];
+  x[3] = vqnegq_s16(s[4]);
+  x[4] = s[6];
+  x[5] = vqnegq_s16(s[14]);
+  x[6] = s[10];
+  x[7] = vqnegq_s16(s[2]);
+  x[8] = s[3];
+  x[9] = vqnegq_s16(s[11]);
+  x[10] = s[15];
+  x[11] = vqnegq_s16(s[7]);
+  x[12] = s[5];
+  x[13] = vqnegq_s16(s[13]);
+  x[14] = s[9];
+  x[15] = vqnegq_s16(s[1]);
+
+  if (stage_is_rectangular) {
+    if (transpose) {
+      int16x8_t output[4];
+      Transpose4x8To8x4(x, output);
+      StoreDst<16, 4>(dst, step, 0, output);
+      Transpose4x8To8x4(&x[8], output);
+      StoreDst<16, 4>(dst, step, 8, output);
+    } else {
+      StoreDst<8, 16>(dst, step, 0, x);
+    }
+  } else {
+    if (transpose) {
+      for (int idx = 0; idx < 16; idx += 8) {
+        int16x8_t output[8];
+        Transpose8x8(&x[idx], output);
+        StoreDst<16, 8>(dst, step, idx, output);
+      }
+    } else {
+      StoreDst<16, 16>(dst, step, 0, x);
+    }
+  }
+}
+
+//------------------------------------------------------------------------------
+// Identity Transforms.
+
+template <bool is_row_shift>
+LIBGAV1_ALWAYS_INLINE void Identity4_NEON(void* dest, const void* source,
+                                          int32_t step) {
+  auto* const dst = static_cast<int16_t*>(dest);
+  const auto* const src = static_cast<const int16_t*>(source);
+
+  if (is_row_shift) {
+    const int shift = 1;
+    const int32x4_t v_dual_round = vdupq_n_s32((1 + (shift << 1)) << 11);
+    const int16x4_t v_multiplier = vdup_n_s16(kIdentity4Multiplier);
+    const int32x4_t v_shift = vdupq_n_s32(-(12 + shift));
+    for (int i = 0; i < 4; i += 2) {
+      const int16x8_t v_src = vld1q_s16(&src[i * step]);
+      const int32x4_t v_src_mult_lo =
+          vmlal_s16(v_dual_round, vget_low_s16(v_src), v_multiplier);
+      const int32x4_t v_src_mult_hi =
+          vmlal_s16(v_dual_round, vget_high_s16(v_src), v_multiplier);
+      const int32x4_t shift_lo = vqshlq_s32(v_src_mult_lo, v_shift);
+      const int32x4_t shift_hi = vqshlq_s32(v_src_mult_hi, v_shift);
+      vst1q_s16(&dst[i * step],
+                vcombine_s16(vqmovn_s32(shift_lo), vqmovn_s32(shift_hi)));
+    }
+  } else {
+    for (int i = 0; i < 4; i += 2) {
+      const int16x8_t v_src = vld1q_s16(&src[i * step]);
+      const int16x8_t a =
+          vqrdmulhq_n_s16(v_src, kIdentity4MultiplierFraction << 3);
+      const int16x8_t b = vqaddq_s16(v_src, a);
+      vst1q_s16(&dst[i * step], b);
+    }
+  }
+}
+
+LIBGAV1_ALWAYS_INLINE void Identity4ColumnStoreToFrame(
+    Array2DView<uint8_t> frame, const int start_x, const int start_y,
+    const int tx_width, const int tx_height, const int16_t* source) {
+  const int stride = frame.columns();
+  uint8_t* dst = frame[start_y] + start_x;
+
+  if (tx_width == 4) {
+    const uint8x8_t zero = vdup_n_u8(0);
+    for (int i = 0; i < tx_height; ++i) {
+      const int16x4_t residual = vld1_s16(&source[i * tx_width]);
+      const int16x4_t residual_fraction =
+          vqrdmulh_n_s16(residual, kIdentity4MultiplierFraction << 3);
+      const int16x4_t v_dst_i = vqadd_s16(residual, residual_fraction);
+      const int16x8_t frame_data =
+          vreinterpretq_s16_u16(vmovl_u8(LoadLo4(dst, zero)));
+      const int16x4_t a = vrshr_n_s16(v_dst_i, 4);
+      const int16x4_t b = vqadd_s16(a, vget_low_s16(frame_data));
+      const uint8x8_t d = vqmovun_s16(vcombine_s16(b, b));
+      StoreLo4(dst, d);
+      dst += stride;
+    }
+  } else {
+    for (int i = 0; i < tx_height; ++i) {
+      const int row = i * tx_width;
+      int j = 0;
+      do {
+        const int16x8_t residual = vld1q_s16(&source[row + j]);
+        const int16x8_t residual_fraction =
+            vqrdmulhq_n_s16(residual, kIdentity4MultiplierFraction << 3);
+        const int16x8_t v_dst_i = vqaddq_s16(residual, residual_fraction);
+        const int16x8_t frame_data =
+            vreinterpretq_s16_u16(vmovl_u8(vld1_u8(dst + j)));
+        const int16x8_t a = vrshrq_n_s16(v_dst_i, 4);
+        const int16x8_t b = vqaddq_s16(a, frame_data);
+        const uint8x8_t d = vqmovun_s16(b);
+        vst1_u8(dst + j, d);
+        j += 8;
+      } while (j < tx_width);
+      dst += stride;
+    }
+  }
+}
+
+LIBGAV1_ALWAYS_INLINE void Identity4RowColumnStoreToFrame(
+    Array2DView<uint8_t> frame, const int start_x, const int start_y,
+    const int tx_width, const int tx_height, const int16_t* source) {
+  const int stride = frame.columns();
+  uint8_t* dst = frame[start_y] + start_x;
+
+  if (tx_width == 4) {
+    const uint8x8_t zero = vdup_n_u8(0);
+    for (int i = 0; i < tx_height; ++i) {
+      const int16x4_t v_src = vld1_s16(&source[i * tx_width]);
+      const int16x4_t v_src_mult =
+          vqrdmulh_n_s16(v_src, kIdentity4MultiplierFraction << 3);
+      const int16x8_t frame_data =
+          vreinterpretq_s16_u16(vmovl_u8(LoadLo4(dst, zero)));
+      const int16x4_t v_dst_row = vqadd_s16(v_src, v_src_mult);
+      const int16x4_t v_src_mult2 =
+          vqrdmulh_n_s16(v_dst_row, kIdentity4MultiplierFraction << 3);
+      const int16x4_t v_dst_col = vqadd_s16(v_dst_row, v_src_mult2);
+      const int16x4_t a = vrshr_n_s16(v_dst_col, 4);
+      const int16x4_t b = vqadd_s16(a, vget_low_s16(frame_data));
+      const uint8x8_t d = vqmovun_s16(vcombine_s16(b, b));
+      StoreLo4(dst, d);
+      dst += stride;
+    }
+  } else {
+    for (int i = 0; i < tx_height; ++i) {
+      const int row = i * tx_width;
+      int j = 0;
+      do {
+        const int16x8_t v_src = vld1q_s16(&source[row + j]);
+        const int16x8_t v_src_round =
+            vqrdmulhq_n_s16(v_src, kTransformRowMultiplier << 3);
+        const int16x8_t v_dst_row = vqaddq_s16(v_src_round, v_src_round);
+        const int16x8_t v_src_mult2 =
+            vqrdmulhq_n_s16(v_dst_row, kIdentity4MultiplierFraction << 3);
+        const int16x8_t frame_data =
+            vreinterpretq_s16_u16(vmovl_u8(vld1_u8(dst + j)));
+        const int16x8_t v_dst_col = vqaddq_s16(v_dst_row, v_src_mult2);
+        const int16x8_t a = vrshrq_n_s16(v_dst_col, 4);
+        const int16x8_t b = vqaddq_s16(a, frame_data);
+        const uint8x8_t d = vqmovun_s16(b);
+        vst1_u8(dst + j, d);
+        j += 8;
+      } while (j < tx_width);
+      dst += stride;
+    }
+  }
+}
+
+LIBGAV1_ALWAYS_INLINE void Identity8Row32_NEON(void* dest, const void* source,
+                                               int32_t step) {
+  auto* const dst = static_cast<int16_t*>(dest);
+  const auto* const src = static_cast<const int16_t*>(source);
+
+  // When combining the identity8 multiplier with the row shift, the
+  // calculations for tx_height equal to 32 can be simplified from
+  // ((A * 2) + 2) >> 2) to ((A + 1) >> 1).
+  for (int i = 0; i < 4; ++i) {
+    const int16x8_t v_src = vld1q_s16(&src[i * step]);
+    const int16x8_t a = vrshrq_n_s16(v_src, 1);
+    vst1q_s16(&dst[i * step], a);
+  }
+}
+
+LIBGAV1_ALWAYS_INLINE void Identity8Row4_NEON(void* dest, const void* source,
+                                              int32_t step) {
+  auto* const dst = static_cast<int16_t*>(dest);
+  const auto* const src = static_cast<const int16_t*>(source);
+
+  for (int i = 0; i < 4; ++i) {
+    const int16x8_t v_src = vld1q_s16(&src[i * step]);
+    // For bitdepth == 8, the identity row clamps to a signed 16bit value, so
+    // saturating add here is ok.
+    const int16x8_t v_srcx2 = vqaddq_s16(v_src, v_src);
+    vst1q_s16(&dst[i * step], v_srcx2);
+  }
+}
+
+LIBGAV1_ALWAYS_INLINE void Identity8ColumnStoreToFrame_NEON(
+    Array2DView<uint8_t> frame, const int start_x, const int start_y,
+    const int tx_width, const int tx_height, const int16_t* source) {
+  const int stride = frame.columns();
+  uint8_t* dst = frame[start_y] + start_x;
+
+  if (tx_width == 4) {
+    const uint8x8_t zero = vdup_n_u8(0);
+    for (int i = 0; i < tx_height; ++i) {
+      const int16x4_t residual = vld1_s16(&source[i * tx_width]);
+      const int16x4_t v_dst_i = vqadd_s16(residual, residual);
+      const int16x8_t frame_data =
+          vreinterpretq_s16_u16(vmovl_u8(LoadLo4(dst, zero)));
+      const int16x4_t a = vrshr_n_s16(v_dst_i, 4);
+      const int16x4_t b = vqadd_s16(a, vget_low_s16(frame_data));
+      const uint8x8_t d = vqmovun_s16(vcombine_s16(b, b));
+      StoreLo4(dst, d);
+      dst += stride;
+    }
+  } else {
+    for (int i = 0; i < tx_height; ++i) {
+      const int row = i * tx_width;
+      for (int j = 0; j < tx_width; j += 8) {
+        const int16x8_t residual = vld1q_s16(&source[row + j]);
+        const int16x8_t v_dst_i = vqaddq_s16(residual, residual);
+        const int16x8_t frame_data =
+            vreinterpretq_s16_u16(vmovl_u8(vld1_u8(dst + j)));
+        const int16x8_t a = vrshrq_n_s16(v_dst_i, 4);
+        const int16x8_t b = vqaddq_s16(a, frame_data);
+        const uint8x8_t d = vqmovun_s16(b);
+        vst1_u8(dst + j, d);
+      }
+      dst += stride;
+    }
+  }
+}
+
+LIBGAV1_ALWAYS_INLINE void Identity16Row_NEON(void* dest, const void* source,
+                                              int32_t step, int shift) {
+  auto* const dst = static_cast<int16_t*>(dest);
+  const auto* const src = static_cast<const int16_t*>(source);
+  const int32x4_t v_dual_round = vdupq_n_s32((1 + (shift << 1)) << 11);
+  const int16x4_t v_multiplier = vdup_n_s16(kIdentity16Multiplier);
+  const int32x4_t v_shift = vdupq_n_s32(-(12 + shift));
+
+  for (int i = 0; i < 4; ++i) {
+    for (int j = 0; j < 2; ++j) {
+      const int16x8_t v_src = vld1q_s16(&src[i * step + j * 8]);
+      const int32x4_t v_src_mult_lo =
+          vmlal_s16(v_dual_round, vget_low_s16(v_src), v_multiplier);
+      const int32x4_t v_src_mult_hi =
+          vmlal_s16(v_dual_round, vget_high_s16(v_src), v_multiplier);
+      const int32x4_t shift_lo = vqshlq_s32(v_src_mult_lo, v_shift);
+      const int32x4_t shift_hi = vqshlq_s32(v_src_mult_hi, v_shift);
+      vst1q_s16(&dst[i * step + j * 8],
+                vcombine_s16(vqmovn_s32(shift_lo), vqmovn_s32(shift_hi)));
+    }
+  }
+}
+
+LIBGAV1_ALWAYS_INLINE void Identity16ColumnStoreToFrame_NEON(
+    Array2DView<uint8_t> frame, const int start_x, const int start_y,
+    const int tx_width, const int tx_height, const int16_t* source) {
+  const int stride = frame.columns();
+  uint8_t* dst = frame[start_y] + start_x;
+
+  if (tx_width == 4) {
+    const uint8x8_t zero = vdup_n_u8(0);
+    for (int i = 0; i < tx_height; ++i) {
+      const int16x4_t v_src = vld1_s16(&source[i * tx_width]);
+      const int16x4_t v_src_mult =
+          vqrdmulh_n_s16(v_src, kIdentity4MultiplierFraction << 4);
+      const int16x8_t frame_data =
+          vreinterpretq_s16_u16(vmovl_u8(LoadLo4(dst, zero)));
+
+      const int16x4_t v_srcx2 = vqadd_s16(v_src, v_src);
+      const int16x4_t v_dst_i = vqadd_s16(v_srcx2, v_src_mult);
+
+      const int16x4_t a = vrshr_n_s16(v_dst_i, 4);
+      const int16x4_t b = vqadd_s16(a, vget_low_s16(frame_data));
+      const uint8x8_t d = vqmovun_s16(vcombine_s16(b, b));
+      StoreLo4(dst, d);
+      dst += stride;
+    }
+  } else {
+    for (int i = 0; i < tx_height; ++i) {
+      const int row = i * tx_width;
+      for (int j = 0; j < tx_width; j += 8) {
+        const int16x8_t v_src = vld1q_s16(&source[row + j]);
+        const int16x8_t v_src_mult =
+            vqrdmulhq_n_s16(v_src, kIdentity4MultiplierFraction << 4);
+        const int16x8_t frame_data =
+            vreinterpretq_s16_u16(vmovl_u8(vld1_u8(dst + j)));
+        const int16x8_t v_srcx2 = vqaddq_s16(v_src, v_src);
+        const int16x8_t v_dst_i = vqaddq_s16(v_src_mult, v_srcx2);
+        const int16x8_t a = vrshrq_n_s16(v_dst_i, 4);
+        const int16x8_t b = vqaddq_s16(a, frame_data);
+        const uint8x8_t d = vqmovun_s16(b);
+        vst1_u8(dst + j, d);
+      }
+      dst += stride;
+    }
+  }
+}
+
+LIBGAV1_ALWAYS_INLINE void Identity32Row16_NEON(void* dest, const void* source,
+                                                const int32_t step) {
+  auto* const dst = static_cast<int16_t*>(dest);
+  const auto* const src = static_cast<const int16_t*>(source);
+
+  // When combining the identity32 multiplier with the row shift, the
+  // calculation for tx_height equal to 16 can be simplified from
+  // ((A * 4) + 1) >> 1) to (A * 2).
+  for (int i = 0; i < 4; ++i) {
+    for (int j = 0; j < 32; j += 8) {
+      const int16x8_t v_src = vld1q_s16(&src[i * step + j]);
+      // For bitdepth == 8, the identity row clamps to a signed 16bit value, so
+      // saturating add here is ok.
+      const int16x8_t v_dst_i = vqaddq_s16(v_src, v_src);
+      vst1q_s16(&dst[i * step + j], v_dst_i);
+    }
+  }
+}
+
+LIBGAV1_ALWAYS_INLINE void Identity32ColumnStoreToFrame(
+    Array2DView<uint8_t> frame, const int start_x, const int start_y,
+    const int tx_width, const int tx_height, const int16_t* source) {
+  const int stride = frame.columns();
+  uint8_t* dst = frame[start_y] + start_x;
+
+  for (int i = 0; i < tx_height; ++i) {
+    const int row = i * tx_width;
+    int j = 0;
+    do {
+      const int16x8_t v_dst_i = vld1q_s16(&source[row + j]);
+      const int16x8_t frame_data =
+          vreinterpretq_s16_u16(vmovl_u8(vld1_u8(dst + j)));
+      const int16x8_t a = vrshrq_n_s16(v_dst_i, 2);
+      const int16x8_t b = vqaddq_s16(a, frame_data);
+      const uint8x8_t d = vqmovun_s16(b);
+      vst1_u8(dst + j, d);
+      j += 8;
+    } while (j < tx_width);
+    dst += stride;
+  }
+}
+
+//------------------------------------------------------------------------------
+// Walsh Hadamard Transform.
+
+// Transposes a 4x4 matrix and then permutes the rows of the transposed matrix
+// for the WHT. The input matrix is in two "wide" int16x8_t variables. The
+// output matrix is in four int16x4_t variables.
+//
+// Input:
+// in[0]: 00 01 02 03  10 11 12 13
+// in[1]: 20 21 22 23  30 31 32 33
+// Output:
+// out[0]: 00 10 20 30
+// out[1]: 03 13 23 33
+// out[2]: 01 11 21 31
+// out[3]: 02 12 22 32
+LIBGAV1_ALWAYS_INLINE void TransposeAndPermute4x4WideInput(
+    const int16x8_t in[2], int16x4_t out[4]) {
+  // Swap 32 bit elements. Goes from:
+  // in[0]: 00 01 02 03  10 11 12 13
+  // in[1]: 20 21 22 23  30 31 32 33
+  // to:
+  // b0.val[0]: 00 01 20 21  10 11 30 31
+  // b0.val[1]: 02 03 22 23  12 13 32 33
+
+  const int32x4x2_t b0 =
+      vtrnq_s32(vreinterpretq_s32_s16(in[0]), vreinterpretq_s32_s16(in[1]));
+
+  // Swap 16 bit elements. Goes from:
+  // vget_low_s32(b0.val[0]):  00 01 20 21
+  // vget_high_s32(b0.val[0]): 10 11 30 31
+  // vget_low_s32(b0.val[1]):  02 03 22 23
+  // vget_high_s32(b0.val[1]): 12 13 32 33
+  // to:
+  // c0.val[0]: 00 10 20 30
+  // c0.val[1]: 01 11 21 32
+  // c1.val[0]: 02 12 22 32
+  // c1.val[1]: 03 13 23 33
+
+  const int16x4x2_t c0 =
+      vtrn_s16(vreinterpret_s16_s32(vget_low_s32(b0.val[0])),
+               vreinterpret_s16_s32(vget_high_s32(b0.val[0])));
+  const int16x4x2_t c1 =
+      vtrn_s16(vreinterpret_s16_s32(vget_low_s32(b0.val[1])),
+               vreinterpret_s16_s32(vget_high_s32(b0.val[1])));
+
+  out[0] = c0.val[0];
+  out[1] = c1.val[1];
+  out[2] = c0.val[1];
+  out[3] = c1.val[0];
+}
+
+// Process 4 wht4 rows and columns.
+LIBGAV1_ALWAYS_INLINE void Wht4_NEON(uint8_t* dst, const int dst_stride,
+                                     const void* source,
+                                     const int non_zero_coeff_count) {
+  const auto* const src = static_cast<const int16_t*>(source);
+  int16x4_t s[4];
+
+  if (non_zero_coeff_count == 1) {
+    // Special case: only src[0] is nonzero.
+    //   src[0]  0   0   0
+    //       0   0   0   0
+    //       0   0   0   0
+    //       0   0   0   0
+    //
+    // After the row and column transforms are applied, we have:
+    //       f   h   h   h
+    //       g   i   i   i
+    //       g   i   i   i
+    //       g   i   i   i
+    // where f, g, h, i are computed as follows.
+    int16_t f = (src[0] >> 2) - (src[0] >> 3);
+    const int16_t g = f >> 1;
+    f = f - (f >> 1);
+    const int16_t h = (src[0] >> 3) - (src[0] >> 4);
+    const int16_t i = (src[0] >> 4);
+    s[0] = vdup_n_s16(h);
+    s[0] = vset_lane_s16(f, s[0], 0);
+    s[1] = vdup_n_s16(i);
+    s[1] = vset_lane_s16(g, s[1], 0);
+    s[2] = s[3] = s[1];
+  } else {
+    // Load the 4x4 source in transposed form.
+    int16x4x4_t columns = vld4_s16(src);
+    // Shift right and permute the columns for the WHT.
+    s[0] = vshr_n_s16(columns.val[0], 2);
+    s[2] = vshr_n_s16(columns.val[1], 2);
+    s[3] = vshr_n_s16(columns.val[2], 2);
+    s[1] = vshr_n_s16(columns.val[3], 2);
+
+    // Row transforms.
+    s[0] = vadd_s16(s[0], s[2]);
+    s[3] = vsub_s16(s[3], s[1]);
+    int16x4_t e = vhsub_s16(s[0], s[3]);  // e = (s[0] - s[3]) >> 1
+    s[1] = vsub_s16(e, s[1]);
+    s[2] = vsub_s16(e, s[2]);
+    s[0] = vsub_s16(s[0], s[1]);
+    s[3] = vadd_s16(s[3], s[2]);
+
+    int16x8_t x[2];
+    x[0] = vcombine_s16(s[0], s[1]);
+    x[1] = vcombine_s16(s[2], s[3]);
+    TransposeAndPermute4x4WideInput(x, s);
+
+    // Column transforms.
+    s[0] = vadd_s16(s[0], s[2]);
+    s[3] = vsub_s16(s[3], s[1]);
+    e = vhsub_s16(s[0], s[3]);  // e = (s[0] - s[3]) >> 1
+    s[1] = vsub_s16(e, s[1]);
+    s[2] = vsub_s16(e, s[2]);
+    s[0] = vsub_s16(s[0], s[1]);
+    s[3] = vadd_s16(s[3], s[2]);
+  }
+
+  // Store to frame.
+  uint8x8_t frame_data = vdup_n_u8(0);
+  for (int row = 0; row < 4; row += 2) {
+    frame_data = LoadLo4(dst, frame_data);
+    frame_data = LoadHi4(dst + dst_stride, frame_data);
+    const int16x8_t a = vreinterpretq_s16_u16(vmovl_u8(frame_data));
+    const int16x8_t residual = vcombine_s16(s[row], s[row + 1]);
+    // Saturate to prevent overflowing int16_t
+    const int16x8_t b = vqaddq_s16(a, residual);
+    frame_data = vqmovun_s16(b);
+    StoreLo4(dst, frame_data);
+    dst += dst_stride;
+    StoreHi4(dst, frame_data);
+    dst += dst_stride;
+  }
+}
+
+//------------------------------------------------------------------------------
+// row/column transform loops
+
+template <int tx_height>
+LIBGAV1_ALWAYS_INLINE void FlipColumns(int16_t* source, int tx_width) {
+  if (tx_width >= 16) {
+    int i = 0;
+    do {
+      const int16x8_t a = vld1q_s16(&source[i]);
+      const int16x8_t b = vld1q_s16(&source[i + 8]);
+      const int16x8_t c = vrev64q_s16(a);
+      const int16x8_t d = vrev64q_s16(b);
+      vst1q_s16(&source[i], vcombine_s16(vget_high_s16(d), vget_low_s16(d)));
+      vst1q_s16(&source[i + 8],
+                vcombine_s16(vget_high_s16(c), vget_low_s16(c)));
+      i += 16;
+    } while (i < tx_width * tx_height);
+  } else if (tx_width == 8) {
+    for (int i = 0; i < 8 * tx_height; i += 8) {
+      const int16x8_t a = vld1q_s16(&source[i]);
+      const int16x8_t b = vrev64q_s16(a);
+      vst1q_s16(&source[i], vcombine_s16(vget_high_s16(b), vget_low_s16(b)));
+    }
+  } else {
+    // Process two rows per iteration.
+    for (int i = 0; i < 4 * tx_height; i += 8) {
+      const int16x8_t a = vld1q_s16(&source[i]);
+      vst1q_s16(&source[i], vrev64q_s16(a));
+    }
+  }
+}
+
+template <int tx_width>
+LIBGAV1_ALWAYS_INLINE void ApplyRounding(int16_t* source, int num_rows) {
+  if (tx_width == 4) {
+    // Process two rows per iteration.
+    int i = 0;
+    do {
+      const int16x8_t a = vld1q_s16(&source[i]);
+      const int16x8_t b = vqrdmulhq_n_s16(a, kTransformRowMultiplier << 3);
+      vst1q_s16(&source[i], b);
+      i += 8;
+    } while (i < tx_width * num_rows);
+  } else {
+    int i = 0;
+    do {
+      // The last 32 values of every row are always zero if the |tx_width| is
+      // 64.
+      const int non_zero_width = (tx_width < 64) ? tx_width : 32;
+      int j = 0;
+      do {
+        const int16x8_t a = vld1q_s16(&source[i * tx_width + j]);
+        const int16x8_t b = vqrdmulhq_n_s16(a, kTransformRowMultiplier << 3);
+        vst1q_s16(&source[i * tx_width + j], b);
+        j += 8;
+      } while (j < non_zero_width);
+    } while (++i < num_rows);
+  }
+}
+
+template <int tx_width>
+LIBGAV1_ALWAYS_INLINE void RowShift(int16_t* source, int num_rows,
+                                    int row_shift) {
+  // vqrshlq_s16 will shift right if shift value is negative.
+  row_shift = -row_shift;
+
+  if (tx_width == 4) {
+    // Process two rows per iteration.
+    int i = 0;
+    do {
+      const int16x8_t residual = vld1q_s16(&source[i]);
+      vst1q_s16(&source[i], vqrshlq_s16(residual, vdupq_n_s16(row_shift)));
+      i += 8;
+    } while (i < tx_width * num_rows);
+  } else {
+    int i = 0;
+    do {
+      for (int j = 0; j < tx_width; j += 8) {
+        const int16x8_t residual = vld1q_s16(&source[i * tx_width + j]);
+        const int16x8_t residual_shifted =
+            vqrshlq_s16(residual, vdupq_n_s16(row_shift));
+        vst1q_s16(&source[i * tx_width + j], residual_shifted);
+      }
+    } while (++i < num_rows);
+  }
+}
+
+template <bool enable_flip_rows = false>
+LIBGAV1_ALWAYS_INLINE void StoreToFrameWithRound(
+    Array2DView<uint8_t> frame, const int start_x, const int start_y,
+    const int tx_width, const int tx_height, const int16_t* source,
+    TransformType tx_type) {
+  const bool flip_rows =
+      enable_flip_rows ? kTransformFlipRowsMask.Contains(tx_type) : false;
+  const int stride = frame.columns();
+  uint8_t* dst = frame[start_y] + start_x;
+
+  if (tx_width == 4) {
+    const uint8x8_t zero = vdup_n_u8(0);
+    for (int i = 0; i < tx_height; ++i) {
+      const int row = flip_rows ? (tx_height - i - 1) * 4 : i * 4;
+      const int16x4_t residual = vld1_s16(&source[row]);
+      const int16x8_t frame_data =
+          vreinterpretq_s16_u16(vmovl_u8(LoadLo4(dst, zero)));
+      const int16x4_t a = vrshr_n_s16(residual, 4);
+      const int16x4_t b = vqadd_s16(a, vget_low_s16(frame_data));
+      const uint8x8_t d = vqmovun_s16(vcombine_s16(b, b));
+      StoreLo4(dst, d);
+      dst += stride;
+    }
+  } else if (tx_width == 8) {
+    for (int i = 0; i < tx_height; ++i) {
+      const int row = flip_rows ? (tx_height - i - 1) * 8 : i * 8;
+      const int16x8_t residual = vld1q_s16(&source[row]);
+      const int16x8_t frame_data =
+          vreinterpretq_s16_u16(vmovl_u8(vld1_u8(dst)));
+      const int16x8_t a = vrshrq_n_s16(residual, 4);
+      const int16x8_t b = vqaddq_s16(a, frame_data);
+      const uint8x8_t d = vqmovun_s16(b);
+      vst1_u8(dst, d);
+      dst += stride;
+    }
+  } else {
+    for (int i = 0; i < tx_height; ++i) {
+      const int y = start_y + i;
+      const int row = flip_rows ? (tx_height - i - 1) * tx_width : i * tx_width;
+      int j = 0;
+      do {
+        const int x = start_x + j;
+        const int16x8_t residual = vld1q_s16(&source[row + j]);
+        const int16x8_t residual_hi = vld1q_s16(&source[row + j + 8]);
+        const uint8x16_t frame_data = vld1q_u8(frame[y] + x);
+        const int16x8_t a = vrshrq_n_s16(residual, 4);
+        const int16x8_t a_hi = vrshrq_n_s16(residual_hi, 4);
+        const int16x8_t d =
+            vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(frame_data)));
+        const int16x8_t d_hi =
+            vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(frame_data)));
+        const int16x8_t e = vqaddq_s16(a, d);
+        const int16x8_t e_hi = vqaddq_s16(a_hi, d_hi);
+        vst1q_u8(frame[y] + x, vcombine_u8(vqmovun_s16(e), vqmovun_s16(e_hi)));
+        j += 16;
+      } while (j < tx_width);
+    }
+  }
+}
+
+void Dct4TransformLoop_NEON(TransformType tx_type, TransformSize tx_size,
+                            void* src_buffer, int start_x, int start_y,
+                            void* dst_frame, bool is_row,
+                            int non_zero_coeff_count) {
+  auto& frame = *reinterpret_cast<Array2DView<uint8_t>*>(dst_frame);
+  auto* src = static_cast<int16_t*>(src_buffer);
+  const int tx_width = kTransformWidth[tx_size];
+  const int tx_height = kTransformHeight[tx_size];
+
+  if (is_row) {
+    const int num_rows = (non_zero_coeff_count == 1) ? 1 : tx_height;
+    const bool should_round = (tx_height == 8);
+    if (should_round) {
+      ApplyRounding<4>(src, num_rows);
+    }
+
+    if (num_rows <= 4) {
+      // Process 4 1d dct4 rows in parallel.
+      Dct4_NEON<ButterflyRotation_4, false>(&src[0], &src[0], /*step=*/4,
+                                            /*transpose=*/true);
+    } else {
+      // Process 8 1d dct4 rows in parallel per iteration.
+      int i = 0;
+      do {
+        Dct4_NEON<ButterflyRotation_8, true>(&src[i * 4], &src[i * 4],
+                                             /*step=*/4, /*transpose=*/true);
+        i += 8;
+      } while (i < num_rows);
+    }
+    if (tx_height == 16) {
+      RowShift<4>(src, num_rows, 1);
+    }
+    return;
+  }
+
+  assert(!is_row);
+  if (kTransformFlipColumnsMask.Contains(tx_type)) {
+    FlipColumns<4>(src, tx_width);
+  }
+
+  if (tx_width == 4) {
+    // Process 4 1d dct4 columns in parallel.
+    Dct4_NEON<ButterflyRotation_4, false>(&src[0], &src[0], tx_width,
+                                          /*transpose=*/false);
+  } else {
+    // Process 8 1d dct4 columns in parallel per iteration.
+    int i = 0;
+    do {
+      Dct4_NEON<ButterflyRotation_8, true>(&src[i], &src[i], tx_width,
+                                           /*transpose=*/false);
+      i += 8;
+    } while (i < tx_width);
+  }
+  StoreToFrameWithRound(frame, start_x, start_y, tx_width, 4, src, tx_type);
+}
+
+void Dct8TransformLoop_NEON(TransformType tx_type, TransformSize tx_size,
+                            void* src_buffer, int start_x, int start_y,
+                            void* dst_frame, bool is_row,
+                            int non_zero_coeff_count) {
+  auto& frame = *reinterpret_cast<Array2DView<uint8_t>*>(dst_frame);
+  auto* src = static_cast<int16_t*>(src_buffer);
+  const int tx_width = kTransformWidth[tx_size];
+  const int tx_height = kTransformHeight[tx_size];
+
+  if (is_row) {
+    const int num_rows = (non_zero_coeff_count == 1) ? 1 : tx_height;
+    if (kShouldRound[tx_size]) {
+      ApplyRounding<8>(src, num_rows);
+    }
+
+    if (num_rows <= 4) {
+      // Process 4 1d dct8 rows in parallel.
+      Dct8_NEON<ButterflyRotation_4, true>(&src[0], &src[0], /*step=*/8,
+                                           /*transpose=*/true);
+    } else {
+      // Process 8 1d dct8 rows in parallel per iteration.
+      int i = 0;
+      do {
+        Dct8_NEON<ButterflyRotation_8, false>(&src[i * 8], &src[i * 8],
+                                              /*step=*/8, /*transpose=*/true);
+        i += 8;
+      } while (i < num_rows);
+    }
+    const uint8_t row_shift = kTransformRowShift[tx_size];
+    if (row_shift > 0) {
+      RowShift<8>(src, num_rows, row_shift);
+    }
+    return;
+  }
+
+  assert(!is_row);
+  if (kTransformFlipColumnsMask.Contains(tx_type)) {
+    FlipColumns<8>(src, tx_width);
+  }
+
+  if (tx_width == 4) {
+    // Process 4 1d dct8 columns in parallel.
+    Dct8_NEON<ButterflyRotation_4, true>(&src[0], &src[0], 4,
+                                         /*transpose=*/false);
+  } else {
+    // Process 8 1d dct8 columns in parallel per iteration.
+    int i = 0;
+    do {
+      Dct8_NEON<ButterflyRotation_8, false>(&src[i], &src[i], tx_width,
+                                            /*transpose=*/false);
+      i += 8;
+    } while (i < tx_width);
+  }
+  StoreToFrameWithRound(frame, start_x, start_y, tx_width, 8, src, tx_type);
+}
+
+void Dct16TransformLoop_NEON(TransformType tx_type, TransformSize tx_size,
+                             void* src_buffer, int start_x, int start_y,
+                             void* dst_frame, bool is_row,
+                             int non_zero_coeff_count) {
+  auto& frame = *reinterpret_cast<Array2DView<uint8_t>*>(dst_frame);
+  auto* src = static_cast<int16_t*>(src_buffer);
+  const int tx_width = kTransformWidth[tx_size];
+  const int tx_height = kTransformHeight[tx_size];
+
+  if (is_row) {
+    const int num_rows =
+        (non_zero_coeff_count == 1) ? 1 : std::min(tx_height, 32);
+    if (kShouldRound[tx_size]) {
+      ApplyRounding<16>(src, num_rows);
+    }
+
+    if (num_rows <= 4) {
+      // Process 4 1d dct16 rows in parallel.
+      Dct16_NEON<ButterflyRotation_4, true>(&src[0], &src[0], 16,
+                                            /*transpose=*/true);
+    } else {
+      int i = 0;
+      do {
+        // Process 8 1d dct16 rows in parallel per iteration.
+        Dct16_NEON<ButterflyRotation_8, false>(&src[i * 16], &src[i * 16], 16,
+                                               /*transpose=*/true);
+        i += 8;
+      } while (i < num_rows);
+    }
+    const uint8_t row_shift = kTransformRowShift[tx_size];
+    // row_shift is always non zero here.
+    RowShift<16>(src, num_rows, row_shift);
+
+    return;
+  }
+
+  assert(!is_row);
+  if (kTransformFlipColumnsMask.Contains(tx_type)) {
+    FlipColumns<16>(src, tx_width);
+  }
+
+  if (tx_width == 4) {
+    // Process 4 1d dct16 columns in parallel.
+    Dct16_NEON<ButterflyRotation_4, true>(&src[0], &src[0], 4,
+                                          /*transpose=*/false);
+  } else {
+    int i = 0;
+    do {
+      // Process 8 1d dct16 columns in parallel per iteration.
+      Dct16_NEON<ButterflyRotation_8, false>(&src[i], &src[i], tx_width,
+                                             /*transpose=*/false);
+      i += 8;
+    } while (i < tx_width);
+  }
+  StoreToFrameWithRound(frame, start_x, start_y, tx_width, 16, src, tx_type);
+}
+
+void Dct32TransformLoop_NEON(TransformType tx_type, TransformSize tx_size,
+                             void* src_buffer, int start_x, int start_y,
+                             void* dst_frame, bool is_row,
+                             int non_zero_coeff_count) {
+  auto& frame = *reinterpret_cast<Array2DView<uint8_t>*>(dst_frame);
+  auto* src = static_cast<int16_t*>(src_buffer);
+  const int tx_width = kTransformWidth[tx_size];
+  const int tx_height = kTransformHeight[tx_size];
+
+  if (is_row) {
+    const int num_rows =
+        (non_zero_coeff_count == 1) ? 1 : std::min(tx_height, 32);
+    if (kShouldRound[tx_size]) {
+      ApplyRounding<32>(src, num_rows);
+    }
+    // Process 8 1d dct32 rows in parallel per iteration.
+    int i = 0;
+    do {
+      Dct32_NEON(&src[i * 32], &src[i * 32], 32, /*transpose=*/true);
+      i += 8;
+    } while (i < num_rows);
+    const uint8_t row_shift = kTransformRowShift[tx_size];
+    // row_shift is always non zero here.
+    RowShift<32>(src, num_rows, row_shift);
+
+    return;
+  }
+
+  assert(!is_row);
+  // Process 8 1d dct32 columns in parallel per iteration.
+  int i = 0;
+  do {
+    Dct32_NEON(&src[i], &src[i], tx_width, /*transpose=*/false);
+    i += 8;
+  } while (i < tx_width);
+  StoreToFrameWithRound(frame, start_x, start_y, tx_width, 32, src, tx_type);
+}
+
+void Dct64TransformLoop_NEON(TransformType tx_type, TransformSize tx_size,
+                             void* src_buffer, int start_x, int start_y,
+                             void* dst_frame, bool is_row,
+                             int non_zero_coeff_count) {
+  auto& frame = *reinterpret_cast<Array2DView<uint8_t>*>(dst_frame);
+  auto* src = static_cast<int16_t*>(src_buffer);
+  const int tx_width = kTransformWidth[tx_size];
+  const int tx_height = kTransformHeight[tx_size];
+
+  if (is_row) {
+    const int num_rows =
+        (non_zero_coeff_count == 1) ? 1 : std::min(tx_height, 32);
+    if (kShouldRound[tx_size]) {
+      ApplyRounding<64>(src, num_rows);
+    }
+    // Process 8 1d dct64 rows in parallel per iteration.
+    int i = 0;
+    do {
+      Dct64_NEON(&src[i * 64], &src[i * 64], 64, /*transpose=*/true);
+      i += 8;
+    } while (i < num_rows);
+    const uint8_t row_shift = kTransformRowShift[tx_size];
+    // row_shift is always non zero here.
+    RowShift<64>(src, num_rows, row_shift);
+
+    return;
+  }
+
+  assert(!is_row);
+  // Process 8 1d dct64 columns in parallel per iteration.
+  int i = 0;
+  do {
+    Dct64_NEON(&src[i], &src[i], tx_width, /*transpose=*/false);
+    i += 8;
+  } while (i < tx_width);
+  StoreToFrameWithRound(frame, start_x, start_y, tx_width, 64, src, tx_type);
+}
+
+void Adst4TransformLoop_NEON(TransformType tx_type, TransformSize tx_size,
+                             void* src_buffer, int start_x, int start_y,
+                             void* dst_frame, bool is_row,
+                             int non_zero_coeff_count) {
+  auto& frame = *reinterpret_cast<Array2DView<uint8_t>*>(dst_frame);
+  auto* src = static_cast<int16_t*>(src_buffer);
+  const int tx_width = kTransformWidth[tx_size];
+  const int tx_height = kTransformHeight[tx_size];
+
+  if (is_row) {
+    const int num_rows = (non_zero_coeff_count == 1) ? 1 : tx_height;
+    const bool should_round = (tx_height == 8);
+    if (should_round) {
+      ApplyRounding<4>(src, num_rows);
+    }
+
+    // Process 4 1d adst4 rows in parallel per iteration.
+    int i = 0;
+    do {
+      Adst4_NEON<false>(&src[i * 4], &src[i * 4], /*step=*/4,
+                        /*transpose=*/true);
+      i += 4;
+    } while (i < num_rows);
+
+    if (tx_height == 16) {
+      RowShift<4>(src, num_rows, 1);
+    }
+    return;
+  }
+
+  assert(!is_row);
+  if (kTransformFlipColumnsMask.Contains(tx_type)) {
+    FlipColumns<4>(src, tx_width);
+  }
+
+  // Process 4 1d adst4 columns in parallel per iteration.
+  int i = 0;
+  do {
+    Adst4_NEON<false>(&src[i], &src[i], tx_width, /*transpose=*/false);
+    i += 4;
+  } while (i < tx_width);
+
+  StoreToFrameWithRound</*enable_flip_rows=*/true>(frame, start_x, start_y,
+                                                   tx_width, 4, src, tx_type);
+}
+
+void Adst8TransformLoop_NEON(TransformType tx_type, TransformSize tx_size,
+                             void* src_buffer, int start_x, int start_y,
+                             void* dst_frame, bool is_row,
+                             int non_zero_coeff_count) {
+  auto& frame = *reinterpret_cast<Array2DView<uint8_t>*>(dst_frame);
+  auto* src = static_cast<int16_t*>(src_buffer);
+  const int tx_width = kTransformWidth[tx_size];
+  const int tx_height = kTransformHeight[tx_size];
+
+  if (is_row) {
+    const int num_rows = (non_zero_coeff_count == 1) ? 1 : tx_height;
+    if (kShouldRound[tx_size]) {
+      ApplyRounding<8>(src, num_rows);
+    }
+
+    if (num_rows <= 4) {
+      // Process 4 1d adst8 rows in parallel.
+      Adst8_NEON<ButterflyRotation_4, true>(&src[0], &src[0], /*step=*/8,
+                                            /*transpose=*/true);
+    } else {
+      // Process 8 1d adst8 rows in parallel per iteration.
+      int i = 0;
+      do {
+        Adst8_NEON<ButterflyRotation_8, false>(&src[i * 8], &src[i * 8],
+                                               /*step=*/8,
+                                               /*transpose=*/true);
+        i += 8;
+      } while (i < num_rows);
+    }
+    const uint8_t row_shift = kTransformRowShift[tx_size];
+    if (row_shift > 0) {
+      RowShift<8>(src, num_rows, row_shift);
+    }
+    return;
+  }
+
+  assert(!is_row);
+  if (kTransformFlipColumnsMask.Contains(tx_type)) {
+    FlipColumns<8>(src, tx_width);
+  }
+
+  if (tx_width == 4) {
+    // Process 4 1d adst8 columns in parallel.
+    Adst8_NEON<ButterflyRotation_4, true>(&src[0], &src[0], 4,
+                                          /*transpose=*/false);
+  } else {
+    // Process 8 1d adst8 columns in parallel per iteration.
+    int i = 0;
+    do {
+      Adst8_NEON<ButterflyRotation_8, false>(&src[i], &src[i], tx_width,
+                                             /*transpose=*/false);
+      i += 8;
+    } while (i < tx_width);
+  }
+  StoreToFrameWithRound</*enable_flip_rows=*/true>(frame, start_x, start_y,
+                                                   tx_width, 8, src, tx_type);
+}
+
+void Adst16TransformLoop_NEON(TransformType tx_type, TransformSize tx_size,
+                              void* src_buffer, int start_x, int start_y,
+                              void* dst_frame, bool is_row,
+                              int non_zero_coeff_count) {
+  auto& frame = *reinterpret_cast<Array2DView<uint8_t>*>(dst_frame);
+  auto* src = static_cast<int16_t*>(src_buffer);
+  const int tx_width = kTransformWidth[tx_size];
+  const int tx_height = kTransformHeight[tx_size];
+
+  if (is_row) {
+    const int num_rows =
+        (non_zero_coeff_count == 1) ? 1 : std::min(tx_height, 32);
+    if (kShouldRound[tx_size]) {
+      ApplyRounding<16>(src, num_rows);
+    }
+
+    if (num_rows <= 4) {
+      // Process 4 1d adst16 rows in parallel.
+      Adst16_NEON<ButterflyRotation_4, true>(&src[0], &src[0], 16,
+                                             /*transpose=*/true);
+    } else {
+      int i = 0;
+      do {
+        // Process 8 1d adst16 rows in parallel per iteration.
+        Adst16_NEON<ButterflyRotation_8, false>(&src[i * 16], &src[i * 16], 16,
+                                                /*transpose=*/true);
+        i += 8;
+      } while (i < num_rows);
+    }
+    const uint8_t row_shift = kTransformRowShift[tx_size];
+    // row_shift is always non zero here.
+    RowShift<16>(src, num_rows, row_shift);
+
+    return;
+  }
+
+  assert(!is_row);
+  if (kTransformFlipColumnsMask.Contains(tx_type)) {
+    FlipColumns<16>(src, tx_width);
+  }
+
+  if (tx_width == 4) {
+    // Process 4 1d adst16 columns in parallel.
+    Adst16_NEON<ButterflyRotation_4, true>(&src[0], &src[0], 4,
+                                           /*transpose=*/false);
+  } else {
+    int i = 0;
+    do {
+      // Process 8 1d adst16 columns in parallel per iteration.
+      Adst16_NEON<ButterflyRotation_8, false>(&src[i], &src[i], tx_width,
+                                              /*transpose=*/false);
+      i += 8;
+    } while (i < tx_width);
+  }
+  StoreToFrameWithRound</*enable_flip_rows=*/true>(frame, start_x, start_y,
+                                                   tx_width, 16, src, tx_type);
+}
+
+void Identity4TransformLoop_NEON(TransformType tx_type, TransformSize tx_size,
+                                 void* src_buffer, int start_x, int start_y,
+                                 void* dst_frame, bool is_row,
+                                 int non_zero_coeff_count) {
+  auto& frame = *reinterpret_cast<Array2DView<uint8_t>*>(dst_frame);
+  auto* src = static_cast<int16_t*>(src_buffer);
+  const int tx_width = kTransformWidth[tx_size];
+  const int tx_height = kTransformHeight[tx_size];
+
+  if (is_row) {
+    // Special case: Process row calculations during column transform call.
+    // Improves performance.
+    if (tx_type == kTransformTypeIdentityIdentity &&
+        tx_size == kTransformSize4x4) {
+      return;
+    }
+
+    const int num_rows = (non_zero_coeff_count == 1) ? 1 : tx_height;
+    const bool should_round = (tx_height == 8);
+    if (should_round) {
+      ApplyRounding<4>(src, num_rows);
+    }
+    if (tx_height < 16) {
+      int i = 0;
+      do {
+        Identity4_NEON<false>(&src[i * 4], &src[i * 4], /*step=*/4);
+        i += 4;
+      } while (i < num_rows);
+    } else {
+      int i = 0;
+      do {
+        Identity4_NEON<true>(&src[i * 4], &src[i * 4], /*step=*/4);
+        i += 4;
+      } while (i < num_rows);
+    }
+    return;
+  }
+  assert(!is_row);
+  // Special case: Process row calculations during column transform call.
+  if (tx_type == kTransformTypeIdentityIdentity &&
+      (tx_size == kTransformSize4x4 || tx_size == kTransformSize8x4)) {
+    Identity4RowColumnStoreToFrame(frame, start_x, start_y, tx_width,
+                                   /*tx_height=*/4, src);
+    return;
+  }
+
+  if (kTransformFlipColumnsMask.Contains(tx_type)) {
+    FlipColumns<4>(src, tx_width);
+  }
+
+  Identity4ColumnStoreToFrame(frame, start_x, start_y, tx_width,
+                              /*tx_height=*/4, src);
+}
+
+void Identity8TransformLoop_NEON(TransformType tx_type, TransformSize tx_size,
+                                 void* src_buffer, int start_x, int start_y,
+                                 void* dst_frame, bool is_row,
+                                 int non_zero_coeff_count) {
+  auto& frame = *reinterpret_cast<Array2DView<uint8_t>*>(dst_frame);
+  auto* src = static_cast<int16_t*>(src_buffer);
+  const int tx_width = kTransformWidth[tx_size];
+  const int tx_height = kTransformHeight[tx_size];
+
+  if (is_row) {
+    // Special case: Process row calculations during column transform call.
+    // Improves performance.
+    if (tx_type == kTransformTypeIdentityIdentity &&
+        tx_size == kTransformSize8x4) {
+      return;
+    }
+    const int num_rows = (non_zero_coeff_count == 1) ? 1 : tx_height;
+    if (kShouldRound[tx_size]) {
+      ApplyRounding<8>(src, num_rows);
+    }
+
+    // When combining the identity8 multiplier with the row shift, the
+    // calculations for tx_height == 8 and tx_height == 16 can be simplified
+    // from ((A * 2) + 1) >> 1) to A.
+    if ((tx_height & 0x18) != 0) {
+      return;
+    }
+    if (tx_height == 32) {
+      for (int i = 0; i < num_rows; i += 4) {
+        Identity8Row32_NEON(&src[i * 8], &src[i * 8], /*step=*/8);
+      }
+      return;
+    }
+
+    // Process kTransformSize8x4
+    assert(tx_size == kTransformSize8x4);
+    for (int i = 0; i < num_rows; i += 4) {
+      Identity8Row4_NEON(&src[i * 8], &src[i * 8], /*step=*/8);
+    }
+    return;
+  }
+
+  assert(!is_row);
+  if (kTransformFlipColumnsMask.Contains(tx_type)) {
+    FlipColumns<8>(src, tx_width);
+  }
+
+  Identity8ColumnStoreToFrame_NEON(frame, start_x, start_y, tx_width,
+                                   /*tx_height=*/8, src);
+}
+
+void Identity16TransformLoop_NEON(TransformType tx_type, TransformSize tx_size,
+                                  void* src_buffer, int start_x, int start_y,
+                                  void* dst_frame, bool is_row,
+                                  int non_zero_coeff_count) {
+  auto& frame = *reinterpret_cast<Array2DView<uint8_t>*>(dst_frame);
+  auto* src = static_cast<int16_t*>(src_buffer);
+  const int tx_width = kTransformWidth[tx_size];
+  const int tx_height = kTransformHeight[tx_size];
+
+  if (is_row) {
+    const int num_rows =
+        (non_zero_coeff_count == 1) ? 1 : std::min(tx_height, 32);
+    if (kShouldRound[tx_size]) {
+      ApplyRounding<16>(src, num_rows);
+    }
+    for (int i = 0; i < num_rows; i += 4) {
+      Identity16Row_NEON(&src[i * 16], &src[i * 16], /*step=*/16,
+                         kTransformRowShift[tx_size]);
+    }
+    return;
+  }
+
+  assert(!is_row);
+  if (kTransformFlipColumnsMask.Contains(tx_type)) {
+    FlipColumns<16>(src, tx_width);
+  }
+  Identity16ColumnStoreToFrame_NEON(frame, start_x, start_y, tx_width,
+                                    /*tx_height=*/16, src);
+}
+
+void Identity32TransformLoop_NEON(TransformType /*tx_type*/,
+                                  TransformSize tx_size, void* src_buffer,
+                                  int start_x, int start_y, void* dst_frame,
+                                  bool is_row, int non_zero_coeff_count) {
+  auto& frame = *reinterpret_cast<Array2DView<uint8_t>*>(dst_frame);
+  auto* src = static_cast<int16_t*>(src_buffer);
+  const int tx_width = kTransformWidth[tx_size];
+  const int tx_height = kTransformHeight[tx_size];
+
+  if (is_row) {
+    const int num_rows = (non_zero_coeff_count == 1) ? 1 : tx_height;
+
+    // When combining the identity32 multiplier with the row shift, the
+    // calculations for tx_height == 8 and tx_height == 32 can be simplified
+    // from ((A * 4) + 2) >> 2) to A.
+    if ((tx_height & 0x28) != 0) {
+      return;
+    }
+
+    // Process kTransformSize32x16
+    assert(tx_size == kTransformSize32x16);
+    ApplyRounding<32>(src, num_rows);
+    for (int i = 0; i < num_rows; i += 4) {
+      Identity32Row16_NEON(&src[i * 32], &src[i * 32], /*step=*/32);
+    }
+    return;
+  }
+
+  assert(!is_row);
+  Identity32ColumnStoreToFrame(frame, start_x, start_y, tx_width,
+                               /*tx_height=*/32, src);
+}
+
+void Wht4TransformLoop_NEON(TransformType tx_type, TransformSize tx_size,
+                            void* src_buffer, int start_x, int start_y,
+                            void* dst_frame, bool is_row,
+                            int non_zero_coeff_count) {
+  assert(tx_type == kTransformTypeDctDct);
+  assert(tx_size == kTransformSize4x4);
+  static_cast<void>(tx_type);
+  static_cast<void>(tx_size);
+  if (is_row) {
+    // Do both row and column transforms in the column-transform pass.
+    return;
+  }
+
+  assert(!is_row);
+  // Process 4 1d wht4 rows and columns in parallel.
+  auto* src = static_cast<int16_t*>(src_buffer);
+  auto& frame = *static_cast<Array2DView<uint8_t>*>(dst_frame);
+  uint8_t* dst = frame[start_y] + start_x;
+  const int dst_stride = frame.columns();
+  Wht4_NEON(dst, dst_stride, src, non_zero_coeff_count);
+}
+
+//------------------------------------------------------------------------------
+
+void Init8bpp() {
+  Dsp* const dsp = dsp_internal::GetWritableDspTable(8);
+  assert(dsp != nullptr);
+  // Maximum transform size for Dct is 64.
+  dsp->inverse_transforms[k1DTransformSize4][k1DTransformDct] =
+      Dct4TransformLoop_NEON;
+  dsp->inverse_transforms[k1DTransformSize8][k1DTransformDct] =
+      Dct8TransformLoop_NEON;
+  dsp->inverse_transforms[k1DTransformSize16][k1DTransformDct] =
+      Dct16TransformLoop_NEON;
+  dsp->inverse_transforms[k1DTransformSize32][k1DTransformDct] =
+      Dct32TransformLoop_NEON;
+  dsp->inverse_transforms[k1DTransformSize64][k1DTransformDct] =
+      Dct64TransformLoop_NEON;
+
+  // Maximum transform size for Adst is 16.
+  dsp->inverse_transforms[k1DTransformSize4][k1DTransformAdst] =
+      Adst4TransformLoop_NEON;
+  dsp->inverse_transforms[k1DTransformSize8][k1DTransformAdst] =
+      Adst8TransformLoop_NEON;
+  dsp->inverse_transforms[k1DTransformSize16][k1DTransformAdst] =
+      Adst16TransformLoop_NEON;
+
+  // Maximum transform size for Identity transform is 32.
+  dsp->inverse_transforms[k1DTransformSize4][k1DTransformIdentity] =
+      Identity4TransformLoop_NEON;
+  dsp->inverse_transforms[k1DTransformSize8][k1DTransformIdentity] =
+      Identity8TransformLoop_NEON;
+  dsp->inverse_transforms[k1DTransformSize16][k1DTransformIdentity] =
+      Identity16TransformLoop_NEON;
+  dsp->inverse_transforms[k1DTransformSize32][k1DTransformIdentity] =
+      Identity32TransformLoop_NEON;
+
+  // Maximum transform size for Wht is 4.
+  dsp->inverse_transforms[k1DTransformSize4][k1DTransformWht] =
+      Wht4TransformLoop_NEON;
+}
+
+}  // namespace
+}  // namespace low_bitdepth
+
+void InverseTransformInit_NEON() { low_bitdepth::Init8bpp(); }
+
+}  // namespace dsp
+}  // namespace libgav1
+#else   // !LIBGAV1_ENABLE_NEON
+namespace libgav1 {
+namespace dsp {
+
+void InverseTransformInit_NEON() {}
+
+}  // namespace dsp
+}  // namespace libgav1
+#endif  // LIBGAV1_ENABLE_NEON
diff --git a/libgav1/src/dsp/arm/inverse_transform_neon.h b/libgav1/src/dsp/arm/inverse_transform_neon.h
new file mode 100644
index 0000000..eefaada
--- /dev/null
+++ b/libgav1/src/dsp/arm/inverse_transform_neon.h
@@ -0,0 +1,36 @@
+#ifndef LIBGAV1_SRC_DSP_ARM_INVERSE_TRANSFORM_NEON_H_
+#define LIBGAV1_SRC_DSP_ARM_INVERSE_TRANSFORM_NEON_H_
+
+#include "src/dsp/cpu.h"
+#include "src/dsp/dsp.h"
+
+namespace libgav1 {
+namespace dsp {
+
+// Initializes Dsp::inverse_transforms, see the defines below for specifics.
+// This function is not thread-safe.
+void InverseTransformInit_NEON();
+
+}  // namespace dsp
+}  // namespace libgav1
+
+#if LIBGAV1_ENABLE_NEON
+#define LIBGAV1_Dsp8bpp_1DTransformSize4_1DTransformDct LIBGAV1_DSP_NEON
+#define LIBGAV1_Dsp8bpp_1DTransformSize8_1DTransformDct LIBGAV1_DSP_NEON
+#define LIBGAV1_Dsp8bpp_1DTransformSize16_1DTransformDct LIBGAV1_DSP_NEON
+#define LIBGAV1_Dsp8bpp_1DTransformSize32_1DTransformDct LIBGAV1_DSP_NEON
+#define LIBGAV1_Dsp8bpp_1DTransformSize64_1DTransformDct LIBGAV1_DSP_NEON
+
+#define LIBGAV1_Dsp8bpp_1DTransformSize4_1DTransformAdst LIBGAV1_DSP_NEON
+#define LIBGAV1_Dsp8bpp_1DTransformSize8_1DTransformAdst LIBGAV1_DSP_NEON
+#define LIBGAV1_Dsp8bpp_1DTransformSize16_1DTransformAdst LIBGAV1_DSP_NEON
+
+#define LIBGAV1_Dsp8bpp_1DTransformSize4_1DTransformIdentity LIBGAV1_DSP_NEON
+#define LIBGAV1_Dsp8bpp_1DTransformSize8_1DTransformIdentity LIBGAV1_DSP_NEON
+#define LIBGAV1_Dsp8bpp_1DTransformSize16_1DTransformIdentity LIBGAV1_DSP_NEON
+#define LIBGAV1_Dsp8bpp_1DTransformSize32_1DTransformIdentity LIBGAV1_DSP_NEON
+
+#define LIBGAV1_Dsp8bpp_1DTransformSize4_1DTransformWht LIBGAV1_DSP_NEON
+#endif  // LIBGAV1_ENABLE_NEON
+
+#endif  // LIBGAV1_SRC_DSP_ARM_INVERSE_TRANSFORM_NEON_H_
diff --git a/libgav1/src/dsp/arm/loop_filter_neon.cc b/libgav1/src/dsp/arm/loop_filter_neon.cc
index 8994aaf..4a1b6a7 100644
--- a/libgav1/src/dsp/arm/loop_filter_neon.cc
+++ b/libgav1/src/dsp/arm/loop_filter_neon.cc
@@ -1,4 +1,5 @@
-#include "src/dsp/arm/loop_filter_neon.h"
+#include "src/dsp/dsp.h"
+#include "src/dsp/loop_filter.h"
 
 #if LIBGAV1_ENABLE_NEON
 
@@ -15,70 +16,6 @@
 namespace low_bitdepth {
 namespace {
 
-// vzipN is exclusive to A64.
-inline uint8x8_t InterleaveLow32(const uint8x8_t a, const uint8x8_t b) {
-#if defined(__aarch64__)
-  return vreinterpret_u8_u32(
-      vzip1_u32(vreinterpret_u32_u8(a), vreinterpret_u32_u8(b)));
-#else
-  // Discard |.val[1]|
-  return vreinterpret_u8_u32(
-      vzip_u32(vreinterpret_u32_u8(a), vreinterpret_u32_u8(b)).val[0]);
-#endif
-}
-
-inline int8x8_t InterleaveLow32(const int8x8_t a, const int8x8_t b) {
-#if defined(__aarch64__)
-  return vreinterpret_s8_u32(
-      vzip1_u32(vreinterpret_u32_s8(a), vreinterpret_u32_s8(b)));
-#else
-  // Discard |.val[1]|
-  return vreinterpret_s8_u32(
-      vzip_u32(vreinterpret_u32_s8(a), vreinterpret_u32_s8(b)).val[0]);
-#endif
-}
-
-inline uint8x8_t InterleaveHigh32(const uint8x8_t a, const uint8x8_t b) {
-#if defined(__aarch64__)
-  return vreinterpret_u8_u32(
-      vzip2_u32(vreinterpret_u32_u8(a), vreinterpret_u32_u8(b)));
-#else
-  return vreinterpret_u8_u32(
-      vzip_u32(vreinterpret_u32_u8(a), vreinterpret_u32_u8(b)).val[1]);
-#endif
-}
-
-inline int8x8_t InterleaveHigh32(const int8x8_t a, const int8x8_t b) {
-#if defined(__aarch64__)
-  return vreinterpret_s8_u32(
-      vzip2_u32(vreinterpret_u32_s8(a), vreinterpret_u32_s8(b)));
-#else
-  return vreinterpret_s8_u32(
-      vzip_u32(vreinterpret_u32_s8(a), vreinterpret_u32_s8(b)).val[1]);
-#endif
-}
-
-// Transpose 32 bit elements such that:
-// a: 00 01
-// b: 02 03
-// returns
-// val[0]: 00 02
-// val[1]: 01 03
-inline uint8x8x2_t Interleave32(const uint8x8_t a, const uint8x8_t b) {
-  const uint32x2_t a_32 = vreinterpret_u32_u8(a);
-  const uint32x2_t b_32 = vreinterpret_u32_u8(b);
-  const uint32x2x2_t c = vtrn_u32(a_32, b_32);
-  const uint8x8x2_t d = {vreinterpret_u8_u32(c.val[0]),
-                         vreinterpret_u8_u32(c.val[1])};
-  return d;
-}
-
-// Swap high and low 32 bit elements.
-inline uint8x8_t Transpose32(const uint8x8_t a) {
-  const uint32x2_t b = vrev64_u32(vreinterpret_u32_u8(a));
-  return vreinterpret_u8_u32(b);
-}
-
 // (abs(p1 - p0) > thresh) || (abs(q1 - q0) > thresh)
 inline uint8x8_t Hev(const uint8x8_t abd_p0p1_q0q1, const uint8_t thresh) {
   const uint8x8_t a = vcgt_u8(abd_p0p1_q0q1, vdup_n_u8(thresh));
diff --git a/libgav1/src/dsp/arm/loop_filter_neon.h b/libgav1/src/dsp/arm/loop_filter_neon.h
index 50ef294..ca80746 100644
--- a/libgav1/src/dsp/arm/loop_filter_neon.h
+++ b/libgav1/src/dsp/arm/loop_filter_neon.h
@@ -3,13 +3,12 @@
 
 #include "src/dsp/cpu.h"
 #include "src/dsp/dsp.h"
-#include "src/dsp/loop_filter.h"
 
 namespace libgav1 {
 namespace dsp {
 
-// Initializes Dsp::loop_filters with neon implementations. This function
-// is not thread-safe.
+// Initializes Dsp::loop_filters, see the defines below for specifics. This
+// function is not thread-safe.
 void LoopFilterInit_NEON();
 
 }  // namespace dsp
diff --git a/libgav1/src/dsp/arm/loop_restoration_neon.cc b/libgav1/src/dsp/arm/loop_restoration_neon.cc
index 367826b..fd0eda5 100644
--- a/libgav1/src/dsp/arm/loop_restoration_neon.cc
+++ b/libgav1/src/dsp/arm/loop_restoration_neon.cc
@@ -1,4 +1,5 @@
-#include "src/dsp/arm/loop_restoration_neon.h"
+#include "src/dsp/dsp.h"
+#include "src/dsp/loop_restoration.h"
 
 #if LIBGAV1_ENABLE_NEON
 
@@ -46,14 +47,13 @@
   sum = vaddq_s16(
       sum, vmulq_n_s16(vreinterpretq_s16_u16(vmovl_u8(a[6])), filter[6]));
 
-  // |inter_round_bits[0]| == 3 for 8 bit inputs.
-  sum = vrshrq_n_s16(sum, 3);
+  sum = vrshrq_n_s16(sum, kInterRoundBitsHorizontal);
 
   // Delaying |horizontal_rounding| until after dowshifting allows the sum to
   // stay in 16 bits.
   // |horizontal_rounding| = 1 << (bitdepth + kWienerFilterBits - 1)
   //                         1 << (       8 +                 7 - 1)
-  // Plus |inter_round_bits[0]| and it works out to 1 << 11.
+  // Plus |kInterRoundBitsHorizontal| and it works out to 1 << 11.
   sum = vaddq_s16(sum, vdupq_n_s16(1 << 11));
 
   // Just like |horizontal_rounding|, adding |filter[3]| at this point allows
@@ -64,13 +64,124 @@
   sum = vaddq_s16(sum, vreinterpretq_s16_u16(vshll_n_u8(a[3], 4)));
 
   // Saturate to
-  // [0, (1 << (bitdepth + 1 + kWienerFilterBits - inter_round_bits[0])) - 1)]
+  // [0, (1 << (bitdepth + 1 + kWienerFilterBits - kInterRoundBitsHorizontal)) -
+  //  1)]
   //     (1 << (       8 + 1 +                 7 -                   3)) - 1)
   sum = vminq_s16(sum, vdupq_n_s16((1 << 13) - 1));
   sum = vmaxq_s16(sum, vdupq_n_s16(0));
   return sum;
 }
 
+template <int min_width>
+inline void VerticalSum(const int16_t* src_base, const int src_stride,
+                        uint8_t* dst_base, const int dst_stride,
+                        const int16x4_t filter[7], const int width,
+                        const int height) {
+  static_assert(min_width == 4 || min_width == 8, "");
+  // -(1 << (bitdepth + kInterRoundBitsVertical - 1))
+  // -(1 << (       8 +                      11 - 1))
+  constexpr int vertical_rounding = -(1 << 18);
+  if (min_width == 8) {
+    int x = 0;
+    do {
+      const int16_t* src = src_base + x;
+      uint8_t* dst = dst_base + x;
+      int16x8_t a[7];
+      a[0] = vld1q_s16(src);
+      src += src_stride;
+      a[1] = vld1q_s16(src);
+      src += src_stride;
+      a[2] = vld1q_s16(src);
+      src += src_stride;
+      a[3] = vld1q_s16(src);
+      src += src_stride;
+      a[4] = vld1q_s16(src);
+      src += src_stride;
+      a[5] = vld1q_s16(src);
+      src += src_stride;
+
+      int y = 0;
+      do {
+        a[6] = vld1q_s16(src);
+        src += src_stride;
+
+        int32x4_t sum_lo = vdupq_n_s32(vertical_rounding);
+        sum_lo = vmlal_s16(sum_lo, vget_low_s16(a[0]), filter[0]);
+        sum_lo = vmlal_s16(sum_lo, vget_low_s16(a[1]), filter[1]);
+        sum_lo = vmlal_s16(sum_lo, vget_low_s16(a[2]), filter[2]);
+        sum_lo = vmlal_s16(sum_lo, vget_low_s16(a[3]), filter[3]);
+        sum_lo = vmlal_s16(sum_lo, vget_low_s16(a[4]), filter[4]);
+        sum_lo = vmlal_s16(sum_lo, vget_low_s16(a[5]), filter[5]);
+        sum_lo = vmlal_s16(sum_lo, vget_low_s16(a[6]), filter[6]);
+        uint16x4_t sum_lo_16 = vqrshrun_n_s32(sum_lo, 11);
+
+        int32x4_t sum_hi = vdupq_n_s32(vertical_rounding);
+        sum_hi = vmlal_s16(sum_hi, vget_high_s16(a[0]), filter[0]);
+        sum_hi = vmlal_s16(sum_hi, vget_high_s16(a[1]), filter[1]);
+        sum_hi = vmlal_s16(sum_hi, vget_high_s16(a[2]), filter[2]);
+        sum_hi = vmlal_s16(sum_hi, vget_high_s16(a[3]), filter[3]);
+        sum_hi = vmlal_s16(sum_hi, vget_high_s16(a[4]), filter[4]);
+        sum_hi = vmlal_s16(sum_hi, vget_high_s16(a[5]), filter[5]);
+        sum_hi = vmlal_s16(sum_hi, vget_high_s16(a[6]), filter[6]);
+        uint16x4_t sum_hi_16 = vqrshrun_n_s32(sum_hi, 11);
+
+        vst1_u8(dst, vqmovn_u16(vcombine_u16(sum_lo_16, sum_hi_16)));
+        dst += dst_stride;
+
+        a[0] = a[1];
+        a[1] = a[2];
+        a[2] = a[3];
+        a[3] = a[4];
+        a[4] = a[5];
+        a[5] = a[6];
+      } while (++y < height);
+      x += 8;
+    } while (x < width);
+  } else if (min_width == 4) {
+    const int16_t* src = src_base;
+    uint8_t* dst = dst_base;
+    int16x4_t a[7];
+    a[0] = vld1_s16(src);
+    src += src_stride;
+    a[1] = vld1_s16(src);
+    src += src_stride;
+    a[2] = vld1_s16(src);
+    src += src_stride;
+    a[3] = vld1_s16(src);
+    src += src_stride;
+    a[4] = vld1_s16(src);
+    src += src_stride;
+    a[5] = vld1_s16(src);
+    src += src_stride;
+
+    int y = 0;
+    do {
+      a[6] = vld1_s16(src);
+      src += src_stride;
+
+      int32x4_t sum = vdupq_n_s32(vertical_rounding);
+      sum = vmlal_s16(sum, a[0], filter[0]);
+      sum = vmlal_s16(sum, a[1], filter[1]);
+      sum = vmlal_s16(sum, a[2], filter[2]);
+      sum = vmlal_s16(sum, a[3], filter[3]);
+      sum = vmlal_s16(sum, a[4], filter[4]);
+      sum = vmlal_s16(sum, a[5], filter[5]);
+      sum = vmlal_s16(sum, a[6], filter[6]);
+      uint16x4_t sum_16 = vqrshrun_n_s32(sum, 11);
+
+      StoreLo4(dst, vqmovn_u16(vcombine_u16(sum_16, sum_16)));
+      dst += dst_stride;
+
+      a[0] = a[1];
+      a[1] = a[2];
+      a[2] = a[3];
+      a[3] = a[4];
+      a[4] = a[5];
+      a[5] = a[6];
+    } while (++y < height);
+  }
+}
+
 void WienerFilter_NEON(const void* const source, void* const dest,
                        const RestorationUnitInfo& restoration_info,
                        const ptrdiff_t source_stride,
@@ -91,9 +202,14 @@
   // left value.
   const int center_tap = 3;
   src -= center_tap * source_stride + center_tap;
-  // This writes out 2 more rows than we need.
-  for (int y = 0; y < height + kSubPixelTaps - 2; y += 8) {
-    for (int x = 0; x < width; x += 8) {
+  // This writes out 2 more rows than we need. It always writes out at least
+  // width 8 for the intermediate buffer.
+  // TODO(johannkoenig): Investigate a 4x specific first pass. May be possible
+  // to do it in half the passes.
+  int y = 0;
+  do {
+    int x = 0;
+    do {
       const uint8_t* src_v = src + x;
       const uint8x16_t a0 = vld1q_u8(src_v);
       src_v += source_stride;
@@ -116,6 +232,7 @@
       // This could load and transpose one 8x8 block to prime the loop, then
       // load and transpose a second block in the loop. The second block could
       // be passed to subsequent iterations.
+      // TODO(johannkoenig): convert these to arrays.
       Transpose16x8(a0, a1, a2, a3, a4, a5, a6, a7, b, b + 1, b + 2, b + 3,
                     b + 4, b + 5, b + 6, b + 7, b + 8, b + 9, b + 10, b + 11,
                     b + 12, b + 13, b + 14, b + 15);
@@ -129,6 +246,7 @@
       int16x8_t sum_6 = HorizontalSum(b + 6, filter);
       int16x8_t sum_7 = HorizontalSum(b + 7, filter);
 
+      // TODO(johannkoenig): convert this to an array.
       Transpose8x8(&sum_0, &sum_1, &sum_2, &sum_3, &sum_4, &sum_5, &sum_6,
                    &sum_7);
 
@@ -148,10 +266,12 @@
       vst1q_s16(wiener_buffer_v, sum_6);
       wiener_buffer_v += buffer_stride;
       vst1q_s16(wiener_buffer_v, sum_7);
-    }
+      x += 8;
+    } while (x < width);
     src += 8 * source_stride;
     wiener_buffer += 8 * buffer_stride;
-  }
+    y += 8;
+  } while (y < height + kSubPixelTaps - 2);
 
   // Vertical filtering.
   wiener_buffer = reinterpret_cast<int16_t*>(buffer->wiener_buffer);
@@ -162,61 +282,13 @@
       vdup_n_s16(filter[0]),       vdup_n_s16(filter[1]), vdup_n_s16(filter[2]),
       vdup_n_s16(filter[3] + 128), vdup_n_s16(filter[4]), vdup_n_s16(filter[5]),
       vdup_n_s16(filter[6])};
-  // |inter_round_bits[1]| == 11 for 8 bit inputs.
-  // -(1 << (bitdepth + inter_round_bits[1] - 1))
-  // -(1 << (       8 +                  11 - 1))
-  const int vertical_rounding = -(1 << 18);
-  for (int x = 0; x < width; x += 8) {
-    int16_t* wiener_v = wiener_buffer + x;
-    uint8_t* dst_v = dst + x;
-    int16x8_t a[7];
-    a[0] = vld1q_s16(wiener_v);
-    wiener_v += buffer_stride;
-    a[1] = vld1q_s16(wiener_v);
-    wiener_v += buffer_stride;
-    a[2] = vld1q_s16(wiener_v);
-    wiener_v += buffer_stride;
-    a[3] = vld1q_s16(wiener_v);
-    wiener_v += buffer_stride;
-    a[4] = vld1q_s16(wiener_v);
-    wiener_v += buffer_stride;
-    a[5] = vld1q_s16(wiener_v);
-    wiener_v += buffer_stride;
 
-    for (int y = 0; y < height; ++y) {
-      a[6] = vld1q_s16(wiener_v);
-      wiener_v += buffer_stride;
-
-      int32x4_t sum_lo = vdupq_n_s32(vertical_rounding);
-      sum_lo = vmlal_s16(sum_lo, vget_low_s16(a[0]), filter_v[0]);
-      sum_lo = vmlal_s16(sum_lo, vget_low_s16(a[1]), filter_v[1]);
-      sum_lo = vmlal_s16(sum_lo, vget_low_s16(a[2]), filter_v[2]);
-      sum_lo = vmlal_s16(sum_lo, vget_low_s16(a[3]), filter_v[3]);
-      sum_lo = vmlal_s16(sum_lo, vget_low_s16(a[4]), filter_v[4]);
-      sum_lo = vmlal_s16(sum_lo, vget_low_s16(a[5]), filter_v[5]);
-      sum_lo = vmlal_s16(sum_lo, vget_low_s16(a[6]), filter_v[6]);
-      uint16x4_t sum_lo_16 = vqrshrun_n_s32(sum_lo, 11);
-
-      int32x4_t sum_hi = vdupq_n_s32(vertical_rounding);
-      sum_hi = vmlal_s16(sum_hi, vget_high_s16(a[0]), filter_v[0]);
-      sum_hi = vmlal_s16(sum_hi, vget_high_s16(a[1]), filter_v[1]);
-      sum_hi = vmlal_s16(sum_hi, vget_high_s16(a[2]), filter_v[2]);
-      sum_hi = vmlal_s16(sum_hi, vget_high_s16(a[3]), filter_v[3]);
-      sum_hi = vmlal_s16(sum_hi, vget_high_s16(a[4]), filter_v[4]);
-      sum_hi = vmlal_s16(sum_hi, vget_high_s16(a[5]), filter_v[5]);
-      sum_hi = vmlal_s16(sum_hi, vget_high_s16(a[6]), filter_v[6]);
-      uint16x4_t sum_hi_16 = vqrshrun_n_s32(sum_hi, 11);
-
-      vst1_u8(dst_v, vqmovn_u16(vcombine_u16(sum_lo_16, sum_hi_16)));
-      dst_v += dest_stride;
-
-      a[0] = a[1];
-      a[1] = a[2];
-      a[2] = a[3];
-      a[3] = a[4];
-      a[4] = a[5];
-      a[5] = a[6];
-    }
+  if (width == 4) {
+    VerticalSum<4>(wiener_buffer, buffer_stride, dst, dest_stride, filter_v,
+                   width, height);
+  } else {
+    VerticalSum<8>(wiener_buffer, buffer_stride, dst, dest_stride, filter_v,
+                   width, height);
   }
 }
 
@@ -463,7 +535,8 @@
     row_sq[1] = vmull_u8(row[1], row[1]);
     row_sq[2] = vmull_u8(row[2], row[2]);
 
-    for (int y = -1; y < height + 1; y += 2) {
+    int y = -1;
+    do {
       row[3] = vld1_u8(column);
       column += stride;
       row[4] = vld1_u8(column);
@@ -488,10 +561,12 @@
       row_sq[0] = row_sq[2];
       row_sq[1] = row_sq[3];
       row_sq[2] = row_sq[4];
-    }
+      y += 2;
+    } while (y < height + 1);
   }
 
-  for (int x = 0; x < width; x += 4) {
+  int x = 0;
+  do {
     // |src_pre_process| is X but we already processed the first column of 4
     // values so we want to start at Y and increment from there.
     // X s s s Y s s
@@ -564,7 +639,8 @@
     // Calculate one output line. Add in the line from the previous pass and
     // output one even row. Sum the new line and output the odd row. Carry the
     // new row into the next pass.
-    for (int y = 0; y < height; y += 2) {
+    int y = 0;
+    do {
       row[3] = vld1_u8(column);
       column += stride;
       row[4] = vld1_u8(column);
@@ -616,8 +692,10 @@
 
       sum565_a0 = sum565_a1;
       sum565_b0 = sum565_b1;
-    }
-  }
+      y += 2;
+    } while (y < height);
+    x += 4;
+  } while (x < width);
 }
 
 inline void BoxFilterPreProcess_SecondPass(const uint8_t* const src,
@@ -637,7 +715,8 @@
   // get 68 values. This doesn't appear to be causing problems yet but it
   // might.
   const uint8_t* const src_top_left_corner = src - 1 - 2 * stride;
-  for (int x = -1; x < width + 1; x += 4) {
+  int x = -1;
+  do {
     const uint8_t* column = src_top_left_corner + x;
     uint16_t* a2_column = a2 + (x + 1);
     uint8x8_t row[3];
@@ -650,7 +729,8 @@
     row_sq[0] = vmull_u8(row[0], row[0]);
     row_sq[1] = vmull_u8(row[1], row[1]);
 
-    for (int y = -1; y < height + 1; ++y) {
+    int y = -1;
+    do {
       row[2] = vld1_u8(column);
       column += stride;
 
@@ -672,8 +752,9 @@
 
       row_sq[0] = row_sq[1];
       row_sq[1] = row_sq[2];
-    }
-  }
+    } while (++y < height + 1);
+    x += 4;
+  } while (x < width + 1);
 }
 
 inline uint16x4_t Sum444(const uint16x8_t a) {
@@ -707,7 +788,8 @@
 
   BoxFilterPreProcess_SecondPass(src, stride, width, height, s, a2);
 
-  for (int x = 0; x < width; x += 4) {
+  int x = 0;
+  do {
     uint16_t* a2_ptr = a2 + x;
     const uint8_t* src_ptr = src + x;
     // |filtered_output| must match how |a2| values are read since they are
@@ -733,7 +815,8 @@
     sum343_b[1] = Sum343W(b_1);
     sum444_b = Sum444W(b_1);
 
-    for (int y = 0; y < height; ++y) {
+    int y = 0;
+    do {
       const uint16x8_t a_2 = vld1q_u16(a2_ptr);
       a2_ptr += kIntermediateStride;
 
@@ -762,8 +845,137 @@
 
       src_ptr += stride;
       filtered_output += kIntermediateStride;
+    } while (++y < height);
+    x += 4;
+  } while (x < width);
+}
+
+template <int min_width>
+inline void SelfGuidedSingleMultiplier(const uint8_t* src,
+                                       const ptrdiff_t src_stride,
+                                       uint16_t* box_filter_process_output,
+                                       uint8_t* dst, const ptrdiff_t dst_stride,
+                                       const int width, const int height,
+                                       const int16_t w_combo,
+                                       const int16x4_t w_single) {
+  static_assert(min_width == 4 || min_width == 8, "");
+
+  int y = 0;
+  do {
+    if (min_width == 8) {
+      int x = 0;
+      do {
+        const int16x8_t u = vreinterpretq_s16_u16(
+            vshll_n_u8(vld1_u8(src + x), kSgrProjRestoreBits));
+        const int16x8_t p =
+            vreinterpretq_s16_u16(vld1q_u16(box_filter_process_output + x));
+
+        // u * w1 + u * wN == u * (w1 + wN)
+        int32x4_t v_lo = vmull_n_s16(vget_low_s16(u), w_combo);
+        v_lo = vmlal_s16(v_lo, vget_low_s16(p), w_single);
+
+        int32x4_t v_hi = vmull_n_s16(vget_high_s16(u), w_combo);
+        v_hi = vmlal_s16(v_hi, vget_high_s16(p), w_single);
+
+        const int16x4_t s_lo =
+            vrshrn_n_s32(v_lo, kSgrProjRestoreBits + kSgrProjPrecisionBits);
+        const int16x4_t s_hi =
+            vrshrn_n_s32(v_hi, kSgrProjRestoreBits + kSgrProjPrecisionBits);
+        vst1_u8(dst + x, vqmovun_s16(vcombine_s16(s_lo, s_hi)));
+        x += 8;
+      } while (x < width);
+    } else if (min_width == 4) {
+      const int16x8_t u =
+          vreinterpretq_s16_u16(vshll_n_u8(vld1_u8(src), kSgrProjRestoreBits));
+      const int16x8_t p =
+          vreinterpretq_s16_u16(vld1q_u16(box_filter_process_output));
+
+      // u * w1 + u * wN == u * (w1 + wN)
+      int32x4_t v_lo = vmull_n_s16(vget_low_s16(u), w_combo);
+      v_lo = vmlal_s16(v_lo, vget_low_s16(p), w_single);
+
+      int32x4_t v_hi = vmull_n_s16(vget_high_s16(u), w_combo);
+      v_hi = vmlal_s16(v_hi, vget_high_s16(p), w_single);
+
+      const int16x4_t s_lo =
+          vrshrn_n_s32(v_lo, kSgrProjRestoreBits + kSgrProjPrecisionBits);
+      const int16x4_t s_hi =
+          vrshrn_n_s32(v_hi, kSgrProjRestoreBits + kSgrProjPrecisionBits);
+      StoreLo4(dst, vqmovun_s16(vcombine_s16(s_lo, s_hi)));
     }
-  }
+    src += src_stride;
+    dst += dst_stride;
+    box_filter_process_output += kIntermediateStride;
+  } while (++y < height);
+}
+
+template <int min_width>
+inline void SelfGuidedDoubleMultiplier(const uint8_t* src,
+                                       const ptrdiff_t src_stride,
+                                       uint16_t* box_filter_process_output[2],
+                                       uint8_t* dst, const ptrdiff_t dst_stride,
+                                       const int width, const int height,
+                                       const int16x4_t w0, const int w1,
+                                       const int16x4_t w2) {
+  static_assert(min_width == 4 || min_width == 8, "");
+
+  int y = 0;
+  do {
+    if (min_width == 8) {
+      int x = 0;
+      do {
+        // |wN| values are signed. |src| values can be treated as int16_t.
+        const int16x8_t u = vreinterpretq_s16_u16(
+            vshll_n_u8(vld1_u8(src + x), kSgrProjRestoreBits));
+        // |box_filter_process_output| is 14 bits, also safe to treat as
+        // int16_t.
+        const int16x8_t p0 =
+            vreinterpretq_s16_u16(vld1q_u16(box_filter_process_output[0] + x));
+        const int16x8_t p1 =
+            vreinterpretq_s16_u16(vld1q_u16(box_filter_process_output[1] + x));
+
+        int32x4_t v_lo = vmull_n_s16(vget_low_s16(u), w1);
+        v_lo = vmlal_s16(v_lo, vget_low_s16(p0), w0);
+        v_lo = vmlal_s16(v_lo, vget_low_s16(p1), w2);
+
+        int32x4_t v_hi = vmull_n_s16(vget_high_s16(u), w1);
+        v_hi = vmlal_s16(v_hi, vget_high_s16(p0), w0);
+        v_hi = vmlal_s16(v_hi, vget_high_s16(p1), w2);
+
+        // |s| is saturated to uint8_t.
+        const int16x4_t s_lo =
+            vrshrn_n_s32(v_lo, kSgrProjRestoreBits + kSgrProjPrecisionBits);
+        const int16x4_t s_hi =
+            vrshrn_n_s32(v_hi, kSgrProjRestoreBits + kSgrProjPrecisionBits);
+        vst1_u8(dst + x, vqmovun_s16(vcombine_s16(s_lo, s_hi)));
+        x += 8;
+      } while (x < width);
+    } else if (min_width == 4) {
+      // |wN| values are signed. |src| values can be treated as int16_t.
+      // Load 8 values but ignore 4.
+      const int16x4_t u = vget_low_s16(
+          vreinterpretq_s16_u16(vshll_n_u8(vld1_u8(src), kSgrProjRestoreBits)));
+      // |box_filter_process_output| is 14 bits, also safe to treat as
+      // int16_t.
+      const int16x4_t p0 =
+          vreinterpret_s16_u16(vld1_u16(box_filter_process_output[0]));
+      const int16x4_t p1 =
+          vreinterpret_s16_u16(vld1_u16(box_filter_process_output[1]));
+
+      int32x4_t v = vmull_n_s16(u, w1);
+      v = vmlal_s16(v, p0, w0);
+      v = vmlal_s16(v, p1, w2);
+
+      // |s| is saturated to uint8_t.
+      const int16x4_t s =
+          vrshrn_n_s32(v, kSgrProjRestoreBits + kSgrProjPrecisionBits);
+      StoreLo4(dst, vqmovun_s16(vcombine_s16(s, s)));
+    }
+    src += src_stride;
+    dst += dst_stride;
+    box_filter_process_output[0] += kIntermediateStride;
+    box_filter_process_output[1] += kIntermediateStride;
+  } while (++y < height);
 }
 
 // Assume box_filter_process_output[2] are allocated before calling
@@ -771,8 +983,17 @@
 void SelfGuidedFilter_NEON(const void* const source, void* const dest,
                            const RestorationUnitInfo& restoration_info,
                            ptrdiff_t source_stride, ptrdiff_t dest_stride,
-                           int width, int height,
+                           const int width, const int height,
                            RestorationBuffer* const /*buffer*/) {
+  // The output frame is broken into blocks of 64x64 (32x32 if U/V are
+  // subsampled). If either dimension is less than 32/64 it indicates it is at
+  // the right or bottom edge of the frame. It is safe to overwrite the output
+  // as it will not be part of the visible frame. This saves us from having to
+  // handle non-multiple-of-8 widths.
+  // We could round here, but the for loop with += 8 does the same thing.
+
+  // width = (width + 7) & ~0x7;
+
   // -96 to 96 (Sgrproj_Xqd_Min/Max)
   const int8_t w0 = restoration_info.sgr_proj_info.multiplier[0];
   const int8_t w1 = restoration_info.sgr_proj_info.multiplier[1];
@@ -815,38 +1036,15 @@
   // is no vmlal_n_s16().
   const int16x4_t w0_v = vdup_n_s16(w0);
   const int16x4_t w2_v = vdup_n_s16(w2);
-  assert(width % 8 == 0);
   if (radius_pass_0 != 0 && radius_pass_1 != 0) {
-    for (int y = 0; y < height; ++y) {
-      for (int x = 0; x < width; x += 8) {
-        // |wN| values are signed. |src| values can be treated as int16_t.
-        const int16x8_t u_v = vreinterpretq_s16_u16(
-            vshll_n_u8(vld1_u8(src + x), kSgrProjRestoreBits));
-        // |filtered_output| is 14 bits, also safe to treat as int16_t.
-        const int16x8_t p0_v =
-            vreinterpretq_s16_u16(vld1q_u16(box_filter_process_output[0] + x));
-        const int16x8_t p1_v =
-            vreinterpretq_s16_u16(vld1q_u16(box_filter_process_output[1] + x));
-
-        int32x4_t v_lo = vmull_n_s16(vget_low_s16(u_v), w1);
-        v_lo = vmlal_s16(v_lo, vget_low_s16(p0_v), w0_v);
-        v_lo = vmlal_s16(v_lo, vget_low_s16(p1_v), w2_v);
-
-        int32x4_t v_hi = vmull_n_s16(vget_high_s16(u_v), w1);
-        v_hi = vmlal_s16(v_hi, vget_high_s16(p0_v), w0_v);
-        v_hi = vmlal_s16(v_hi, vget_high_s16(p1_v), w2_v);
-
-        // |s| is saturated to uint8_t.
-        const int16x4_t s_lo =
-            vrshrn_n_s32(v_lo, kSgrProjRestoreBits + kSgrProjPrecisionBits);
-        const int16x4_t s_hi =
-            vrshrn_n_s32(v_hi, kSgrProjRestoreBits + kSgrProjPrecisionBits);
-        vst1_u8(dst + x, vqmovun_s16(vcombine_s16(s_lo, s_hi)));
-      }
-      src += source_stride;
-      dst += dest_stride;
-      box_filter_process_output[0] += kIntermediateStride;
-      box_filter_process_output[1] += kIntermediateStride;
+    if (width > 4) {
+      SelfGuidedDoubleMultiplier<8>(src, source_stride,
+                                    box_filter_process_output, dst, dest_stride,
+                                    width, height, w0_v, w1, w2_v);
+    } else /* if (width == 4) */ {
+      SelfGuidedDoubleMultiplier<4>(src, source_stride,
+                                    box_filter_process_output, dst, dest_stride,
+                                    width, height, w0_v, w1, w2_v);
     }
   } else {
     int16_t w_combo;
@@ -862,29 +1060,14 @@
       box_filter_process_output_n = box_filter_process_output[1];
     }
 
-    for (int y = 0; y < height; ++y) {
-      for (int x = 0; x < width; x += 8) {
-        const int16x8_t u_v = vreinterpretq_s16_u16(
-            vshll_n_u8(vld1_u8(src + x), kSgrProjRestoreBits));
-        const int16x8_t pN_v =
-            vreinterpretq_s16_u16(vld1q_u16(box_filter_process_output_n + x));
-
-        // u_v * w1 + u_v * wN == u_v * (w1 + wN)
-        int32x4_t v_lo = vmull_n_s16(vget_low_s16(u_v), w_combo);
-        v_lo = vmlal_s16(v_lo, vget_low_s16(pN_v), w_single);
-
-        int32x4_t v_hi = vmull_n_s16(vget_high_s16(u_v), w_combo);
-        v_hi = vmlal_s16(v_hi, vget_high_s16(pN_v), w_single);
-
-        const int16x4_t s_lo =
-            vrshrn_n_s32(v_lo, kSgrProjRestoreBits + kSgrProjPrecisionBits);
-        const int16x4_t s_hi =
-            vrshrn_n_s32(v_hi, kSgrProjRestoreBits + kSgrProjPrecisionBits);
-        vst1_u8(dst + x, vqmovun_s16(vcombine_s16(s_lo, s_hi)));
-      }
-      src += source_stride;
-      dst += dest_stride;
-      box_filter_process_output_n += kIntermediateStride;
+    if (width > 4) {
+      SelfGuidedSingleMultiplier<8>(
+          src, source_stride, box_filter_process_output_n, dst, dest_stride,
+          width, height, w_combo, w_single);
+    } else /* if (width == 4) */ {
+      SelfGuidedSingleMultiplier<4>(
+          src, source_stride, box_filter_process_output_n, dst, dest_stride,
+          width, height, w_combo, w_single);
     }
   }
 }
diff --git a/libgav1/src/dsp/arm/loop_restoration_neon.h b/libgav1/src/dsp/arm/loop_restoration_neon.h
index 85a06c8..1723c50 100644
--- a/libgav1/src/dsp/arm/loop_restoration_neon.h
+++ b/libgav1/src/dsp/arm/loop_restoration_neon.h
@@ -3,13 +3,12 @@
 
 #include "src/dsp/cpu.h"
 #include "src/dsp/dsp.h"
-#include "src/dsp/loop_filter.h"
 
 namespace libgav1 {
 namespace dsp {
 
-// Initializes Dsp::loop_restorations with neon implementations. This function
-// is not thread-safe.
+// Initializes Dsp::loop_restorations, see the defines below for specifics.
+// This function is not thread-safe.
 void LoopRestorationInit_NEON();
 
 }  // namespace dsp
diff --git a/libgav1/src/dsp/arm/mask_blend_neon.cc b/libgav1/src/dsp/arm/mask_blend_neon.cc
new file mode 100644
index 0000000..bc69c55
--- /dev/null
+++ b/libgav1/src/dsp/arm/mask_blend_neon.cc
@@ -0,0 +1,319 @@
+#include "src/dsp/dsp.h"
+#include "src/dsp/mask_blend.h"
+
+#if LIBGAV1_ENABLE_NEON
+
+#include <arm_neon.h>
+
+#include <cassert>
+#include <cstddef>
+#include <cstdint>
+
+#include "src/dsp/arm/common_neon.h"
+#include "src/utils/common.h"
+
+namespace libgav1 {
+namespace dsp {
+namespace low_bitdepth {
+namespace {
+
+constexpr int kBitdepth8 = 8;
+
+template <int subsampling_x, int subsampling_y>
+inline uint16x8_t GetMask4x2(const uint8_t* mask, ptrdiff_t mask_stride) {
+  if (subsampling_x == 1) {
+    const uint16x4_t mask_val0 = vpaddl_u8(vld1_u8(mask));
+    const uint16x4_t mask_val1 =
+        vpaddl_u8(vld1_u8(mask + (mask_stride << subsampling_y)));
+    uint16x8_t final_val;
+    if (subsampling_y == 1) {
+      const uint16x4_t next_mask_val0 = vpaddl_u8(vld1_u8(mask + mask_stride));
+      const uint16x4_t next_mask_val1 =
+          vpaddl_u8(vld1_u8(mask + mask_stride * 3));
+      final_val = vaddq_u16(vcombine_u16(mask_val0, mask_val1),
+                            vcombine_u16(next_mask_val0, next_mask_val1));
+    } else {
+      final_val = vpaddlq_u8(vcombine_u8(mask_val0, mask_val1));
+    }
+    return vrshrq_n_u16(final_val, subsampling_y + 1);
+  }
+  assert(subsampling_y == 0 && subsampling_x == 0);
+  const uint8x8_t mask_val0 = LoadLo4(mask, vdup_n_u8(0));
+  const uint8x8_t mask_val =
+      LoadHi4(mask + (mask_stride << subsampling_y), mask_val0);
+  return vmovl_u8(mask_val);
+}
+
+template <int subsampling_x, int subsampling_y>
+inline uint16x8_t GetMask8(const uint8_t* mask, ptrdiff_t mask_stride) {
+  if (subsampling_x == 1) {
+    uint16x8_t mask_val = vpaddlq_u8(vld1q_u8(mask));
+    if (subsampling_y == 1) {
+      const uint16x8_t next_mask_val = vpaddlq_u8(vld1q_u8(mask + mask_stride));
+      mask_val = vaddq_u16(mask_val, next_mask_val);
+    }
+    return vrshrq_n_u16(mask_val, 1 + subsampling_y);
+  }
+  assert(subsampling_y == 0 && subsampling_x == 0);
+  const uint8x8_t mask_val = vld1_u8(mask);
+  return vmovl_u8(mask_val);
+}
+
+template <bool is_inter_intra>
+inline void WriteMaskBlendLine4x2(const uint16_t* const pred_0,
+                                  const ptrdiff_t pred_stride_0,
+                                  const uint16_t* const pred_1,
+                                  const ptrdiff_t pred_stride_1,
+                                  const uint16x8_t pred_mask_0,
+                                  const uint16x8_t pred_mask_1, uint8_t* dst,
+                                  const ptrdiff_t dst_stride) {
+  const uint16x4_t pred_val_0_lo = vld1_u16(pred_0);
+  const uint16x4_t pred_val_0_hi = vld1_u16(pred_0 + pred_stride_0);
+  uint16x4_t pred_val_1_lo = vld1_u16(pred_1);
+  uint16x4_t pred_val_1_hi = vld1_u16(pred_1 + pred_stride_1);
+  uint8x8_t result;
+  if (is_inter_intra) {
+    // An offset to cancel offsets used in compound predictor generation
+    // that make intermediate computations non negative.
+    const uint16x8_t single_round_offset =
+        vdupq_n_u16((1 << kBitdepth8) + (1 << (kBitdepth8 - 1)));
+    // pred_0 and pred_1 are switched at the beginning with is_inter_intra.
+    // Clip3(prediction_0[x] - single_round_offset, 0, (1 << kBitdepth8) - 1)
+    const uint16x8_t pred_val_1 = vmovl_u8(vqmovn_u16(vqsubq_u16(
+        vcombine_u16(pred_val_1_lo, pred_val_1_hi), single_round_offset)));
+
+    const uint16x8_t pred_val_0 = vcombine_u16(pred_val_0_lo, pred_val_0_hi);
+    const uint16x8_t weighted_pred_0 = vmulq_u16(pred_val_0, pred_mask_0);
+    const uint16x8_t weighted_combo =
+        vmlaq_u16(weighted_pred_0, pred_mask_1, pred_val_1);
+    result = vrshrn_n_u16(weighted_combo, 6);
+  } else {
+    // int res = (mask_value * prediction_0[x] +
+    //      (64 - mask_value) * prediction_1[x]) >> 6;
+    const uint32x4_t weighted_pred_0_lo =
+        vmull_u16(vget_low_u16(pred_mask_0), pred_val_0_lo);
+    const uint32x4_t weighted_pred_0_hi =
+        vmull_u16(vget_high_u16(pred_mask_0), pred_val_0_hi);
+    const uint32x4_t weighted_combo_lo =
+        vmlal_u16(weighted_pred_0_lo, vget_low_u16(pred_mask_1), pred_val_1_lo);
+    const uint32x4_t weighted_combo_hi = vmlal_u16(
+        weighted_pred_0_hi, vget_high_u16(pred_mask_1), pred_val_1_hi);
+    // res -= compound_round_offset;
+    // dst[x] = static_cast<Pixel>(
+    //     Clip3(RightShiftWithRounding(res, inter_post_round_bits), 0,
+    //         (1 << kBitdepth8) - 1));
+    const int16x8_t compound_round_offset =
+        vdupq_n_s16((1 << (kBitdepth8 + 4)) + (1 << (kBitdepth8 + 3)));
+    result = vqrshrun_n_s16(vsubq_s16(vreinterpretq_s16_u16(vcombine_u16(
+                                          vshrn_n_u32(weighted_combo_lo, 6),
+                                          vshrn_n_u32(weighted_combo_hi, 6))),
+                                      compound_round_offset),
+                            4);
+  }
+  StoreLo4(dst, result);
+  StoreHi4(dst + dst_stride, result);
+}
+
+template <bool is_inter_intra, int subsampling_x, int subsampling_y>
+inline void MaskBlending4x4_NEON(const uint16_t* pred_0,
+                                 const ptrdiff_t prediction_stride_0,
+                                 const uint16_t* pred_1,
+                                 const ptrdiff_t prediction_stride_1,
+                                 const uint8_t* mask,
+                                 const ptrdiff_t mask_stride, uint8_t* dst,
+                                 const ptrdiff_t dst_stride) {
+  const uint16x8_t mask_inverter = vdupq_n_u16(64);
+  uint16x8_t pred_mask_0 =
+      GetMask4x2<subsampling_x, subsampling_y>(mask, mask_stride);
+  uint16x8_t pred_mask_1 = vsubq_u16(mask_inverter, pred_mask_0);
+  WriteMaskBlendLine4x2<is_inter_intra>(pred_0, prediction_stride_0, pred_1,
+                                        prediction_stride_1, pred_mask_0,
+                                        pred_mask_1, dst, dst_stride);
+  pred_0 += prediction_stride_0 << 1;
+  pred_1 += prediction_stride_1 << 1;
+  mask += mask_stride << (1 + subsampling_y);
+  dst += dst_stride << 1;
+
+  pred_mask_0 = GetMask4x2<subsampling_x, subsampling_y>(mask, mask_stride);
+  pred_mask_1 = vsubq_u16(mask_inverter, pred_mask_0);
+  WriteMaskBlendLine4x2<is_inter_intra>(pred_0, prediction_stride_0, pred_1,
+                                        prediction_stride_1, pred_mask_0,
+                                        pred_mask_1, dst, dst_stride);
+}
+
+template <bool is_inter_intra, int subsampling_x, int subsampling_y>
+inline void MaskBlending4xH_NEON(const uint16_t* pred_0,
+                                 const ptrdiff_t pred_stride_0,
+                                 const int height, const uint16_t* pred_1,
+                                 const ptrdiff_t pred_stride_1,
+                                 const uint8_t* const mask_ptr,
+                                 const ptrdiff_t mask_stride, uint8_t* dst,
+                                 const ptrdiff_t dst_stride) {
+  const uint8_t* mask = mask_ptr;
+  if (height == 4) {
+    MaskBlending4x4_NEON<is_inter_intra, subsampling_x, subsampling_y>(
+        pred_0, pred_stride_0, pred_1, pred_stride_1, mask, mask_stride, dst,
+        dst_stride);
+    return;
+  }
+  const uint16x8_t mask_inverter = vdupq_n_u16(64);
+  int y = 0;
+  do {
+    uint16x8_t pred_mask_0 =
+        GetMask4x2<subsampling_x, subsampling_y>(mask, mask_stride);
+    uint16x8_t pred_mask_1 = vsubq_u16(mask_inverter, pred_mask_0);
+
+    WriteMaskBlendLine4x2<is_inter_intra>(pred_0, pred_stride_0, pred_1,
+                                          pred_stride_1, pred_mask_0,
+                                          pred_mask_1, dst, dst_stride);
+    pred_0 += pred_stride_0 << 1;
+    pred_1 += pred_stride_1 << 1;
+    mask += mask_stride << (1 + subsampling_y);
+    dst += dst_stride << 1;
+
+    pred_mask_0 = GetMask4x2<subsampling_x, subsampling_y>(mask, mask_stride);
+    pred_mask_1 = vsubq_u16(mask_inverter, pred_mask_0);
+    WriteMaskBlendLine4x2<is_inter_intra>(pred_0, pred_stride_0, pred_1,
+                                          pred_stride_1, pred_mask_0,
+                                          pred_mask_1, dst, dst_stride);
+    pred_0 += pred_stride_0 << 1;
+    pred_1 += pred_stride_1 << 1;
+    mask += mask_stride << (1 + subsampling_y);
+    dst += dst_stride << 1;
+
+    pred_mask_0 = GetMask4x2<subsampling_x, subsampling_y>(mask, mask_stride);
+    pred_mask_1 = vsubq_u16(mask_inverter, pred_mask_0);
+    WriteMaskBlendLine4x2<is_inter_intra>(pred_0, pred_stride_0, pred_1,
+                                          pred_stride_1, pred_mask_0,
+                                          pred_mask_1, dst, dst_stride);
+    pred_0 += pred_stride_0 << 1;
+    pred_1 += pred_stride_1 << 1;
+    mask += mask_stride << (1 + subsampling_y);
+    dst += dst_stride << 1;
+
+    pred_mask_0 = GetMask4x2<subsampling_x, subsampling_y>(mask, mask_stride);
+    pred_mask_1 = vsubq_u16(mask_inverter, pred_mask_0);
+    WriteMaskBlendLine4x2<is_inter_intra>(pred_0, pred_stride_0, pred_1,
+                                          pred_stride_1, pred_mask_0,
+                                          pred_mask_1, dst, dst_stride);
+    pred_0 += pred_stride_0 << 1;
+    pred_1 += pred_stride_1 << 1;
+    mask += mask_stride << (1 + subsampling_y);
+    dst += dst_stride << 1;
+    y += 8;
+  } while (y < height);
+}
+
+template <bool is_inter_intra, int subsampling_x, int subsampling_y>
+inline void MaskBlend_NEON(
+    const uint16_t* prediction_0, const ptrdiff_t prediction_stride_0,
+    const uint16_t* prediction_1, const ptrdiff_t prediction_stride_1,
+    const uint8_t* const mask_ptr, const ptrdiff_t mask_stride, const int width,
+    const int height, void* dest, const ptrdiff_t dst_stride) {
+  uint8_t* dst = reinterpret_cast<uint8_t*>(dest);
+  const uint16_t* pred_0 = is_inter_intra ? prediction_1 : prediction_0;
+  const uint16_t* pred_1 = is_inter_intra ? prediction_0 : prediction_1;
+  const ptrdiff_t pred_stride_0 =
+      is_inter_intra ? prediction_stride_1 : prediction_stride_0;
+  const ptrdiff_t pred_stride_1 =
+      is_inter_intra ? prediction_stride_0 : prediction_stride_1;
+  if (width == 4) {
+    MaskBlending4xH_NEON<is_inter_intra, subsampling_x, subsampling_y>(
+        pred_0, pred_stride_0, height, pred_1, pred_stride_1, mask_ptr,
+        mask_stride, dst, dst_stride);
+    return;
+  }
+  const uint8_t* mask = mask_ptr;
+  const uint16x8_t mask_inverter = vdupq_n_u16(64);
+  int y = 0;
+  do {
+    int x = 0;
+    do {
+      const uint16x8_t pred_mask_0 = GetMask8<subsampling_x, subsampling_y>(
+          mask + (x << subsampling_x), mask_stride);
+      // 64 - mask
+      const uint16x8_t pred_mask_1 = vsubq_u16(mask_inverter, pred_mask_0);
+      const uint16x8_t pred_val_0 = vld1q_u16(pred_0 + x);
+      uint16x8_t pred_val_1 = vld1q_u16(pred_1 + x);
+      if (is_inter_intra) {
+        // An offset to cancel offsets used in compound predictor generation
+        // that make intermediate computations non negative.
+        const uint16x8_t single_round_offset =
+            vdupq_n_u16((1 << kBitdepth8) + (1 << (kBitdepth8 - 1)));
+        pred_val_1 =
+            vmovl_u8(vqmovn_u16(vqsubq_u16(pred_val_1, single_round_offset)));
+      }
+      uint8x8_t result;
+      if (is_inter_intra) {
+        const uint16x8_t weighted_pred_0 = vmulq_u16(pred_mask_0, pred_val_0);
+        // weighted_pred0 + weighted_pred1
+        const uint16x8_t weighted_combo =
+            vmlaq_u16(weighted_pred_0, pred_mask_1, pred_val_1);
+        result = vrshrn_n_u16(weighted_combo, 6);
+      } else {
+        // int res = (mask_value * prediction_0[x] +
+        //      (64 - mask_value) * prediction_1[x]) >> 6;
+        const uint32x4_t weighted_pred_0_lo =
+            vmull_u16(vget_low_u16(pred_mask_0), vget_low_u16(pred_val_0));
+        const uint32x4_t weighted_pred_0_hi =
+            vmull_u16(vget_high_u16(pred_mask_0), vget_high_u16(pred_val_0));
+        const uint32x4_t weighted_combo_lo =
+            vmlal_u16(weighted_pred_0_lo, vget_low_u16(pred_mask_1),
+                      vget_low_u16(pred_val_1));
+        const uint32x4_t weighted_combo_hi =
+            vmlal_u16(weighted_pred_0_hi, vget_high_u16(pred_mask_1),
+                      vget_high_u16(pred_val_1));
+
+        const int16x8_t compound_round_offset =
+            vdupq_n_s16((1 << (kBitdepth8 + 4)) + (1 << (kBitdepth8 + 3)));
+        // res -= compound_round_offset;
+        // dst[x] = static_cast<Pixel>(
+        //     Clip3(RightShiftWithRounding(res, inter_post_round_bits), 0,
+        //           (1 << kBitdepth8) - 1));
+        result =
+            vqrshrun_n_s16(vsubq_s16(vreinterpretq_s16_u16(vcombine_u16(
+                                         vshrn_n_u32(weighted_combo_lo, 6),
+                                         vshrn_n_u32(weighted_combo_hi, 6))),
+                                     compound_round_offset),
+                           4);
+      }
+      vst1_u8(dst + x, result);
+
+      x += 8;
+    } while (x < width);
+    dst += dst_stride;
+    pred_0 += pred_stride_0;
+    pred_1 += pred_stride_1;
+    mask += mask_stride << subsampling_y;
+  } while (++y < height);
+}
+
+void Init8bpp() {
+  Dsp* const dsp = dsp_internal::GetWritableDspTable(8);
+  assert(dsp != nullptr);
+  dsp->mask_blend[0][0] = MaskBlend_NEON<false, 0, 0>;
+  dsp->mask_blend[1][0] = MaskBlend_NEON<false, 1, 0>;
+  dsp->mask_blend[2][0] = MaskBlend_NEON<false, 1, 1>;
+  dsp->mask_blend[0][1] = MaskBlend_NEON<true, 0, 0>;
+  dsp->mask_blend[1][1] = MaskBlend_NEON<true, 1, 0>;
+  dsp->mask_blend[2][1] = MaskBlend_NEON<true, 1, 1>;
+}
+
+}  // namespace
+}  // namespace low_bitdepth
+
+void MaskBlendInit_NEON() { low_bitdepth::Init8bpp(); }
+
+}  // namespace dsp
+}  // namespace libgav1
+
+#else   // !LIBGAV1_ENABLE_NEON
+
+namespace libgav1 {
+namespace dsp {
+
+void MaskBlendInit_NEON() {}
+
+}  // namespace dsp
+}  // namespace libgav1
+#endif  // LIBGAV1_ENABLE_NEON
diff --git a/libgav1/src/dsp/arm/mask_blend_neon.h b/libgav1/src/dsp/arm/mask_blend_neon.h
new file mode 100644
index 0000000..eed54dd
--- /dev/null
+++ b/libgav1/src/dsp/arm/mask_blend_neon.h
@@ -0,0 +1,25 @@
+#ifndef LIBGAV1_SRC_DSP_ARM_MASK_BLEND_NEON_H_
+#define LIBGAV1_SRC_DSP_ARM_MASK_BLEND_NEON_H_
+
+#include "src/dsp/cpu.h"
+#include "src/dsp/dsp.h"
+
+namespace libgav1 {
+namespace dsp {
+
+// Initializes Dsp::mask_blend. This function is not thread-safe.
+void MaskBlendInit_NEON();
+
+}  // namespace dsp
+}  // namespace libgav1
+
+#if LIBGAV1_ENABLE_NEON
+#define LIBGAV1_Dsp8bpp_MaskBlend444 LIBGAV1_DSP_NEON
+#define LIBGAV1_Dsp8bpp_MaskBlend422 LIBGAV1_DSP_NEON
+#define LIBGAV1_Dsp8bpp_MaskBlend420 LIBGAV1_DSP_NEON
+#define LIBGAV1_Dsp8bpp_MaskBlendInterIntra444 LIBGAV1_DSP_NEON
+#define LIBGAV1_Dsp8bpp_MaskBlendInterIntra422 LIBGAV1_DSP_NEON
+#define LIBGAV1_Dsp8bpp_MaskBlendInterIntra420 LIBGAV1_DSP_NEON
+#endif  // LIBGAV1_ENABLE_NEON
+
+#endif  // LIBGAV1_SRC_DSP_ARM_MASK_BLEND_NEON_H_
diff --git a/libgav1/src/dsp/arm/obmc_neon.cc b/libgav1/src/dsp/arm/obmc_neon.cc
new file mode 100644
index 0000000..88c42c0
--- /dev/null
+++ b/libgav1/src/dsp/arm/obmc_neon.cc
@@ -0,0 +1,387 @@
+#include "src/dsp/dsp.h"
+#include "src/dsp/obmc.h"
+
+#if LIBGAV1_ENABLE_NEON
+
+#include <arm_neon.h>
+
+#include <algorithm>
+#include <cassert>
+#include <cstddef>
+#include <cstdint>
+#include <cstring>
+
+#include "src/dsp/arm/common_neon.h"
+#include "src/utils/common.h"
+
+namespace libgav1 {
+namespace dsp {
+namespace {
+
+#include "src/dsp/obmc.inc"
+
+inline uint8x8_t Load2(const uint8_t* src) {
+  uint16_t tmp;
+  memcpy(&tmp, src, 2);
+  uint16x4_t result = vcreate_u16(tmp);
+  return vreinterpret_u8_u16(result);
+}
+
+template <int lane>
+inline void StoreLane2(uint8_t* dst, uint8x8_t src) {
+  const uint16_t out_val = vget_lane_u16(vreinterpret_u16_u8(src), lane);
+  memcpy(dst, &out_val, 2);
+}
+
+inline void WriteObmcLine4(uint8_t* const pred, const uint8_t* const obmc_pred,
+                           const uint8x8_t pred_mask,
+                           const uint8x8_t obmc_pred_mask) {
+  const uint8x8_t pred_val = LoadLo4(pred, vdup_n_u8(0));
+  const uint8x8_t obmc_pred_val = LoadLo4(obmc_pred, vdup_n_u8(0));
+  const uint16x8_t weighted_pred = vmull_u8(pred_mask, pred_val);
+  const uint8x8_t result =
+      vrshrn_n_u16(vmlal_u8(weighted_pred, obmc_pred_mask, obmc_pred_val), 6);
+  StoreLo4(pred, result);
+}
+
+template <bool from_left>
+inline void OverlapBlend2xH_NEON(uint8_t* const prediction,
+                                 const ptrdiff_t prediction_stride,
+                                 const int height,
+                                 const uint8_t* const obmc_prediction,
+                                 const ptrdiff_t obmc_prediction_stride) {
+  uint8_t* pred = prediction;
+  const uint8x8_t mask_inverter = vdup_n_u8(64);
+  const uint8_t* obmc_pred = obmc_prediction;
+  uint8x8_t pred_mask;
+  uint8x8_t obmc_pred_mask;
+  int compute_height;
+  const int mask_offset = height - 2;
+  if (from_left) {
+    pred_mask = Load2(kObmcMask);
+    obmc_pred_mask = vsub_u8(mask_inverter, pred_mask);
+    compute_height = height;
+  } else {
+    // Weights for the last line are all 64, which is a no-op.
+    compute_height = height - 1;
+  }
+  int y = 0;
+  do {
+    if (!from_left) {
+      pred_mask = vdup_n_u8(kObmcMask[mask_offset + y]);
+      obmc_pred_mask = vsub_u8(mask_inverter, pred_mask);
+    }
+    const uint8x8_t pred_val = Load2(pred);
+    const uint16x8_t weighted_pred = vmull_u8(pred_mask, pred_val);
+    const uint8x8_t obmc_pred_val = Load2(obmc_pred);
+    const uint8x8_t result =
+        vrshrn_n_u16(vmlal_u8(weighted_pred, obmc_pred_mask, obmc_pred_val), 6);
+    StoreLane2<0>(pred, result);
+
+    pred += prediction_stride;
+    obmc_pred += obmc_prediction_stride;
+  } while (++y != compute_height);
+}
+
+inline void OverlapBlendFromLeft4xH_NEON(
+    uint8_t* const prediction, const ptrdiff_t prediction_stride,
+    const int height, const uint8_t* const obmc_prediction,
+    const ptrdiff_t obmc_prediction_stride) {
+  uint8_t* pred = prediction;
+  const uint8_t* obmc_pred = obmc_prediction;
+
+  const uint8x8_t mask_inverter = vdup_n_u8(64);
+  const uint8x8_t pred_mask = LoadLo4(kObmcMask + 2, vdup_n_u8(0));
+  // 64 - mask
+  const uint8x8_t obmc_pred_mask = vsub_u8(mask_inverter, pred_mask);
+  int y = 0;
+  do {
+    WriteObmcLine4(pred, obmc_pred, pred_mask, obmc_pred_mask);
+    pred += prediction_stride;
+    obmc_pred += obmc_prediction_stride;
+
+    WriteObmcLine4(pred, obmc_pred, pred_mask, obmc_pred_mask);
+    pred += prediction_stride;
+    obmc_pred += obmc_prediction_stride;
+
+    y += 2;
+  } while (y != height);
+}
+
+inline void OverlapBlendFromLeft8xH_NEON(
+    uint8_t* const prediction, const ptrdiff_t prediction_stride,
+    const int height, const uint8_t* const obmc_prediction,
+    const ptrdiff_t obmc_prediction_stride) {
+  uint8_t* pred = prediction;
+  const uint8_t* obmc_pred = obmc_prediction;
+  const uint8x8_t mask_inverter = vdup_n_u8(64);
+  const uint8x8_t pred_mask = vld1_u8(kObmcMask + 6);
+  // 64 - mask
+  const uint8x8_t obmc_pred_mask = vsub_u8(mask_inverter, pred_mask);
+  int y = 0;
+  do {
+    const uint8x8_t pred_val = vld1_u8(pred);
+    const uint16x8_t weighted_pred = vmull_u8(pred_mask, pred_val);
+    const uint8x8_t obmc_pred_val = vld1_u8(obmc_pred);
+    const uint8x8_t result =
+        vrshrn_n_u16(vmlal_u8(weighted_pred, obmc_pred_mask, obmc_pred_val), 6);
+
+    vst1_u8(pred, result);
+    pred += prediction_stride;
+    obmc_pred += obmc_prediction_stride;
+  } while (++y != height);
+}
+
+void OverlapBlendFromLeft_NEON(void* const prediction,
+                               const ptrdiff_t prediction_stride,
+                               const int width, const int height,
+                               const void* const obmc_prediction,
+                               const ptrdiff_t obmc_prediction_stride) {
+  auto* pred = static_cast<uint8_t*>(prediction);
+  const auto* obmc_pred = static_cast<const uint8_t*>(obmc_prediction);
+
+  if (width == 2) {
+    OverlapBlend2xH_NEON<true>(pred, prediction_stride, height, obmc_pred,
+                               obmc_prediction_stride);
+    return;
+  }
+  if (width == 4) {
+    OverlapBlendFromLeft4xH_NEON(pred, prediction_stride, height, obmc_pred,
+                                 obmc_prediction_stride);
+    return;
+  }
+  if (width == 8) {
+    OverlapBlendFromLeft8xH_NEON(pred, prediction_stride, height, obmc_pred,
+                                 obmc_prediction_stride);
+    return;
+  }
+  const uint8x16_t mask_inverter = vdupq_n_u8(64);
+  const uint8_t* mask = kObmcMask + width - 2;
+  int x = 0;
+  do {
+    pred = static_cast<uint8_t*>(prediction) + x;
+    obmc_pred = static_cast<const uint8_t*>(obmc_prediction) + x;
+    const uint8x16_t pred_mask = vld1q_u8(mask + x);
+    // 64 - mask
+    const uint8x16_t obmc_pred_mask = vsubq_u8(mask_inverter, pred_mask);
+    int y = 0;
+    do {
+      const uint8x16_t pred_val = vld1q_u8(pred);
+      const uint8x16_t obmc_pred_val = vld1q_u8(obmc_pred);
+      const uint16x8_t weighted_pred_lo =
+          vmull_u8(vget_low_u8(pred_mask), vget_low_u8(pred_val));
+      const uint8x8_t result_lo =
+          vrshrn_n_u16(vmlal_u8(weighted_pred_lo, vget_low_u8(obmc_pred_mask),
+                                vget_low_u8(obmc_pred_val)),
+                       6);
+      const uint16x8_t weighted_pred_hi =
+          vmull_u8(vget_high_u8(pred_mask), vget_high_u8(pred_val));
+      const uint8x8_t result_hi =
+          vrshrn_n_u16(vmlal_u8(weighted_pred_hi, vget_high_u8(obmc_pred_mask),
+                                vget_high_u8(obmc_pred_val)),
+                       6);
+      vst1q_u8(pred, vcombine_u8(result_lo, result_hi));
+
+      pred += prediction_stride;
+      obmc_pred += obmc_prediction_stride;
+    } while (++y < height);
+    x += 16;
+  } while (x < width);
+}
+
+inline void OverlapBlendFromTop4x4_NEON(uint8_t* const prediction,
+                                        const ptrdiff_t prediction_stride,
+                                        const uint8_t* const obmc_prediction,
+                                        const ptrdiff_t obmc_prediction_stride,
+                                        const int height) {
+  uint8_t* pred = prediction;
+  const uint8_t* obmc_pred = obmc_prediction;
+  uint8x8_t pred_mask = vdup_n_u8(kObmcMask[height - 2]);
+  const uint8x8_t mask_inverter = vdup_n_u8(64);
+  uint8x8_t obmc_pred_mask = vsub_u8(mask_inverter, pred_mask);
+  WriteObmcLine4(pred, obmc_pred, pred_mask, obmc_pred_mask);
+  pred += prediction_stride;
+  obmc_pred += obmc_prediction_stride;
+
+  if (height == 2) {
+    return;
+  }
+
+  pred_mask = vdup_n_u8(kObmcMask[3]);
+  obmc_pred_mask = vsub_u8(mask_inverter, pred_mask);
+  WriteObmcLine4(pred, obmc_pred, pred_mask, obmc_pred_mask);
+  pred += prediction_stride;
+  obmc_pred += obmc_prediction_stride;
+
+  pred_mask = vdup_n_u8(kObmcMask[4]);
+  obmc_pred_mask = vsub_u8(mask_inverter, pred_mask);
+  WriteObmcLine4(pred, obmc_pred, pred_mask, obmc_pred_mask);
+}
+
+inline void OverlapBlendFromTop4xH_NEON(
+    uint8_t* const prediction, const ptrdiff_t prediction_stride,
+    const int height, const uint8_t* const obmc_prediction,
+    const ptrdiff_t obmc_prediction_stride) {
+  if (height < 8) {
+    OverlapBlendFromTop4x4_NEON(prediction, prediction_stride, obmc_prediction,
+                                obmc_prediction_stride, height);
+    return;
+  }
+  uint8_t* pred = prediction;
+  const uint8_t* obmc_pred = obmc_prediction;
+  const uint8_t* mask = kObmcMask + height - 2;
+  const uint8x8_t mask_inverter = vdup_n_u8(64);
+  int y = 0;
+  // Compute 6 lines for height 8, or 12 lines for height 16. The remaining
+  // lines are unchanged as the corresponding mask value is 64.
+  do {
+    uint8x8_t pred_mask = vdup_n_u8(mask[y]);
+    uint8x8_t obmc_pred_mask = vsub_u8(mask_inverter, pred_mask);
+    WriteObmcLine4(pred, obmc_pred, pred_mask, obmc_pred_mask);
+    pred += prediction_stride;
+    obmc_pred += obmc_prediction_stride;
+
+    pred_mask = vdup_n_u8(mask[y + 1]);
+    obmc_pred_mask = vsub_u8(mask_inverter, pred_mask);
+    WriteObmcLine4(pred, obmc_pred, pred_mask, obmc_pred_mask);
+    pred += prediction_stride;
+    obmc_pred += obmc_prediction_stride;
+
+    pred_mask = vdup_n_u8(mask[y + 2]);
+    obmc_pred_mask = vsub_u8(mask_inverter, pred_mask);
+    WriteObmcLine4(pred, obmc_pred, pred_mask, obmc_pred_mask);
+    pred += prediction_stride;
+    obmc_pred += obmc_prediction_stride;
+
+    pred_mask = vdup_n_u8(mask[y + 3]);
+    obmc_pred_mask = vsub_u8(mask_inverter, pred_mask);
+    WriteObmcLine4(pred, obmc_pred, pred_mask, obmc_pred_mask);
+    pred += prediction_stride;
+    obmc_pred += obmc_prediction_stride;
+
+    pred_mask = vdup_n_u8(mask[y + 4]);
+    obmc_pred_mask = vsub_u8(mask_inverter, pred_mask);
+    WriteObmcLine4(pred, obmc_pred, pred_mask, obmc_pred_mask);
+    pred += prediction_stride;
+    obmc_pred += obmc_prediction_stride;
+
+    pred_mask = vdup_n_u8(mask[y + 5]);
+    obmc_pred_mask = vsub_u8(mask_inverter, pred_mask);
+    WriteObmcLine4(pred, obmc_pred, pred_mask, obmc_pred_mask);
+    pred += prediction_stride;
+    obmc_pred += obmc_prediction_stride;
+
+    // Increment for the right mask index.
+    y += 6;
+  } while (y < height - 4);
+}
+
+inline void OverlapBlendFromTop8xH_NEON(
+    uint8_t* const prediction, const ptrdiff_t prediction_stride,
+    const int height, const uint8_t* const obmc_prediction,
+    const ptrdiff_t obmc_prediction_stride) {
+  uint8_t* pred = prediction;
+  const uint8_t* obmc_pred = obmc_prediction;
+  const uint8x8_t mask_inverter = vdup_n_u8(64);
+  const uint8_t* mask = kObmcMask + height - 2;
+  const int compute_height = height - (height >> 2);
+  int y = 0;
+  do {
+    const uint8x8_t pred_mask = vdup_n_u8(mask[y]);
+    // 64 - mask
+    const uint8x8_t obmc_pred_mask = vsub_u8(mask_inverter, pred_mask);
+    const uint8x8_t pred_val = vld1_u8(pred);
+    const uint16x8_t weighted_pred = vmull_u8(pred_mask, pred_val);
+    const uint8x8_t obmc_pred_val = vld1_u8(obmc_pred);
+    const uint8x8_t result =
+        vrshrn_n_u16(vmlal_u8(weighted_pred, obmc_pred_mask, obmc_pred_val), 6);
+
+    vst1_u8(pred, result);
+    pred += prediction_stride;
+    obmc_pred += obmc_prediction_stride;
+  } while (++y != compute_height);
+}
+
+void OverlapBlendFromTop_NEON(void* const prediction,
+                              const ptrdiff_t prediction_stride,
+                              const int width, const int height,
+                              const void* const obmc_prediction,
+                              const ptrdiff_t obmc_prediction_stride) {
+  auto* pred = static_cast<uint8_t*>(prediction);
+  const auto* obmc_pred = static_cast<const uint8_t*>(obmc_prediction);
+
+  if (width == 2) {
+    OverlapBlend2xH_NEON<false>(pred, prediction_stride, height, obmc_pred,
+                                obmc_prediction_stride);
+    return;
+  }
+  if (width == 4) {
+    OverlapBlendFromTop4xH_NEON(pred, prediction_stride, height, obmc_pred,
+                                obmc_prediction_stride);
+    return;
+  }
+
+  if (width == 8) {
+    OverlapBlendFromTop8xH_NEON(pred, prediction_stride, height, obmc_pred,
+                                obmc_prediction_stride);
+    return;
+  }
+
+  const uint8_t* mask = kObmcMask + height - 2;
+  const uint8x8_t mask_inverter = vdup_n_u8(64);
+  // Stop when mask value becomes 64. This is inferred for 4xH.
+  const int compute_height = height - (height >> 2);
+  int y = 0;
+  do {
+    const uint8x8_t pred_mask = vdup_n_u8(mask[y]);
+    // 64 - mask
+    const uint8x8_t obmc_pred_mask = vsub_u8(mask_inverter, pred_mask);
+    int x = 0;
+    do {
+      const uint8x16_t pred_val = vld1q_u8(pred + x);
+      const uint8x16_t obmc_pred_val = vld1q_u8(obmc_pred + x);
+      const uint16x8_t weighted_pred_lo =
+          vmull_u8(pred_mask, vget_low_u8(pred_val));
+      const uint8x8_t result_lo =
+          vrshrn_n_u16(vmlal_u8(weighted_pred_lo, obmc_pred_mask,
+                                vget_low_u8(obmc_pred_val)),
+                       6);
+      const uint16x8_t weighted_pred_hi =
+          vmull_u8(pred_mask, vget_high_u8(pred_val));
+      const uint8x8_t result_hi =
+          vrshrn_n_u16(vmlal_u8(weighted_pred_hi, obmc_pred_mask,
+                                vget_high_u8(obmc_pred_val)),
+                       6);
+      vst1q_u8(pred + x, vcombine_u8(result_lo, result_hi));
+
+      x += 16;
+    } while (x < width);
+    pred += prediction_stride;
+    obmc_pred += obmc_prediction_stride;
+  } while (++y < compute_height);
+}
+
+void Init8bpp() {
+  Dsp* const dsp = dsp_internal::GetWritableDspTable(8);
+  assert(dsp != nullptr);
+  dsp->obmc_blend[kObmcDirectionVertical] = OverlapBlendFromTop_NEON;
+  dsp->obmc_blend[kObmcDirectionHorizontal] = OverlapBlendFromLeft_NEON;
+}
+
+}  // namespace
+
+void ObmcInit_NEON() { Init8bpp(); }
+
+}  // namespace dsp
+}  // namespace libgav1
+
+#else   // !LIBGAV1_ENABLE_NEON
+
+namespace libgav1 {
+namespace dsp {
+
+void ObmcInit_NEON() {}
+
+}  // namespace dsp
+}  // namespace libgav1
+#endif  // LIBGAV1_ENABLE_NEON
diff --git a/libgav1/src/dsp/arm/obmc_neon.h b/libgav1/src/dsp/arm/obmc_neon.h
new file mode 100644
index 0000000..77a9fa8
--- /dev/null
+++ b/libgav1/src/dsp/arm/obmc_neon.h
@@ -0,0 +1,22 @@
+#ifndef LIBGAV1_SRC_DSP_ARM_OBMC_NEON_H_
+#define LIBGAV1_SRC_DSP_ARM_OBMC_NEON_H_
+
+#include "src/dsp/cpu.h"
+#include "src/dsp/dsp.h"
+
+namespace libgav1 {
+namespace dsp {
+
+// Initializes Dsp::obmc_blend. This function is not thread-safe.
+void ObmcInit_NEON();
+
+}  // namespace dsp
+}  // namespace libgav1
+
+// If NEON is enabled, signal the NEON implementation should be used.
+#if LIBGAV1_ENABLE_NEON
+#define LIBGAV1_Dsp8bpp_ObmcVertical LIBGAV1_DSP_NEON
+#define LIBGAV1_Dsp8bpp_ObmcHorizontal LIBGAV1_DSP_NEON
+#endif  // LIBGAV1_ENABLE_NEON
+
+#endif  // LIBGAV1_SRC_DSP_ARM_OBMC_NEON_H_
diff --git a/libgav1/src/dsp/arm/warp_neon.cc b/libgav1/src/dsp/arm/warp_neon.cc
new file mode 100644
index 0000000..7ab3d27
--- /dev/null
+++ b/libgav1/src/dsp/arm/warp_neon.cc
@@ -0,0 +1,321 @@
+#include "src/dsp/dsp.h"
+#include "src/dsp/warp.h"
+
+#if LIBGAV1_ENABLE_NEON
+
+#include <arm_neon.h>
+
+#include <algorithm>
+#include <cassert>
+#include <cstddef>
+#include <cstdint>
+#include <cstdlib>
+
+#include "src/dsp/arm/common_neon.h"
+#include "src/dsp/constants.h"
+#include "src/utils/common.h"
+#include "src/utils/constants.h"
+
+namespace libgav1 {
+namespace dsp {
+namespace low_bitdepth {
+namespace {
+
+// Number of extra bits of precision in warped filtering.
+constexpr int kWarpedDiffPrecisionBits = 10;
+
+void Warp_NEON(const void* const source, const ptrdiff_t source_stride,
+               const int source_width, const int source_height,
+               const int* const warp_params, const int subsampling_x,
+               const int subsampling_y, const uint8_t inter_round_bits_vertical,
+               const int block_start_x, const int block_start_y,
+               const int block_width, const int block_height,
+               const int16_t alpha, const int16_t beta, const int16_t gamma,
+               const int16_t delta, uint16_t* dest,
+               const ptrdiff_t dest_stride) {
+  constexpr int bitdepth = 8;
+  // Intermediate_result is the output of the horizontal filtering and rounding.
+  // The range is within 13 (= bitdepth + kFilterBits + 1 -
+  // kInterRoundBitsHorizontal) bits (unsigned). We use the signed int16_t type
+  // so that we can multiply it by kWarpedFilters (which has signed values)
+  // using vmlal_s16().
+  int16_t intermediate_result[15][8];  // 15 rows, 8 columns.
+  const int horizontal_offset = 1 << (bitdepth + kFilterBits - 1);
+  const int vertical_offset =
+      1 << (bitdepth + 2 * kFilterBits - kInterRoundBitsHorizontal);
+  const auto* const src = static_cast<const uint8_t*>(source);
+
+  assert(block_width >= 8);
+  assert(block_height >= 8);
+
+  const uint8x16_t index_vec = vcombine_u8(vcreate_u8(0x0706050403020100),
+                                           vcreate_u8(0x0f0e0d0c0b0a0908));
+
+  // Warp process applies for each 8x8 block (or smaller).
+  int start_y = block_start_y;
+  do {
+    int start_x = block_start_x;
+    do {
+      const int src_x = (start_x + 4) << subsampling_x;
+      const int src_y = (start_y + 4) << subsampling_y;
+      const int dst_x =
+          src_x * warp_params[2] + src_y * warp_params[3] + warp_params[0];
+      const int dst_y =
+          src_x * warp_params[4] + src_y * warp_params[5] + warp_params[1];
+      const int x4 = dst_x >> subsampling_x;
+      const int y4 = dst_y >> subsampling_y;
+      const int ix4 = x4 >> kWarpedModelPrecisionBits;
+      const int iy4 = y4 >> kWarpedModelPrecisionBits;
+
+      // Horizontal filter.
+      int sx4 = (x4 & ((1 << kWarpedModelPrecisionBits) - 1)) - beta * 7;
+      for (int y = -7; y < 8; ++y) {
+        // TODO(chengchen):
+        // Because of warping, the index could be out of frame boundary. Thus
+        // clip is needed. However, can we remove or reduce usage of clip?
+        // Besides, special cases exist, for example,
+        // if iy4 - 7 >= source_height or iy4 + 7 < 0, there's no need to do the
+        // filtering.
+        const int row = Clip3(iy4 + y, 0, source_height - 1);
+        const uint8_t* const src_row = src + row * source_stride;
+        // Check for two simple special cases.
+        if (ix4 - 7 >= source_width - 1) {
+          // Every sample is equal to src_row[source_width - 1]. Since the sum
+          // of the warped filter coefficients is 128 (= 2^7), the filtering is
+          // equivalent to multiplying src_row[source_width - 1] by 128.
+          const int16_t s =
+              (horizontal_offset >> kInterRoundBitsHorizontal) +
+              (src_row[source_width - 1] << (7 - kInterRoundBitsHorizontal));
+          const int16x8_t sum = vdupq_n_s16(s);
+          vst1q_s16(&intermediate_result[y + 7][0], sum);
+          sx4 += beta;
+          continue;
+        }
+        if (ix4 + 7 <= 0) {
+          // Every sample is equal to src_row[0]. Since the sum of the warped
+          // filter coefficients is 128 (= 2^7), the filtering is equivalent to
+          // multiplying src_row[0] by 128.
+          const int16_t s = (horizontal_offset >> kInterRoundBitsHorizontal) +
+                            (src_row[0] << (7 - kInterRoundBitsHorizontal));
+          const int16x8_t sum = vdupq_n_s16(s);
+          vst1q_s16(&intermediate_result[y + 7][0], sum);
+          sx4 += beta;
+          continue;
+        }
+        // Read 15 samples from &src_row[ix4 - 7]. The 16th sample is also
+        // read but is ignored.
+        //
+        // NOTE: This may read up to 13 bytes before src_row[0] or up to 14
+        // bytes after src_row[source_width - 1]. There must be enough padding
+        // before and after the |source| buffer.
+        static_assert(kBorderPixels >= 14, "");
+        uint8x16_t src_row_u8 = vld1q_u8(&src_row[ix4 - 7]);
+        // If clipping is needed, duplicate the border samples.
+        //
+        // Here is the correspondence between the index for the src_row
+        // buffer and the index for the src_row_u8 vector:
+        //
+        // src_row index    : (ix4 - 7) (ix4 - 6) ... (ix4 + 6) (ix4 + 7)
+        // src_row_u8 index :     0         1     ...     13        14
+        if (ix4 - 7 < 0) {
+          const int out_of_boundary_left = -(ix4 - 7);
+          const uint8x16_t cmp_vec = vdupq_n_u8(out_of_boundary_left);
+          const uint8x16_t vec_dup = vdupq_n_u8(src_row[0]);
+          const uint8x16_t mask_val = vcltq_u8(index_vec, cmp_vec);
+          src_row_u8 = vbslq_u8(mask_val, vec_dup, src_row_u8);
+        }
+        if (ix4 + 7 > source_width - 1) {
+          const int out_of_boundary_right = ix4 + 8 - source_width;
+          const uint8x16_t cmp_vec = vdupq_n_u8(14 - out_of_boundary_right);
+          const uint8x16_t vec_dup = vdupq_n_u8(src_row[source_width - 1]);
+          const uint8x16_t mask_val = vcgtq_u8(index_vec, cmp_vec);
+          src_row_u8 = vbslq_u8(mask_val, vec_dup, src_row_u8);
+        }
+        const int16x8_t src_row_low_s16 =
+            vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(src_row_u8)));
+        const int16x8_t src_row_high_s16 =
+            vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(src_row_u8)));
+        int sx = sx4 - MultiplyBy4(alpha);
+        int16x8_t filter[8];
+        for (int x = 0; x < 8; ++x) {
+          const int offset =
+              RightShiftWithRounding(sx, kWarpedDiffPrecisionBits) +
+              kWarpedPixelPrecisionShifts;
+          filter[x] = vld1q_s16(kWarpedFilters[offset]);
+          sx += alpha;
+        }
+        Transpose8x8(&filter[0], &filter[1], &filter[2], &filter[3], &filter[4],
+                     &filter[5], &filter[6], &filter[7]);
+        // For 8 bit, the range of sum is within uint16_t, if we add a
+        // horizontal offset. horizontal_offset guarantees sum is nonnegative.
+        //
+        // Proof:
+        // Given that the minimum (most negative) sum of the negative filter
+        // coefficients is -47 and the maximum sum of the positive filter
+        // coefficients is 175, the range of the horizontal filter output is
+        //   -47 * 255 <= output <= 175 * 255
+        // Since -2^14 < -47 * 255, adding -2^14 (= horizontal_offset) to the
+        // horizontal filter output produces a positive value:
+        //   0 < output + 2^14 <= 175 * 255 + 2^14
+        // The final rounding right shift by 3 (= kInterRoundBitsHorizontal)
+        // bits adds 2^2 to the sum:
+        //   0 < output + 2^14 + 2^2 <= 175 * 255 + 2^14 + 2^2 = 61013
+        // Since 61013 < 2^16, the final sum (right before the right shift by 3
+        // bits) will not overflow uint16_t. In addition, the value after the
+        // right shift by 3 bits is in the following range:
+        //   0 <= intermediate_result[y][x] < 2^13
+        // This property is used in determining the range of the vertical
+        // filtering output. [End of proof.]
+        //
+        // We can do signed int16_t arithmetic and just treat the final result
+        // as uint16_t when we shift it right.
+        int16x8_t sum = vdupq_n_s16(horizontal_offset);
+        // Unrolled k = 0..7 loop. We need to manually unroll the loop because
+        // the third argument (an index value) to vextq_s16() must be a
+        // constant (immediate).
+        // k = 0.
+        int16x8_t src_row_v_s16 = src_row_low_s16;
+        sum = vmlaq_s16(sum, filter[0], src_row_v_s16);
+        // k = 1.
+        src_row_v_s16 = vextq_s16(src_row_low_s16, src_row_high_s16, 1);
+        sum = vmlaq_s16(sum, filter[1], src_row_v_s16);
+        // k = 2.
+        src_row_v_s16 = vextq_s16(src_row_low_s16, src_row_high_s16, 2);
+        sum = vmlaq_s16(sum, filter[2], src_row_v_s16);
+        // k = 3.
+        src_row_v_s16 = vextq_s16(src_row_low_s16, src_row_high_s16, 3);
+        sum = vmlaq_s16(sum, filter[3], src_row_v_s16);
+        // k = 4.
+        src_row_v_s16 = vextq_s16(src_row_low_s16, src_row_high_s16, 4);
+        sum = vmlaq_s16(sum, filter[4], src_row_v_s16);
+        // k = 5.
+        src_row_v_s16 = vextq_s16(src_row_low_s16, src_row_high_s16, 5);
+        sum = vmlaq_s16(sum, filter[5], src_row_v_s16);
+        // k = 6.
+        src_row_v_s16 = vextq_s16(src_row_low_s16, src_row_high_s16, 6);
+        sum = vmlaq_s16(sum, filter[6], src_row_v_s16);
+        // k = 7.
+        src_row_v_s16 = vextq_s16(src_row_low_s16, src_row_high_s16, 7);
+        sum = vmlaq_s16(sum, filter[7], src_row_v_s16);
+        // End of unrolled k = 0..7 loop.
+        // Treat sum as unsigned for the right shift.
+        sum = vreinterpretq_s16_u16(vrshrq_n_u16(vreinterpretq_u16_s16(sum),
+                                                 kInterRoundBitsHorizontal));
+        vst1q_s16(&intermediate_result[y + 7][0], sum);
+        sx4 += beta;
+      }
+
+      // Vertical filter.
+      uint16_t* dst_row = dest + start_x - block_start_x;
+      int sy4 =
+          (y4 & ((1 << kWarpedModelPrecisionBits) - 1)) - MultiplyBy4(delta);
+      for (int y = -4; y < 4; ++y) {
+        int sy = sy4 - MultiplyBy4(gamma);
+        int16x8_t filter[8];
+        for (int x = 0; x < 8; ++x) {
+          const int offset =
+              RightShiftWithRounding(sy, kWarpedDiffPrecisionBits) +
+              kWarpedPixelPrecisionShifts;
+          filter[x] = vld1q_s16(kWarpedFilters[offset]);
+          sy += gamma;
+        }
+        Transpose8x8(&filter[0], &filter[1], &filter[2], &filter[3], &filter[4],
+                     &filter[5], &filter[6], &filter[7]);
+        // Similar to horizontal_offset, vertical_offset guarantees sum before
+        // shifting is nonnegative.
+        //
+        // Proof:
+        // The range of an entry in intermediate_result is
+        //   0 <= intermediate_result[y][x] < 2^13
+        // The range of the vertical filter output is
+        //   -47 * 2^13 < output < 175 * 2^13
+        // Since -2^19 < -47 * 2^13, adding -2^19 (= vertical_offset) to the
+        // vertical filter output produces a positive value:
+        //   0 < output + 2^19 < 175 * 2^13 + 2^19
+        // The final rounding right shift by either 7 or 11 bits adds at most
+        // 2^10 to the sum:
+        //   0 < output + 2^19 + rounding < 175 * 2^13 + 2^19 + 2^10 = 1958912
+        // Since 1958912 = 0x1DE400 < 2^22, shifting it right by 7 or 11 bits
+        // brings the value under 2^15, which fits in uint16_t.
+        int32x4_t sum_low = vdupq_n_s32(vertical_offset);
+        int32x4_t sum_high = sum_low;
+        for (int k = 0; k < 8; ++k) {
+          const int16x8_t intermediate =
+              vld1q_s16(intermediate_result[y + 4 + k]);
+          sum_low = vmlal_s16(sum_low, vget_low_s16(filter[k]),
+                              vget_low_s16(intermediate));
+          sum_high = vmlal_s16(sum_high, vget_high_s16(filter[k]),
+                               vget_high_s16(intermediate));
+        }
+        assert(inter_round_bits_vertical == 7 ||
+               inter_round_bits_vertical == 11);
+        // Since inter_round_bits_vertical can be 7 or 11, and all the narrowing
+        // shift intrinsics require the shift argument to be a constant
+        // (immediate), we have two options:
+        // 1. Call a non-narrowing shift, followed by a narrowing extract.
+        // 2. Call a narrowing shift (with a constant shift of 7 or 11) in an
+        //    if-else statement.
+#if defined(__aarch64__)
+        // This version is slightly faster for arm64 (1106 ms vs 1112 ms).
+        // This version is slower for 32-bit arm (1235 ms vs 1149 ms).
+        const int32x4_t shift = vdupq_n_s32(-inter_round_bits_vertical);
+        const uint32x4_t sum_low_shifted =
+            vrshlq_u32(vreinterpretq_u32_s32(sum_low), shift);
+        const uint32x4_t sum_high_shifted =
+            vrshlq_u32(vreinterpretq_u32_s32(sum_high), shift);
+        const uint16x4_t sum_low_16 = vmovn_u32(sum_low_shifted);
+        const uint16x4_t sum_high_16 = vmovn_u32(sum_high_shifted);
+#else   // !defined(__aarch64__)
+        // This version is faster for 32-bit arm.
+        // This version is slightly slower for arm64.
+        uint16x4_t sum_low_16;
+        uint16x4_t sum_high_16;
+        if (inter_round_bits_vertical == 7) {
+          sum_low_16 = vrshrn_n_u32(vreinterpretq_u32_s32(sum_low), 7);
+          sum_high_16 = vrshrn_n_u32(vreinterpretq_u32_s32(sum_high), 7);
+        } else {
+          sum_low_16 = vrshrn_n_u32(vreinterpretq_u32_s32(sum_low), 11);
+          sum_high_16 = vrshrn_n_u32(vreinterpretq_u32_s32(sum_high), 11);
+        }
+#endif  // defined(__aarch64__)
+        // vst1q_u16 can also be used:
+        //   vst1q_u16(dst_row, vcombine_u16(sum_low_16, sum_high_16));
+        // But it is slightly slower for arm64 (the same speed for 32-bit arm).
+        //
+        // vst1_u16_x2 could be used, but it is also slightly slower for arm64
+        // and causes a bus error for 32-bit arm. Also, it is not supported by
+        // gcc 7.2.0.
+        vst1_u16(dst_row, sum_low_16);
+        vst1_u16(dst_row + 4, sum_high_16);
+        dst_row += dest_stride;
+        sy4 += delta;
+      }
+      start_x += 8;
+    } while (start_x < block_start_x + block_width);
+    dest += 8 * dest_stride;
+    start_y += 8;
+  } while (start_y < block_start_y + block_height);
+}
+
+void Init8bpp() {
+  Dsp* const dsp = dsp_internal::GetWritableDspTable(8);
+  assert(dsp != nullptr);
+  dsp->warp = Warp_NEON;
+}
+
+}  // namespace
+}  // namespace low_bitdepth
+
+void WarpInit_NEON() { low_bitdepth::Init8bpp(); }
+
+}  // namespace dsp
+}  // namespace libgav1
+#else   // !LIBGAV1_ENABLE_NEON
+namespace libgav1 {
+namespace dsp {
+
+void WarpInit_NEON() {}
+
+}  // namespace dsp
+}  // namespace libgav1
+#endif  // LIBGAV1_ENABLE_NEON
diff --git a/libgav1/src/dsp/arm/warp_neon.h b/libgav1/src/dsp/arm/warp_neon.h
new file mode 100644
index 0000000..c5e1bc0
--- /dev/null
+++ b/libgav1/src/dsp/arm/warp_neon.h
@@ -0,0 +1,20 @@
+#ifndef LIBGAV1_SRC_DSP_ARM_WARP_NEON_H_
+#define LIBGAV1_SRC_DSP_ARM_WARP_NEON_H_
+
+#include "src/dsp/cpu.h"
+#include "src/dsp/dsp.h"
+
+namespace libgav1 {
+namespace dsp {
+
+// Initializes Dsp::warp. This function is not thread-safe.
+void WarpInit_NEON();
+
+}  // namespace dsp
+}  // namespace libgav1
+
+#if LIBGAV1_ENABLE_NEON
+#define LIBGAV1_Dsp8bpp_Warp LIBGAV1_DSP_NEON
+#endif  // LIBGAV1_ENABLE_NEON
+
+#endif  // LIBGAV1_SRC_DSP_ARM_WARP_NEON_H_
diff --git a/libgav1/src/dsp/average_blend.cc b/libgav1/src/dsp/average_blend.cc
index 4b2b7d7..a5de6d3 100644
--- a/libgav1/src/dsp/average_blend.cc
+++ b/libgav1/src/dsp/average_blend.cc
@@ -12,26 +12,29 @@
 namespace {
 
 template <int bitdepth, typename Pixel>
-void AverageBlending_C(const uint16_t* prediction_0,
-                       const ptrdiff_t prediction_stride_0,
-                       const uint16_t* prediction_1,
-                       const ptrdiff_t prediction_stride_1,
-                       const int inter_post_round_bit, const int width,
-                       const int height, void* const dest,
-                       const ptrdiff_t dest_stride) {
+void AverageBlend_C(const uint16_t* prediction_0,
+                    const ptrdiff_t prediction_stride_0,
+                    const uint16_t* prediction_1,
+                    const ptrdiff_t prediction_stride_1, const int width,
+                    const int height, void* const dest,
+                    const ptrdiff_t dest_stride) {
   // An offset to cancel offsets used in compound predictor generation that
   // make intermediate computations non negative.
-  const int compound_round_offset =
-      (1 << (bitdepth + 4)) + (1 << (bitdepth + 3));
+  constexpr int compound_round_offset =
+      (2 << (bitdepth + 4)) + (2 << (bitdepth + 3));
+  // 7.11.3.2 Rounding variables derivation process
+  //   2 * FILTER_BITS(7) - (InterRound0(3|5) + InterRound1(7))
+  constexpr int inter_post_round_bits = (bitdepth == 12) ? 2 : 4;
   auto* dst = static_cast<Pixel*>(dest);
   const ptrdiff_t dst_stride = dest_stride / sizeof(Pixel);
 
   for (int y = 0; y < height; ++y) {
     for (int x = 0; x < width; ++x) {
-      int res = (prediction_0[x] + prediction_1[x]) >> 1;
+      // prediction range: 8bpp: [0, 15471] 10bpp: [0, 61983] 12bpp: [0, 62007].
+      int res = prediction_0[x] + prediction_1[x];
       res -= compound_round_offset;
       dst[x] = static_cast<Pixel>(
-          Clip3(RightShiftWithRounding(res, inter_post_round_bit), 0,
+          Clip3(RightShiftWithRounding(res, inter_post_round_bits + 1), 0,
                 (1 << bitdepth) - 1));
     }
     dst += dst_stride;
@@ -43,14 +46,30 @@
 void Init8bpp() {
   Dsp* const dsp = dsp_internal::GetWritableDspTable(8);
   assert(dsp != nullptr);
-  dsp->average_blend = AverageBlending_C<8, uint8_t>;
+#if LIBGAV1_ENABLE_ALL_DSP_FUNCTIONS
+  dsp->average_blend = AverageBlend_C<8, uint8_t>;
+#else  // !LIBGAV1_ENABLE_ALL_DSP_FUNCTIONS
+  static_cast<void>(dsp);
+#ifndef LIBGAV1_Dsp8bpp_AverageBlend
+  dsp->average_blend = AverageBlend_C<8, uint8_t>;
+#endif
+#endif  // LIBGAV1_ENABLE_ALL_DSP_FUNCTIONS
 }
 
 #if LIBGAV1_MAX_BITDEPTH >= 10
 void Init10bpp() {
   Dsp* const dsp = dsp_internal::GetWritableDspTable(10);
   assert(dsp != nullptr);
-  dsp->average_blend = AverageBlending_C<10, uint16_t>;
+#if LIBGAV1_ENABLE_ALL_DSP_FUNCTIONS
+#ifndef LIBGAV1_Dsp10bpp_AverageBlend
+  dsp->average_blend = AverageBlend_C<10, uint16_t>;
+#endif
+#else  // !LIBGAV1_ENABLE_ALL_DSP_FUNCTIONS
+  static_cast<void>(dsp);
+#ifndef LIBGAV1_Dsp10bpp_AverageBlend
+  dsp->average_blend = AverageBlend_C<10, uint16_t>;
+#endif
+#endif  // LIBGAV1_ENABLE_ALL_DSP_FUNCTIONS
 }
 #endif
 
diff --git a/libgav1/src/dsp/average_blend.h b/libgav1/src/dsp/average_blend.h
index e126d39..f91a102 100644
--- a/libgav1/src/dsp/average_blend.h
+++ b/libgav1/src/dsp/average_blend.h
@@ -1,11 +1,28 @@
 #ifndef LIBGAV1_SRC_DSP_AVERAGE_BLEND_H_
 #define LIBGAV1_SRC_DSP_AVERAGE_BLEND_H_
 
+// Pull in LIBGAV1_DspXXX defines representing the implementation status
+// of each function. The resulting value of each can be used by each module to
+// determine whether an implementation is needed at compile time.
+// IWYU pragma: begin_exports
+
+// ARM:
+#include "src/dsp/arm/average_blend_neon.h"
+
+// x86:
+// Note includes should be sorted in logical order avx2/avx/sse4, etc.
+// The order of includes is important as each tests for a superior version
+// before setting the base.
+// clang-format off
+#include "src/dsp/x86/average_blend_sse4.h"
+// clang-format on
+
+// IWYU pragma: end_exports
+
 namespace libgav1 {
 namespace dsp {
 
-// Initializes Dsp::average_blend.
-// This function is not thread-safe.
+// Initializes Dsp::average_blend. This function is not thread-safe.
 void AverageBlendInit_C();
 
 }  // namespace dsp
diff --git a/libgav1/src/dsp/cdef.cc b/libgav1/src/dsp/cdef.cc
index e3095d5..e412342 100644
--- a/libgav1/src/dsp/cdef.cc
+++ b/libgav1/src/dsp/cdef.cc
@@ -13,6 +13,8 @@
 namespace dsp {
 namespace {
 
+constexpr uint16_t kCdefLargeValue = 30000;
+
 constexpr int16_t kDivisionTable[] = {0,   840, 420, 280, 210,
                                       168, 140, 120, 105};
 
@@ -32,14 +34,6 @@
          Clip3(threshold - (std::abs(diff) >> damping), 0, std::abs(diff));
 }
 
-// 5.11.52.
-bool InsideFrame(int x, int y, int subsampling_x, int subsampling_y,
-                 int rows4x4, int columns4x4) {
-  const int row = DivideBy4(LeftShift(y, subsampling_y));
-  const int column = DivideBy4(LeftShift(x, subsampling_x));
-  return row >= 0 && row < rows4x4 && column >= 0 && column < columns4x4;
-}
-
 int32_t Square(int32_t x) { return x * x; }
 
 template <int bitdepth, typename Pixel>
@@ -102,53 +96,41 @@
   *variance = (best_cost - cost[(*direction + 4) & 7]) >> 10;
 }
 
+// Filters the source block. It doesn't check whether the candidate pixel is
+// inside the frame. However it requires the source input to be padded with a
+// constant large value if at the boundary. And the input should be uint16_t.
 template <int bitdepth, typename Pixel>
-void CdefFiltering_C(const void* const source, const ptrdiff_t source_stride,
-                     const int rows4x4, const int columns4x4, const int curr_x,
-                     const int curr_y, const int subsampling_x,
-                     const int subsampling_y, const int primary_strength,
-                     const int secondary_strength, const int damping,
-                     const int direction, void* const dest,
-                     const ptrdiff_t dest_stride) {
+void CdefFilter_C(const void* const source, const ptrdiff_t source_stride,
+                  const int rows4x4, const int columns4x4, const int curr_x,
+                  const int curr_y, const int subsampling_x,
+                  const int subsampling_y, const int primary_strength,
+                  const int secondary_strength, const int damping,
+                  const int direction, void* const dest,
+                  const ptrdiff_t dest_stride) {
   const int coeff_shift = bitdepth - 8;
   const int plane_width = MultiplyBy4(columns4x4) >> subsampling_x;
   const int plane_height = MultiplyBy4(rows4x4) >> subsampling_y;
   const int block_width = std::min(8 >> subsampling_x, plane_width - curr_x);
   const int block_height = std::min(8 >> subsampling_y, plane_height - curr_y);
-  const auto* src = static_cast<const Pixel*>(source);
-  const ptrdiff_t src_stride = source_stride / sizeof(Pixel);
+  const auto* src = static_cast<const uint16_t*>(source);
   auto* dst = static_cast<Pixel*>(dest);
   const ptrdiff_t dst_stride = dest_stride / sizeof(Pixel);
   for (int y = 0; y < block_height; ++y) {
     for (int x = 0; x < block_width; ++x) {
       int16_t sum = 0;
-      const Pixel pixel_value = src[x];
-      Pixel max_value = pixel_value;
-      Pixel min_value = pixel_value;
+      const uint16_t pixel_value = src[x];
+      uint16_t max_value = pixel_value;
+      uint16_t min_value = pixel_value;
       for (int k = 0; k < 2; ++k) {
         const int signs[] = {-1, 1};
         for (const int& sign : signs) {
           int dy = sign * kCdefDirections[direction][k][0];
           int dx = sign * kCdefDirections[direction][k][1];
-          int y0 = curr_y + y + dy;
-          int x0 = curr_x + x + dx;
-          // TODO(chengchen): Optimize cdef data fetching.
-          // Cdef needs to get pixel values from 3x3 neighborhood.
-          // It could happen that the target position is out of the frame.
-          // When it's out of frame, that pixel should not be taken into
-          // calculation.
-          // In libaom's implementation, borders are padded around the whole
-          // frame such that out of frame access gets a large value. The
-          // large value is defined as 30000. This implementation has a problem
-          // because 8-bit input can't represent 30000. It has to allocate a
-          // 16-bit frame buffer to set large values for the borders.
-          // In this implementation, we detect whether it's out of frame,
-          // which is not friendly for SIMD implementation.
-          // We can avoid the extra frame buffer by allocating a 16-bit block
-          // buffer, like the implementation of loop restoration.
-          if (InsideFrame(x0, y0, subsampling_x, subsampling_y, rows4x4,
-                          columns4x4)) {
-            const Pixel value = src[dy * src_stride + dx + x];
+          uint16_t value = src[dy * source_stride + dx + x];
+          // Note: the summation can ignore the condition check in SIMD
+          // implementation, because Constrain() will return 0 when
+          // value == kCdefLargeValue.
+          if (value != kCdefLargeValue) {
             sum += Constrain(value - pixel_value, primary_strength, damping) *
                    kPrimaryTaps[(primary_strength >> coeff_shift) & 1][k];
             max_value = std::max(value, max_value);
@@ -158,11 +140,10 @@
           for (const int& offset : offsets) {
             dy = sign * kCdefDirections[(direction + offset) & 7][k][0];
             dx = sign * kCdefDirections[(direction + offset) & 7][k][1];
-            y0 = curr_y + y + dy;
-            x0 = curr_x + x + dx;
-            if (InsideFrame(x0, y0, subsampling_x, subsampling_y, rows4x4,
-                            columns4x4)) {
-              const Pixel value = src[dy * src_stride + dx + x];
+            value = src[dy * source_stride + dx + x];
+            // Note: the summation can ignore the condition check in SIMD
+            // implementation.
+            if (value != kCdefLargeValue) {
               sum +=
                   Constrain(value - pixel_value, secondary_strength, damping) *
                   kSecondaryTaps[(primary_strength >> coeff_shift) & 1][k];
@@ -173,10 +154,10 @@
         }
       }
 
-      dst[x] = Clip3(pixel_value + ((8 + sum - (sum < 0)) >> 4), min_value,
-                     max_value);
+      dst[x] = static_cast<Pixel>(Clip3(
+          pixel_value + ((8 + sum - (sum < 0)) >> 4), min_value, max_value));
     }
-    src += src_stride;
+    src += source_stride;
     dst += dst_stride;
   }
 }
@@ -186,11 +167,14 @@
   assert(dsp != nullptr);
 #if LIBGAV1_ENABLE_ALL_DSP_FUNCTIONS
   dsp->cdef_direction = CdefDirection_C<8, uint8_t>;
-  dsp->cdef_filter = CdefFiltering_C<8, uint8_t>;
+  dsp->cdef_filter = CdefFilter_C<8, uint8_t>;
 #else  // !LIBGAV1_ENABLE_ALL_DSP_FUNCTIONS
+  static_cast<void>(dsp);
 #ifndef LIBGAV1_Dsp8bpp_CdefDirection
   dsp->cdef_direction = CdefDirection_C<8, uint8_t>;
-  dsp->cdef_filter = CdefFiltering_C<8, uint8_t>;
+#endif
+#ifndef LIBGAV1_Dsp8bpp_CdefFilter
+  dsp->cdef_filter = CdefFilter_C<8, uint8_t>;
 #endif
 #endif  // LIBGAV1_ENABLE_ALL_DSP_FUNCTIONS
 }
@@ -201,11 +185,14 @@
   assert(dsp != nullptr);
 #if LIBGAV1_ENABLE_ALL_DSP_FUNCTIONS
   dsp->cdef_direction = CdefDirection_C<10, uint16_t>;
-  dsp->cdef_filter = CdefFiltering_C<10, uint16_t>;
+  dsp->cdef_filter = CdefFilter_C<10, uint16_t>;
 #else  // !LIBGAV1_ENABLE_ALL_DSP_FUNCTIONS
+  static_cast<void>(dsp);
 #ifndef LIBGAV1_Dsp10bpp_CdefDirection
   dsp->cdef_direction = CdefDirection_C<10, uint16_t>;
-  dsp->cdef_filter = CdefFiltering_C<10, uint16_t>;
+#endif
+#ifndef LIBGAV1_Dsp10bpp_CdefFilter
+  dsp->cdef_filter = CdefFilter_C<10, uint16_t>;
 #endif
 #endif  // LIBGAV1_ENABLE_ALL_DSP_FUNCTIONS
 }
diff --git a/libgav1/src/dsp/cdef.h b/libgav1/src/dsp/cdef.h
index 5da4aba..3c1ab80 100644
--- a/libgav1/src/dsp/cdef.h
+++ b/libgav1/src/dsp/cdef.h
@@ -4,7 +4,8 @@
 namespace libgav1 {
 namespace dsp {
 
-// Initializes Dsp::cdef_direction. This function is not thread-safe.
+// Initializes Dsp::cdef_direction and cdef::filter. This function is not
+// thread-safe.
 void CdefInit_C();
 
 }  // namespace dsp
diff --git a/libgav1/src/dsp/common.h b/libgav1/src/dsp/common.h
index e9842a2..ddd67e4 100644
--- a/libgav1/src/dsp/common.h
+++ b/libgav1/src/dsp/common.h
@@ -6,6 +6,7 @@
 
 #include "src/dsp/constants.h"
 #include "src/utils/constants.h"
+#include "src/utils/memory.h"
 
 namespace libgav1 {
 
@@ -24,10 +25,10 @@
   static const int kVertical = 0;
   static const int kHorizontal = 1;
 
-  alignas(16) int16_t filter[2][kSubPixelTaps];
+  alignas(kMaxAlignment) int16_t filter[2][kSubPixelTaps];
 };
 
-struct RestorationUnitInfo {
+struct RestorationUnitInfo : public Allocable {
   LoopRestorationType type;
   SgrProjInfo sgr_proj_info;
   WienerInfo wiener_info;
@@ -42,7 +43,6 @@
   // For wiener filter.
   uint16_t* wiener_buffer;
   ptrdiff_t wiener_buffer_stride;
-  int inter_round_bits[2];
 };
 
 // Section 6.8.20.
diff --git a/libgav1/src/dsp/constants.h b/libgav1/src/dsp/constants.h
index f042150..0889f8f 100644
--- a/libgav1/src/dsp/constants.h
+++ b/libgav1/src/dsp/constants.h
@@ -15,6 +15,11 @@
   // 2^kSmoothWeightScale.
   kSmoothWeightScale = 8,
   kCflLumaBufferStride = 32,
+  // InterRound0, Section 7.11.3.2.
+  kInterRoundBitsHorizontal = 3,  // 8 & 10-bit.
+  kInterRoundBitsHorizontal12bpp = 5,
+  kInterRoundBitsVertical = 11,  // 8 & 10-bit, single prediction.
+  kInterRoundBitsVertical12bpp = 9,
 };  // anonymous enum
 
 extern const int8_t kFilterIntraTaps[kNumFilterIntraPredictors][8][8];
diff --git a/libgav1/src/dsp/convolve.cc b/libgav1/src/dsp/convolve.cc
index 0fdebb7..8c409c8 100644
--- a/libgav1/src/dsp/convolve.cc
+++ b/libgav1/src/dsp/convolve.cc
@@ -33,12 +33,15 @@
 }
 
 template <int bitdepth, typename Pixel>
-void Convolve2DScaleSingle_C(
+void ConvolveScale2D_C(
     const void* const reference, const ptrdiff_t reference_stride,
     const int horizontal_filter_index, const int vertical_filter_index,
-    const uint8_t inter_round_bits[2], const int subpixel_x,
+    const uint8_t inter_round_bits_vertical, const int subpixel_x,
     const int subpixel_y, const int step_x, const int step_y, const int width,
     const int height, void* prediction, const ptrdiff_t pred_stride) {
+  constexpr int kRoundBitsHorizontal = (bitdepth == 12)
+                                           ? kInterRoundBitsHorizontal12bpp
+                                           : kInterRoundBitsHorizontal;
   const int intermediate_height =
       (((height - 1) * step_y + (1 << kScaleSubPixelBits) - 1) >>
        kScaleSubPixelBits) +
@@ -76,7 +79,7 @@
       }
       assert(sum >= 0 && sum < (1 << (bitdepth + kFilterBits + 1)));
       intermediate[x] = static_cast<int16_t>(
-          RightShiftWithRounding(sum, inter_round_bits[0]));
+          RightShiftWithRounding(sum, kRoundBitsHorizontal));
     }
     src += src_stride;
     intermediate += intermediate_stride;
@@ -84,7 +87,7 @@
   // Vertical filter.
   filter_index = GetFilterIndex(vertical_filter_index, height);
   intermediate = intermediate_result;
-  const int offset_bits = bitdepth + 2 * kFilterBits - inter_round_bits[0];
+  const int offset_bits = bitdepth + 2 * kFilterBits - kRoundBitsHorizontal;
   for (int y = 0, p = subpixel_y & 1023; y < height; ++y, p += step_y) {
     const int filter_id = (p >> 6) & kSubPixelMask;
     for (int x = 0; x < width; ++x) {
@@ -98,7 +101,7 @@
       }
       assert(sum >= 0 && sum < (1 << (offset_bits + 2)));
       dest[x] = static_cast<Pixel>(
-          Clip3(RightShiftWithRounding(sum, inter_round_bits[1]) -
+          Clip3(RightShiftWithRounding(sum, inter_round_bits_vertical) -
                     single_round_offset,
                 0, max_pixel_value));
     }
@@ -107,14 +110,15 @@
 }
 
 template <int bitdepth, typename Pixel>
-void Convolve2DScale_C(const void* const reference,
-                       const ptrdiff_t reference_stride,
-                       const int horizontal_filter_index,
-                       const int vertical_filter_index,
-                       const uint8_t inter_round_bits[2], const int subpixel_x,
-                       const int subpixel_y, const int step_x, const int step_y,
-                       const int width, const int height,
-                       void* prediction, const ptrdiff_t pred_stride) {
+void ConvolveCompoundScale2D_C(
+    const void* const reference, const ptrdiff_t reference_stride,
+    const int horizontal_filter_index, const int vertical_filter_index,
+    const uint8_t inter_round_bits_vertical, const int subpixel_x,
+    const int subpixel_y, const int step_x, const int step_y, const int width,
+    const int height, void* prediction, const ptrdiff_t pred_stride) {
+  constexpr int kRoundBitsHorizontal = (bitdepth == 12)
+                                           ? kInterRoundBitsHorizontal12bpp
+                                           : kInterRoundBitsHorizontal;
   const int intermediate_height =
       (((height - 1) * step_y + (1 << kScaleSubPixelBits) - 1) >>
        kScaleSubPixelBits) +
@@ -149,7 +153,7 @@
       }
       assert(sum >= 0 && sum < (1 << (bitdepth + kFilterBits + 1)));
       intermediate[x] = static_cast<int16_t>(
-          RightShiftWithRounding(sum, inter_round_bits[0]));
+          RightShiftWithRounding(sum, kRoundBitsHorizontal));
     }
     src += src_stride;
     intermediate += intermediate_stride;
@@ -157,7 +161,7 @@
   // Vertical filter.
   filter_index = GetFilterIndex(vertical_filter_index, height);
   intermediate = intermediate_result;
-  const int offset_bits = bitdepth + 2 * kFilterBits - inter_round_bits[0];
+  const int offset_bits = bitdepth + 2 * kFilterBits - kRoundBitsHorizontal;
   for (int y = 0, p = subpixel_y & 1023; y < height; ++y, p += step_y) {
     const int filter_id = (p >> 6) & kSubPixelMask;
     for (int x = 0; x < width; ++x) {
@@ -171,25 +175,30 @@
       }
       assert(sum >= 0 && sum < (1 << (offset_bits + 2)));
       dest[x] = static_cast<uint16_t>(
-          RightShiftWithRounding(sum, inter_round_bits[1]));
+          RightShiftWithRounding(sum, inter_round_bits_vertical));
     }
     dest += pred_stride;
   }
 }
 
 template <int bitdepth, typename Pixel>
-void Convolve2D_C(const void* const reference, const ptrdiff_t reference_stride,
-                  const int horizontal_filter_index,
-                  const int vertical_filter_index,
-                  const uint8_t inter_round_bits[2], const int subpixel_x,
-                  const int subpixel_y, const int /*step_x*/,
-                  const int /*step_y*/, const int width, const int height,
-                  void* prediction, const ptrdiff_t pred_stride) {
+void ConvolveCompound2D_C(const void* const reference,
+                          const ptrdiff_t reference_stride,
+                          const int horizontal_filter_index,
+                          const int vertical_filter_index,
+                          const uint8_t inter_round_bits_vertical,
+                          const int subpixel_x, const int subpixel_y,
+                          const int /*step_x*/, const int /*step_y*/,
+                          const int width, const int height, void* prediction,
+                          const ptrdiff_t pred_stride) {
+  constexpr int kRoundBitsHorizontal = (bitdepth == 12)
+                                           ? kInterRoundBitsHorizontal12bpp
+                                           : kInterRoundBitsHorizontal;
   const int intermediate_height = height + kSubPixelTaps - 1;
   // The output of the horizontal filter, i.e. the intermediate_result, is
   // guaranteed to fit in int16_t.
   int16_t intermediate_result[kMaxSuperBlockSizeInPixels *
-                              (2 * kMaxSuperBlockSizeInPixels + 8)];
+                              (kMaxSuperBlockSizeInPixels + kSubPixelTaps - 1)];
   const int intermediate_stride = kMaxSuperBlockSizeInPixels;
 
   // Horizontal filter.
@@ -213,7 +222,7 @@
       }
       assert(sum >= 0 && sum < (1 << (bitdepth + kFilterBits + 1)));
       intermediate[x] = static_cast<int16_t>(
-          RightShiftWithRounding(sum, inter_round_bits[0]));
+          RightShiftWithRounding(sum, kRoundBitsHorizontal));
     }
     src += src_stride;
     intermediate += intermediate_stride;
@@ -222,7 +231,7 @@
   filter_index = GetFilterIndex(vertical_filter_index, height);
   intermediate = intermediate_result;
   filter_id = ((subpixel_y & 1023) >> 6) & kSubPixelMask;
-  const int offset_bits = bitdepth + 2 * kFilterBits - inter_round_bits[0];
+  const int offset_bits = bitdepth + 2 * kFilterBits - kRoundBitsHorizontal;
   for (int y = 0; y < height; ++y) {
     for (int x = 0; x < width; ++x) {
       // An offset to guarantee the sum is non negative.
@@ -233,32 +242,34 @@
       }
       assert(sum >= 0 && sum < (1 << (offset_bits + 2)));
       dest[x] = static_cast<uint16_t>(
-          RightShiftWithRounding(sum, inter_round_bits[1]));
+          RightShiftWithRounding(sum, inter_round_bits_vertical));
     }
     dest += pred_stride;
     intermediate += intermediate_stride;
   }
 }
 
-// This function is a simplified version of Convolve2D_C.
-// It is called when it is single prediction mode, where only horizontal
-// filtering is required.
+// This function is a simplified version of ConvolveCompound2D_C.
+// It is called when it is single prediction mode, where both horizontal and
+// vertical filtering are required.
 // The output is the single prediction of the block, clipped to valid pixel
 // range.
 template <int bitdepth, typename Pixel>
-void Convolve2DSingle_C(const void* const reference,
-                        const ptrdiff_t reference_stride,
-                        const int horizontal_filter_index,
-                        const int vertical_filter_index,
-                        const uint8_t inter_round_bits[2], const int subpixel_x,
-                        const int subpixel_y, const int /*step_x*/,
-                        const int /*step_y*/, const int width, const int height,
-                        void* prediction, const ptrdiff_t pred_stride) {
+void Convolve2D_C(const void* const reference, const ptrdiff_t reference_stride,
+                  const int horizontal_filter_index,
+                  const int vertical_filter_index,
+                  const uint8_t inter_round_bits_vertical, const int subpixel_x,
+                  const int subpixel_y, const int /*step_x*/,
+                  const int /*step_y*/, const int width, const int height,
+                  void* prediction, const ptrdiff_t pred_stride) {
+  constexpr int kRoundBitsHorizontal = (bitdepth == 12)
+                                           ? kInterRoundBitsHorizontal12bpp
+                                           : kInterRoundBitsHorizontal;
   const int intermediate_height = height + kSubPixelTaps - 1;
   // The output of the horizontal filter, i.e. the intermediate_result, is
   // guaranteed to fit in int16_t.
   int16_t intermediate_result[kMaxSuperBlockSizeInPixels *
-                              (2 * kMaxSuperBlockSizeInPixels + 8)];
+                              (kMaxSuperBlockSizeInPixels + kSubPixelTaps - 1)];
   const int intermediate_stride = kMaxSuperBlockSizeInPixels;
   const int single_round_offset = (1 << bitdepth) + (1 << (bitdepth - 1));
   const int max_pixel_value = (1 << bitdepth) - 1;
@@ -285,7 +296,7 @@
       }
       assert(sum >= 0 && sum < (1 << (bitdepth + kFilterBits + 1)));
       intermediate[x] = static_cast<int16_t>(
-          RightShiftWithRounding(sum, inter_round_bits[0]));
+          RightShiftWithRounding(sum, kRoundBitsHorizontal));
     }
     src += src_stride;
     intermediate += intermediate_stride;
@@ -294,7 +305,7 @@
   filter_index = GetFilterIndex(vertical_filter_index, height);
   intermediate = intermediate_result;
   filter_id = ((subpixel_y & 1023) >> 6) & kSubPixelMask;
-  const int offset_bits = bitdepth + 2 * kFilterBits - inter_round_bits[0];
+  const int offset_bits = bitdepth + 2 * kFilterBits - kRoundBitsHorizontal;
   for (int y = 0; y < height; ++y) {
     for (int x = 0; x < width; ++x) {
       // An offset to guarantee the sum is non negative.
@@ -305,7 +316,7 @@
       }
       assert(sum >= 0 && sum < (1 << (offset_bits + 2)));
       dest[x] = static_cast<Pixel>(
-          Clip3(RightShiftWithRounding(sum, inter_round_bits[1]) -
+          Clip3(RightShiftWithRounding(sum, inter_round_bits_vertical) -
                     single_round_offset,
                 0, max_pixel_value));
     }
@@ -324,13 +335,16 @@
                           const ptrdiff_t reference_stride,
                           const int horizontal_filter_index,
                           const int /*vertical_filter_index*/,
-                          const uint8_t inter_round_bits[2],
+                          const uint8_t /*inter_round_bits_vertical*/,
                           const int subpixel_x, const int /*subpixel_y*/,
                           const int /*step_x*/, const int /*step_y*/,
                           const int width, const int height, void* prediction,
                           const ptrdiff_t pred_stride) {
+  constexpr int kRoundBitsHorizontal = (bitdepth == 12)
+                                           ? kInterRoundBitsHorizontal12bpp
+                                           : kInterRoundBitsHorizontal;
   const int filter_index = GetFilterIndex(horizontal_filter_index, width);
-  const int bits = kFilterBits - inter_round_bits[0];
+  const int bits = kFilterBits - kRoundBitsHorizontal;
   const auto* src = static_cast<const Pixel*>(reference) - kHorizontalOffset;
   const ptrdiff_t src_stride = reference_stride / sizeof(Pixel);
   auto* dest = static_cast<Pixel*>(prediction);
@@ -343,7 +357,7 @@
       for (int k = 0; k < kSubPixelTaps; ++k) {
         sum += kSubPixelFilters[filter_index][filter_id][k] * src[x + k];
       }
-      sum = RightShiftWithRounding(sum, inter_round_bits[0]);
+      sum = RightShiftWithRounding(sum, kRoundBitsHorizontal);
       dest[x] = static_cast<Pixel>(
           Clip3(RightShiftWithRounding(sum, bits), 0, max_pixel_value));
     }
@@ -362,7 +376,7 @@
                         const ptrdiff_t reference_stride,
                         const int /*horizontal_filter_index*/,
                         const int vertical_filter_index,
-                        const uint8_t /*inter_round_bits*/[2],
+                        const uint8_t /*inter_round_bits_vertical*/,
                         const int /*subpixel_x*/, const int subpixel_y,
                         const int /*step_x*/, const int /*step_y*/,
                         const int width, const int height, void* prediction,
@@ -374,6 +388,18 @@
   auto* dest = static_cast<Pixel*>(prediction);
   const ptrdiff_t dest_stride = pred_stride / sizeof(Pixel);
   const int filter_id = (subpixel_y >> 6) & kSubPixelMask;
+  // First filter is always a copy.
+  if (filter_id == 0) {
+    // Move |src| down the actual values and not the start of the context.
+    src = static_cast<const Pixel*>(reference);
+    int y = 0;
+    do {
+      memcpy(dest, src, width * sizeof(src[0]));
+      src += src_stride;
+      dest += dest_stride;
+    } while (++y < height);
+    return;
+  }
   const int max_pixel_value = (1 << bitdepth) - 1;
   for (int y = 0; y < height; ++y) {
     for (int x = 0; x < width; ++x) {
@@ -395,7 +421,7 @@
                     const ptrdiff_t reference_stride,
                     const int /*horizontal_filter_index*/,
                     const int /*vertical_filter_index*/,
-                    const uint8_t /*inter_round_bits*/[2],
+                    const uint8_t /*inter_round_bits_vertical*/,
                     const int /*subpixel_x*/, const int /*subpixel_y*/,
                     const int /*step_x*/, const int /*step_y*/, const int width,
                     const int height, void* prediction,
@@ -414,7 +440,7 @@
                             const ptrdiff_t reference_stride,
                             const int /*horizontal_filter_index*/,
                             const int /*vertical_filter_index*/,
-                            const uint8_t /*inter_round_bits*/[2],
+                            const uint8_t /*inter_round_bits_vertical*/,
                             const int /*subpixel_x*/, const int /*subpixel_y*/,
                             const int /*step_x*/, const int /*step_y*/,
                             const int width, const int height, void* prediction,
@@ -433,7 +459,7 @@
   }
 }
 
-// This function is a simplified version of Convolve2D_C.
+// This function is a simplified version of ConvolveCompound2D_C.
 // It is called when it is compound prediction mode, where only horizontal
 // filtering is required.
 // The output is not clipped to valid pixel range. Its output will be
@@ -442,16 +468,19 @@
 void ConvolveCompoundHorizontal_C(
     const void* const reference, const ptrdiff_t reference_stride,
     const int horizontal_filter_index, const int /*vertical_filter_index*/,
-    const uint8_t inter_round_bits[2], const int subpixel_x,
+    const uint8_t inter_round_bits_vertical, const int subpixel_x,
     const int /*subpixel_y*/, const int /*step_x*/, const int /*step_y*/,
     const int width, const int height, void* prediction,
     const ptrdiff_t pred_stride) {
+  constexpr int kRoundBitsHorizontal = (bitdepth == 12)
+                                           ? kInterRoundBitsHorizontal12bpp
+                                           : kInterRoundBitsHorizontal;
   const int filter_index = GetFilterIndex(horizontal_filter_index, width);
   const auto* src = static_cast<const Pixel*>(reference) - kHorizontalOffset;
   const ptrdiff_t src_stride = reference_stride / sizeof(Pixel);
   auto* dest = static_cast<uint16_t*>(prediction);
   const int filter_id = (subpixel_x >> 6) & kSubPixelMask;
-  const int bits_shift = kFilterBits - inter_round_bits[1];
+  const int bits_shift = kFilterBits - inter_round_bits_vertical;
   const int compound_round_offset =
       (1 << (bitdepth + 4)) + (1 << (bitdepth + 3));
   for (int y = 0; y < height; ++y) {
@@ -460,7 +489,7 @@
       for (int k = 0; k < kSubPixelTaps; ++k) {
         sum += kSubPixelFilters[filter_index][filter_id][k] * src[x + k];
       }
-      sum = RightShiftWithRounding(sum, inter_round_bits[0]) << bits_shift;
+      sum = RightShiftWithRounding(sum, kRoundBitsHorizontal) << bits_shift;
       dest[x] = sum + compound_round_offset;
     }
     src += src_stride;
@@ -468,7 +497,7 @@
   }
 }
 
-// This function is a simplified version of Convolve2D_C.
+// This function is a simplified version of ConvolveCompound2D_C.
 // It is called when it is compound prediction mode, where only vertical
 // filtering is required.
 // The output is not clipped to valid pixel range. Its output will be
@@ -478,18 +507,21 @@
                                 const ptrdiff_t reference_stride,
                                 const int /*horizontal_filter_index*/,
                                 const int vertical_filter_index,
-                                const uint8_t inter_round_bits[2],
+                                const uint8_t inter_round_bits_vertical,
                                 const int /*subpixel_x*/, const int subpixel_y,
                                 const int /*step_x*/, const int /*step_y*/,
                                 const int width, const int height,
                                 void* prediction, const ptrdiff_t pred_stride) {
+  constexpr int kRoundBitsHorizontal = (bitdepth == 12)
+                                           ? kInterRoundBitsHorizontal12bpp
+                                           : kInterRoundBitsHorizontal;
   const int filter_index = GetFilterIndex(vertical_filter_index, height);
   const ptrdiff_t src_stride = reference_stride / sizeof(Pixel);
   const auto* src =
       static_cast<const Pixel*>(reference) - kVerticalOffset * src_stride;
   auto* dest = static_cast<uint16_t*>(prediction);
   const int filter_id = (subpixel_y >> 6) & kSubPixelMask;
-  const int bits_shift = kFilterBits - inter_round_bits[0];
+  const int bits_shift = kFilterBits - kRoundBitsHorizontal;
   const int compound_round_offset =
       (1 << (bitdepth + 4)) + (1 << (bitdepth + 3));
   for (int y = 0; y < height; ++y) {
@@ -500,7 +532,7 @@
                src[k * src_stride + x];
       }
       dest[x] = RightShiftWithRounding(LeftShift(sum, bits_shift),
-                                       inter_round_bits[1]) +
+                                       inter_round_bits_vertical) +
                 compound_round_offset;
     }
     src += src_stride;
@@ -518,7 +550,7 @@
 void ConvolveIntraBlockCopy2D_C(
     const void* const reference, const ptrdiff_t reference_stride,
     const int /*horizontal_filter_index*/, const int /*vertical_filter_index*/,
-    const uint8_t /*inter_round_bits*/[2], const int /*subpixel_x*/,
+    const uint8_t /*inter_round_bits_vertical*/, const int /*subpixel_x*/,
     const int /*subpixel_y*/, const int /*step_x*/, const int /*step_y*/,
     const int width, const int height, void* prediction,
     const ptrdiff_t pred_stride) {
@@ -526,13 +558,14 @@
   const ptrdiff_t src_stride = reference_stride / sizeof(Pixel);
   auto* dest = reinterpret_cast<Pixel*>(prediction);
   const ptrdiff_t dest_stride = pred_stride / sizeof(Pixel);
+  const int intermediate_height = height + 1;
   uint16_t intermediate_result[kMaxSuperBlockSizeInPixels *
-                               (2 * kMaxSuperBlockSizeInPixels + 8)];
+                               (kMaxSuperBlockSizeInPixels + 1)];
   uint16_t* intermediate = intermediate_result;
   // Note: allow vertical access to height + 1. Because this function is only
   // for u/v plane of intra block copy, such access is guaranteed to be within
   // the prediction block.
-  for (int y = 0; y <= height; ++y) {
+  for (int y = 0; y < intermediate_height; ++y) {
     for (int x = 0; x < width; ++x) {
       intermediate[x] = src[x] + src[x + 1];
     }
@@ -562,7 +595,7 @@
 void ConvolveIntraBlockCopy1D_C(
     const void* const reference, const ptrdiff_t reference_stride,
     const int /*horizontal_filter_index*/, const int /*vertical_filter_index*/,
-    const uint8_t /*inter_round_bits*/[2], const int /*subpixel_x*/,
+    const uint8_t /*inter_round_bits_vertical*/, const int /*subpixel_x*/,
     const int /*subpixel_y*/, const int /*step_x*/, const int /*step_y*/,
     const int width, const int height, void* prediction,
     const ptrdiff_t pred_stride) {
@@ -584,15 +617,16 @@
 void Init8bpp() {
   Dsp* const dsp = dsp_internal::GetWritableDspTable(8);
   assert(dsp != nullptr);
+#if LIBGAV1_ENABLE_ALL_DSP_FUNCTIONS
   dsp->convolve[0][0][0][0] = ConvolveCopy_C<8, uint8_t>;
   dsp->convolve[0][0][0][1] = ConvolveHorizontal_C<8, uint8_t>;
   dsp->convolve[0][0][1][0] = ConvolveVertical_C<8, uint8_t>;
-  dsp->convolve[0][0][1][1] = Convolve2DSingle_C<8, uint8_t>;
+  dsp->convolve[0][0][1][1] = Convolve2D_C<8, uint8_t>;
 
   dsp->convolve[0][1][0][0] = ConvolveCompoundCopy_C<8, uint8_t>;
   dsp->convolve[0][1][0][1] = ConvolveCompoundHorizontal_C<8, uint8_t>;
   dsp->convolve[0][1][1][0] = ConvolveCompoundVertical_C<8, uint8_t>;
-  dsp->convolve[0][1][1][1] = Convolve2D_C<8, uint8_t>;
+  dsp->convolve[0][1][1][1] = ConvolveCompound2D_C<8, uint8_t>;
 
   dsp->convolve[1][0][0][0] = ConvolveCopy_C<8, uint8_t>;
   dsp->convolve[1][0][0][1] =
@@ -606,23 +640,78 @@
   dsp->convolve[1][1][1][0] = nullptr;
   dsp->convolve[1][1][1][1] = nullptr;
 
-  dsp->convolve_scale[0] = Convolve2DScaleSingle_C<8, uint8_t>;
-  dsp->convolve_scale[1] = Convolve2DScale_C<8, uint8_t>;
+  dsp->convolve_scale[0] = ConvolveScale2D_C<8, uint8_t>;
+  dsp->convolve_scale[1] = ConvolveCompoundScale2D_C<8, uint8_t>;
+#else  // LIBGAV1_ENABLE_ALL_DSP_FUNCTIONS
+#ifndef LIBGAV1_Dsp8bpp_ConvolveCopy
+  dsp->convolve[0][0][0][0] = ConvolveCopy_C<8, uint8_t>;
+#endif
+#ifndef LIBGAV1_Dsp8bpp_ConvolveHorizontal
+  dsp->convolve[0][0][0][1] = ConvolveHorizontal_C<8, uint8_t>;
+#endif
+#ifndef LIBGAV1_Dsp8bpp_ConvolveVertical
+  dsp->convolve[0][0][1][0] = ConvolveVertical_C<8, uint8_t>;
+#endif
+#ifndef LIBGAV1_Dsp8bpp_Convolve2D
+  dsp->convolve[0][0][1][1] = Convolve2D_C<8, uint8_t>;
+#endif
+
+#ifndef LIBGAV1_Dsp8bpp_ConvolveCompoundCopy
+  dsp->convolve[0][1][0][0] = ConvolveCompoundCopy_C<8, uint8_t>;
+#endif
+#ifndef LIBGAV1_Dsp8bpp_ConvolveCompoundHorizontal
+  dsp->convolve[0][1][0][1] = ConvolveCompoundHorizontal_C<8, uint8_t>;
+#endif
+#ifndef LIBGAV1_Dsp8bpp_ConvolveCompoundVertical
+  dsp->convolve[0][1][1][0] = ConvolveCompoundVertical_C<8, uint8_t>;
+#endif
+#ifndef LIBGAV1_Dsp8bpp_ConvolveCompound2D
+  dsp->convolve[0][1][1][1] = ConvolveCompound2D_C<8, uint8_t>;
+#endif
+
+#ifndef LIBGAV1_Dsp8bpp_ConvolveIntraBlockCopy
+  dsp->convolve[1][0][0][0] = ConvolveCopy_C<8, uint8_t>;
+#endif
+#ifndef LIBGAV1_Dsp8bpp_ConvolveIntraBlockCopyHorizontal
+  dsp->convolve[1][0][0][1] =
+      ConvolveIntraBlockCopy1D_C<8, uint8_t, /*is_horizontal=*/true>;
+#endif
+#ifndef LIBGAV1_Dsp8bpp_ConvolveIntraBlockCopyVertical
+  dsp->convolve[1][0][1][0] =
+      ConvolveIntraBlockCopy1D_C<8, uint8_t, /*is_horizontal=*/false>;
+#endif
+#ifndef LIBGAV1_Dsp8bpp_ConvolveIntraBlockCopy2D
+  dsp->convolve[1][0][1][1] = ConvolveIntraBlockCopy2D_C<8, uint8_t>;
+#endif
+
+  dsp->convolve[1][1][0][0] = nullptr;
+  dsp->convolve[1][1][0][1] = nullptr;
+  dsp->convolve[1][1][1][0] = nullptr;
+  dsp->convolve[1][1][1][1] = nullptr;
+
+#ifndef LIBGAV1_Dsp8bpp_ConvolveScale2D
+  dsp->convolve_scale[0] = ConvolveScale2D_C<8, uint8_t>;
+#endif
+#ifndef LIBGAV1_Dsp8bpp_ConvolveCompoundScale2D
+  dsp->convolve_scale[1] = ConvolveCompoundScale2D_C<8, uint8_t>;
+#endif
+#endif  // LIBGAV1_ENABLE_ALL_DSP_FUNCTIONS
 }
 
 #if LIBGAV1_MAX_BITDEPTH >= 10
 void Init10bpp() {
   Dsp* const dsp = dsp_internal::GetWritableDspTable(10);
   assert(dsp != nullptr);
+#if LIBGAV1_ENABLE_ALL_DSP_FUNCTIONS
   dsp->convolve[0][0][0][0] = ConvolveCopy_C<10, uint16_t>;
   dsp->convolve[0][0][0][1] = ConvolveHorizontal_C<10, uint16_t>;
   dsp->convolve[0][0][1][0] = ConvolveVertical_C<10, uint16_t>;
-  dsp->convolve[0][0][1][1] = Convolve2DSingle_C<10, uint16_t>;
+  dsp->convolve[0][0][1][1] = Convolve2D_C<10, uint16_t>;
 
   dsp->convolve[0][1][0][0] = ConvolveCompoundCopy_C<10, uint16_t>;
   dsp->convolve[0][1][0][1] = ConvolveCompoundHorizontal_C<10, uint16_t>;
   dsp->convolve[0][1][1][0] = ConvolveCompoundVertical_C<10, uint16_t>;
-  dsp->convolve[0][1][1][1] = Convolve2D_C<10, uint16_t>;
+  dsp->convolve[0][1][1][1] = ConvolveCompound2D_C<10, uint16_t>;
 
   dsp->convolve[1][0][0][0] = ConvolveCopy_C<10, uint16_t>;
   dsp->convolve[1][0][0][1] =
@@ -636,8 +725,62 @@
   dsp->convolve[1][1][1][0] = nullptr;
   dsp->convolve[1][1][1][1] = nullptr;
 
-  dsp->convolve_scale[0] = Convolve2DScaleSingle_C<10, uint16_t>;
-  dsp->convolve_scale[1] = Convolve2DScale_C<10, uint16_t>;
+  dsp->convolve_scale[0] = ConvolveScale2D_C<10, uint16_t>;
+  dsp->convolve_scale[1] = ConvolveCompoundScale2D_C<10, uint16_t>;
+#else  // LIBGAV1_ENABLE_ALL_DSP_FUNCTIONS
+#ifndef LIBGAV1_Dsp10bpp_ConvolveCopy
+  dsp->convolve[0][0][0][0] = ConvolveCopy_C<10, uint16_t>;
+#endif
+#ifndef LIBGAV1_Dsp10bpp_ConvolveHorizontal
+  dsp->convolve[0][0][0][1] = ConvolveHorizontal_C<10, uint16_t>;
+#endif
+#ifndef LIBGAV1_Dsp10bpp_ConvolveVertical
+  dsp->convolve[0][0][1][0] = ConvolveVertical_C<10, uint16_t>;
+#endif
+#ifndef LIBGAV1_Dsp10bpp_Convolve2D
+  dsp->convolve[0][0][1][1] = Convolve2D_C<10, uint16_t>;
+#endif
+
+#ifndef LIBGAV1_Dsp10bpp_ConvolveCompoundCopy
+  dsp->convolve[0][1][0][0] = ConvolveCompoundCopy_C<10, uint16_t>;
+#endif
+#ifndef LIBGAV1_Dsp10bpp_ConvolveCompoundHorizontal
+  dsp->convolve[0][1][0][1] = ConvolveCompoundHorizontal_C<10, uint16_t>;
+#endif
+#ifndef LIBGAV1_Dsp10bpp_ConvolveCompoundVertical
+  dsp->convolve[0][1][1][0] = ConvolveCompoundVertical_C<10, uint16_t>;
+#endif
+#ifndef LIBGAV1_Dsp10bpp_ConvolveCompound2D
+  dsp->convolve[0][1][1][1] = ConvolveCompound2D_C<10, uint16_t>;
+#endif
+
+#ifndef LIBGAV1_Dsp10bpp_ConvolveIntraBlockCopy
+  dsp->convolve[1][0][0][0] = ConvolveCopy_C<10, uint16_t>;
+#endif
+#ifndef LIBGAV1_Dsp10bpp_ConvolveIntraBlockHorizontal
+  dsp->convolve[1][0][0][1] =
+      ConvolveIntraBlockCopy1D_C<10, uint16_t, /*is_horizontal=*/true>;
+#endif
+#ifndef LIBGAV1_Dsp10bpp_ConvolveIntraBlockVertical
+  dsp->convolve[1][0][1][0] =
+      ConvolveIntraBlockCopy1D_C<10, uint16_t, /*is_horizontal=*/false>;
+#endif
+#ifndef LIBGAV1_Dsp10bpp_ConvolveIntraBlock2D
+  dsp->convolve[1][0][1][1] = ConvolveIntraBlockCopy2D_C<10, uint16_t>;
+#endif
+
+  dsp->convolve[1][1][0][0] = nullptr;
+  dsp->convolve[1][1][0][1] = nullptr;
+  dsp->convolve[1][1][1][0] = nullptr;
+  dsp->convolve[1][1][1][1] = nullptr;
+
+#ifndef LIBGAV1_Dsp10bpp_ConvolveScale2D
+  dsp->convolve_scale[0] = ConvolveScale2D_C<10, uint16_t>;
+#endif
+#ifndef LIBGAV1_Dsp10bpp_ConvolveCompoundScale2D
+  dsp->convolve_scale[1] = ConvolveCompoundScale2D_C<10, uint16_t>;
+#endif
+#endif  // LIBGAV1_ENABLE_ALL_DSP_FUNCTIONS
 }
 #endif
 
diff --git a/libgav1/src/dsp/convolve.h b/libgav1/src/dsp/convolve.h
index 9839bb2..545e6ee 100644
--- a/libgav1/src/dsp/convolve.h
+++ b/libgav1/src/dsp/convolve.h
@@ -1,9 +1,29 @@
 #ifndef LIBGAV1_SRC_DSP_CONVOLVE_H_
 #define LIBGAV1_SRC_DSP_CONVOLVE_H_
 
+// Pull in LIBGAV1_DspXXX defines representing the implementation status
+// of each function. The resulting value of each can be used by each module to
+// determine whether an implementation is needed at compile time.
+// IWYU pragma: begin_exports
+
+// ARM:
+#include "src/dsp/arm/convolve_neon.h"
+
+// x86:
+// Note includes should be sorted in logical order avx2/avx/sse4, etc.
+// The order of includes is important as each tests for a superior version
+// before setting the base.
+// clang-format off
+#include "src/dsp/x86/convolve_sse4.h"
+// clang-format on
+
+// IWYU pragma: end_exports
+
 namespace libgav1 {
 namespace dsp {
 
+// Initializes Dsp::convolve and Dsp::convolve_scale. This function is not
+// thread-safe.
 void ConvolveInit_C();
 
 }  // namespace dsp
diff --git a/libgav1/src/dsp/distance_weighted_blend.cc b/libgav1/src/dsp/distance_weighted_blend.cc
index fda84ee..05e8e3f 100644
--- a/libgav1/src/dsp/distance_weighted_blend.cc
+++ b/libgav1/src/dsp/distance_weighted_blend.cc
@@ -12,27 +12,31 @@
 namespace {
 
 template <int bitdepth, typename Pixel>
-void DistanceWeightedBlending_C(const uint16_t* prediction_0,
-                                const ptrdiff_t prediction_stride_0,
-                                const uint16_t* prediction_1,
-                                const ptrdiff_t prediction_stride_1,
-                                const uint8_t weight_0, const uint8_t weight_1,
-                                const int inter_post_round_bit, const int width,
-                                const int height, void* const dest,
-                                const ptrdiff_t dest_stride) {
+void DistanceWeightedBlend_C(const uint16_t* prediction_0,
+                             const ptrdiff_t prediction_stride_0,
+                             const uint16_t* prediction_1,
+                             const ptrdiff_t prediction_stride_1,
+                             const uint8_t weight_0, const uint8_t weight_1,
+                             const int width, const int height,
+                             void* const dest, const ptrdiff_t dest_stride) {
   // An offset to cancel offsets used in compound predictor generation that
   // make intermediate computations non negative.
-  const int compound_round_offset =
-      (1 << (bitdepth + 4)) + (1 << (bitdepth + 3));
+  constexpr int compound_round_offset =
+      (16 << (bitdepth + 4)) + (16 << (bitdepth + 3));
+  // 7.11.3.2 Rounding variables derivation process
+  //   2 * FILTER_BITS(7) - (InterRound0(3|5) + InterRound1(7))
+  constexpr int inter_post_round_bits = (bitdepth == 12) ? 2 : 4;
   auto* dst = static_cast<Pixel*>(dest);
   const ptrdiff_t dst_stride = dest_stride / sizeof(Pixel);
 
   for (int y = 0; y < height; ++y) {
     for (int x = 0; x < width; ++x) {
-      int res = (prediction_0[x] * weight_0 + prediction_1[x] * weight_1) >> 4;
+      // prediction range: 8bpp: [0, 15471] 10bpp: [0, 61983] 12bpp: [0, 62007].
+      // weight_0 + weight_1 = 16.
+      int res = prediction_0[x] * weight_0 + prediction_1[x] * weight_1;
       res -= compound_round_offset;
       dst[x] = static_cast<Pixel>(
-          Clip3(RightShiftWithRounding(res, inter_post_round_bit), 0,
+          Clip3(RightShiftWithRounding(res, inter_post_round_bits + 4), 0,
                 (1 << bitdepth) - 1));
     }
     dst += dst_stride;
@@ -44,14 +48,28 @@
 void Init8bpp() {
   Dsp* const dsp = dsp_internal::GetWritableDspTable(8);
   assert(dsp != nullptr);
-  dsp->distance_weighted_blend = DistanceWeightedBlending_C<8, uint8_t>;
+#if LIBGAV1_ENABLE_ALL_DSP_FUNCTIONS
+  dsp->distance_weighted_blend = DistanceWeightedBlend_C<8, uint8_t>;
+#else  // !LIBGAV1_ENABLE_ALL_DSP_FUNCTIONS
+  static_cast<void>(dsp);
+#ifndef LIBGAV1_Dsp8bpp_DistanceWeightedBlend
+  dsp->distance_weighted_blend = DistanceWeightedBlend_C<8, uint8_t>;
+#endif
+#endif  // LIBGAV1_ENABLE_ALL_DSP_FUNCTIONS
 }
 
 #if LIBGAV1_MAX_BITDEPTH >= 10
 void Init10bpp() {
   Dsp* const dsp = dsp_internal::GetWritableDspTable(10);
   assert(dsp != nullptr);
-  dsp->distance_weighted_blend = DistanceWeightedBlending_C<10, uint16_t>;
+#if LIBGAV1_ENABLE_ALL_DSP_FUNCTIONS
+  dsp->distance_weighted_blend = DistanceWeightedBlend_C<10, uint16_t>;
+#else  // !LIBGAV1_ENABLE_ALL_DSP_FUNCTIONS
+  static_cast<void>(dsp);
+#ifndef LIBGAV1_Dsp10bpp_DistanceWeightedBlend
+  dsp->distance_weighted_blend = DistanceWeightedBlend_C<10, uint16_t>;
+#endif
+#endif  // LIBGAV1_ENABLE_ALL_DSP_FUNCTIONS
 }
 #endif
 
diff --git a/libgav1/src/dsp/distance_weighted_blend.h b/libgav1/src/dsp/distance_weighted_blend.h
index d71f09a..b37bd64 100644
--- a/libgav1/src/dsp/distance_weighted_blend.h
+++ b/libgav1/src/dsp/distance_weighted_blend.h
@@ -1,9 +1,28 @@
 #ifndef LIBGAV1_SRC_DSP_DISTANCE_WEIGHTED_BLEND_H_
 #define LIBGAV1_SRC_DSP_DISTANCE_WEIGHTED_BLEND_H_
 
+// Pull in LIBGAV1_DspXXX defines representing the implementation status
+// of each function. The resulting value of each can be used by each module to
+// determine whether an implementation is needed at compile time.
+// IWYU pragma: begin_exports
+
+// ARM:
+#include "src/dsp/arm/distance_weighted_blend_neon.h"
+
+// x86:
+// Note includes should be sorted in logical order avx2/avx/sse4, etc.
+// The order of includes is important as each tests for a superior version
+// before setting the base.
+// clang-format off
+#include "src/dsp/x86/distance_weighted_blend_sse4.h"
+// clang-format on
+
+// IWYU pragma: end_exports
+
 namespace libgav1 {
 namespace dsp {
 
+// Initializes Dsp::distance_weighted_blend. This function is not thread-safe.
 void DistanceWeightedBlendInit_C();
 
 }  // namespace dsp
diff --git a/libgav1/src/dsp/dsp.cc b/libgav1/src/dsp/dsp.cc
index c28692b..c4bfc96 100644
--- a/libgav1/src/dsp/dsp.cc
+++ b/libgav1/src/dsp/dsp.cc
@@ -2,9 +2,6 @@
 
 #include <mutex>  // NOLINT (unapproved c++11 header)
 
-#include "src/dsp/arm/intrapred_neon.h"
-#include "src/dsp/arm/loop_filter_neon.h"
-#include "src/dsp/arm/loop_restoration_neon.h"
 #include "src/dsp/average_blend.h"
 #include "src/dsp/cdef.h"
 #include "src/dsp/convolve.h"
@@ -16,15 +13,9 @@
 #include "src/dsp/inverse_transform.h"
 #include "src/dsp/loop_filter.h"
 #include "src/dsp/loop_restoration.h"
-#include "src/dsp/mask_blending.h"
+#include "src/dsp/mask_blend.h"
 #include "src/dsp/obmc.h"
 #include "src/dsp/warp.h"
-#include "src/dsp/x86/intra_edge_sse4.h"
-#include "src/dsp/x86/intrapred_smooth_sse4.h"
-#include "src/dsp/x86/intrapred_sse4.h"
-#include "src/dsp/x86/inverse_transform_sse4.h"
-#include "src/dsp/x86/loop_filter_sse4.h"
-#include "src/dsp/x86/loop_restoration_sse4.h"
 
 namespace libgav1 {
 namespace dsp_internal {
@@ -62,12 +53,15 @@
     InverseTransformInit_C();
     LoopFilterInit_C();
     LoopRestorationInit_C();
-    MaskBlendingInit_C();
+    MaskBlendInit_C();
     ObmcInit_C();
     WarpInit_C();
 #if LIBGAV1_ENABLE_SSE4_1
     const uint32_t cpu_features = GetCpuInfo();
     if ((cpu_features & kSSE4_1) != 0) {
+      AverageBlendInit_SSE4_1();
+      ConvolveInit_SSE4_1();
+      DistanceWeightedBlendInit_SSE4_1();
       IntraEdgeInit_SSE4_1();
       IntraPredInit_SSE4_1();
       IntraPredCflInit_SSE4_1();
@@ -75,16 +69,25 @@
       InverseTransformInit_SSE4_1();
       LoopFilterInit_SSE4_1();
       LoopRestorationInit_SSE4_1();
+      ObmcInit_SSE4_1();
     }
 #endif  // LIBGAV1_ENABLE_SSE4_1
 #if LIBGAV1_ENABLE_NEON
+    AverageBlendInit_NEON();
+    ConvolveInit_NEON();
+    DistanceWeightedBlendInit_NEON();
+    IntraEdgeInit_NEON();
     IntraPredCflInit_NEON();
     IntraPredDirectionalInit_NEON();
     IntraPredFilterIntraInit_NEON();
     IntraPredInit_NEON();
     IntraPredSmoothInit_NEON();
+    InverseTransformInit_NEON();
     LoopFilterInit_NEON();
     LoopRestorationInit_NEON();
+    MaskBlendInit_NEON();
+    ObmcInit_NEON();
+    WarpInit_NEON();
 #endif  // LIBGAV1_ENABLE_NEON
   });
 }
diff --git a/libgav1/src/dsp/dsp.h b/libgav1/src/dsp/dsp.h
index 8c7deb3..631a42c 100644
--- a/libgav1/src/dsp/dsp.h
+++ b/libgav1/src/dsp/dsp.h
@@ -306,7 +306,7 @@
 // signals the direction of the transform loop. |non_zero_coeff_count| is the
 // number of non zero coefficients in the block.
 using InverseTransformAddFunc = void (*)(TransformType tx_type,
-                                         TransformSize tx_size, int8_t bitdepth,
+                                         TransformSize tx_size,
                                          void* src_buffer, int start_x,
                                          int start_y, void* dst_frame,
                                          bool is_row, int non_zero_coeff_count);
@@ -339,7 +339,7 @@
 // plane.
 // |primary_strength|, |secondary_strength|, and |damping| are Cdef filtering
 // parameters.
-// |direction| is the filtering diretion.
+// |direction| is the filtering direction.
 // |dest| is the output buffer. |dest_stride| is given in bytes.
 using CdefFilteringFunc = void (*)(const void* source, ptrdiff_t source_stride,
                                    int rows4x4, int columns4x4, int curr_x,
@@ -362,7 +362,7 @@
     RestorationBuffer* buffer);
 
 // Index 0 is Wiener Filter.
-// Index 1 is Self Guilded Restoration Filter.
+// Index 1 is Self Guided Restoration Filter.
 // This can be accessed as LoopRestorationType - 2.
 using LoopRestorationFuncs = LoopRestorationFunc[2];
 
@@ -373,8 +373,9 @@
 // |vertical_filter_index|/|horizontal_filter_index| is the index to
 // retrieve the type of filter to be applied for vertical/horizontal direction
 // from the filter lookup table 'kSubPixelFilters'.
-// |inter_round_bits| is rounding prediction used in horizontal
-// (inter_round_bits[0]) and vertical (inter_round_bits[1]) filtering.
+// |inter_round_bits_vertical| is the rounding precision used after vertical
+// filtering (7 or 11). kInterRoundBitsHorizontal &
+// kInterRoundBitsHorizontal12bpp can be used after the horizontal pass.
 // |subpixel_x| and |subpixel_y| are starting positions in units of 1/1024.
 // |step_x| and |step_y| are step sizes in units of 1/1024 of a pixel.
 // |width| and |height| are width and height of the block to be filtered.
@@ -384,15 +385,22 @@
 using ConvolveFunc = void (*)(const void* reference, ptrdiff_t reference_stride,
                               int vertical_filter_index,
                               int horizontal_filter_index,
-                              const uint8_t inter_round_bits[2], int subpixel_x,
-                              int subpixel_y, int step_x, int step_y, int width,
-                              int height, void* prediction,
-                              ptrdiff_t pred_stride);
+                              const uint8_t inter_round_bits_vertical,
+                              int subpixel_x, int subpixel_y, int step_x,
+                              int step_y, int width, int height,
+                              void* prediction, ptrdiff_t pred_stride);
 
 // Convolve functions signature. Each points to one convolve function with
 // a specific setting:
 // ConvolveFunc[is_intra_block_copy][is_compound][has_vertical_filter]
 // [has_horizontal_filter].
+// If is_compound is false, the prediction is clipped to pixel.
+// If is_compound is true, the range of prediction is:
+//   8bpp: [0, 15471]
+//   10bpp: [0, 61983]
+//   12bpp: [0, 62007]
+// See:
+// https://docs.google.com/document/d/1f5YlLk02ETNxpilvsmjBtWgDXjtZYO33hjl6bAdvmxc
 using ConvolveFuncs = ConvolveFunc[2][2][2][2];
 
 // Convolve functions signature for scaling version.
@@ -407,18 +415,15 @@
 // |prediction_0| is the first input block.
 // |prediction_1| is the second input block.
 // |prediction_stride_0| and |prediction_stride_1| are corresponding strides.
-// |inter_post_round_bit| is a rounding bit. It is required since the value
-// range of inputs is scaled in the inter frame prediction process.
 // |width| and |height| are the same for the first and second input blocks.
 // The valid range of block size is [8x8, 128x128] for the luma plane.
 // |dest| is the output buffer. |dest_stride| is the output buffer stride.
-using AverageBlendingFunc = void (*)(const uint16_t* prediction_0,
-                                     ptrdiff_t prediction_stride_0,
-                                     const uint16_t* prediction_1,
-                                     ptrdiff_t prediction_stride_1,
-                                     int inter_post_round_bit, int width,
-                                     int height, void* dest,
-                                     ptrdiff_t dest_stride);
+using AverageBlendFunc = void (*)(const uint16_t* prediction_0,
+                                  ptrdiff_t prediction_stride_0,
+                                  const uint16_t* prediction_1,
+                                  ptrdiff_t prediction_stride_1, int width,
+                                  int height, void* dest,
+                                  ptrdiff_t dest_stride);
 
 // Distance weighted blending function signature.
 // Weights are generated in Section 7.11.3.15.
@@ -432,25 +437,25 @@
 // distance of the first reference frame and the current frame.
 // |weight_1| is the weight for the second block. It is derived from the
 // relative distance of the second reference frame and the current frame.
-// |inter_post_round_bit| is a rounding bit. It is required since the value
-// range of inputs is scaled in the inter frame prediction process.
 // |width| and |height| are the same for the first and second input blocks.
 // The valid range of block size is [8x8, 128x128] for the luma plane.
 // |dest| is the output buffer. |dest_stride| is the output buffer stride.
-using DistanceWeightedBlendFunc =
-    void (*)(const uint16_t* prediction_0, ptrdiff_t prediction_stride_0,
-             const uint16_t* prediction_1, ptrdiff_t prediction_stride_1,
-             uint8_t weight_0, uint8_t weight_1, int inter_post_round_bit,
-             int width, int height, void* dest, ptrdiff_t dest_stride);
+using DistanceWeightedBlendFunc = void (*)(const uint16_t* prediction_0,
+                                           ptrdiff_t prediction_stride_0,
+                                           const uint16_t* prediction_1,
+                                           ptrdiff_t prediction_stride_1,
+                                           uint8_t weight_0, uint8_t weight_1,
+                                           int width, int height, void* dest,
+                                           ptrdiff_t dest_stride);
 
 // Mask blending function signature. Section 7.11.3.14.
-// This function takes two blocks and produces a blended output stored onto the
-// dest. The blending is a weighted average process, controlled by
-// values of the mask.
+// This function takes two blocks and produces a blended output stored into the
+// output block |dest|. The blending is a weighted average process, controlled
+// by values of the mask.
 // |prediction_0| is the first input block. When prediction mode is inter_intra
 // (or wedge_inter_intra), this refers to the inter frame prediction.
 // |prediction_stride_0| is the stride, given in units of uint16_t.
-// |prediction_1| is the second input block. When prediciton mode is inter_intra
+// |prediction_1| is the second input block. When prediction mode is inter_intra
 // (or wedge_inter_intra), this refers to the intra frame prediction.
 // |prediction_stride_1| is the stride, given in units of uint16_t.
 // |mask| is an integer array, whose value indicates the weight of the blending.
@@ -466,19 +471,24 @@
 // prediction blocks is from intra prediction of current frame. Otherwise, two
 // prediction blocks are both inter frame predictions.
 // |is_wedge_inter_intra| indicates if the mask is for the wedge prediction.
-// |inter_post_round_bits| is the rounding bits.
 // |dest| is the output block.
 // |dest_stride| is the corresponding stride for dest.
-using MaskBlendFunc =
-    void (*)(const uint16_t* prediction_0, ptrdiff_t prediction_stride_0,
-             const uint16_t* prediction_1, ptrdiff_t prediction_stride_1,
-             const uint8_t* mask, ptrdiff_t mask_stride, int width, int height,
-             int subsampling_x, int subsampling_y, bool is_inter_intra,
-             bool is_wedge_inter_intra, int inter_post_round_bits, void* dest,
-             ptrdiff_t dest_stride);
+using MaskBlendFunc = void (*)(const uint16_t* prediction_0,
+                               ptrdiff_t prediction_stride_0,
+                               const uint16_t* prediction_1,
+                               ptrdiff_t prediction_stride_1,
+                               const uint8_t* mask, ptrdiff_t mask_stride,
+                               int width, int height, void* dest,
+                               ptrdiff_t dest_stride);
 
-// Blending function signature. Section 7.11.3.10.
-// This function takes two blocks and produces a blended output stored onto the
+// Mask blending functions signature. Each points to one function with
+// a specific setting:
+// MaskBlendFunc[subsampling_x + subsampling_y][is_inter_intra].
+using MaskBlendFuncs = MaskBlendFunc[3][2];
+
+// Obmc (overlapped block motion compensation) blending function signature.
+// Section 7.11.3.10.
+// This function takes two blocks and produces a blended output stored into the
 // first input block. The blending is a weighted average process, controlled by
 // values of the mask.
 // Obmc is not a compound mode. It is different from other compound blending,
@@ -488,15 +498,13 @@
 // |prediction| is the first input block, which will be overwritten.
 // |prediction_stride| is the stride, given in bytes.
 // |width|, |height| are the same for both input blocks.
-// |blending_direction|, 0 stands for the second block is above the
-// first block; 1 stands for the second block is to the left of the first block.
-// |mask| is an integer array, whose value indicates the weight of the blending.
 // |obmc_prediction| is the second input block.
 // |obmc_prediction_stride| is its stride, given in bytes.
 using ObmcBlendFunc = void (*)(void* prediction, ptrdiff_t prediction_stride,
-                               int width, int height, int blending_direction,
-                               const uint8_t* mask, const void* obmc_prediction,
+                               int width, int height,
+                               const void* obmc_prediction,
                                ptrdiff_t obmc_prediction_stride);
+using ObmcBlendFuncs = ObmcBlendFunc[kNumObmcDirections];
 
 // Warp function signature. Section 7.11.3.5.
 // This function applies warp filtering for each 8x8 block inside the current
@@ -512,19 +520,27 @@
 //     z .  y'  =   m4 m5 m1 *  y
 //          1]      m6 m7 1)    1]
 // |subsampling_x/y| is the current frame's plane subsampling factor.
-// |inter_round_bits| is rounding prediction used in horizontal
-// (inter_round_bits[0]) and vertical (inter_round_bits[1]) filtering.
+// |inter_round_bits_vertical| is the rounding precision used after vertical
+// filtering (7 or 11). kInterRoundBitsHorizontal &
+// kInterRoundBitsHorizontal12bpp can be used for the horizontal pass.
 // |block_start_x| and |block_start_y| are the starting position the current
 // coding block.
 // |block_width| and |block_height| are width and height of the current coding
-// block.
-// |alpha|, |beta|, |gamma|, |delta| are warp parameters.
+// block. |block_width| and |block_height| are at least 8.
+// |alpha|, |beta|, |gamma|, |delta| are valid warp parameters. See the
+// comments in the definition of struct GlobalMotion for the range of their
+// values.
 // |dest| is the output buffer. It is a predictor, whose type is int16_t.
 // |dest_stride| is the stride, in units of int16_t.
+//
+// NOTE: The ARM NEON implementation of WarpFunc may read up to 13 bytes before
+// the |source| buffer or up to 14 bytes after the |source| buffer. Therefore,
+// there must be enough padding before and after the |source| buffer.
 using WarpFunc = void (*)(const void* source, ptrdiff_t source_stride,
                           int source_width, int source_height,
                           const int* warp_params, int subsampling_x,
-                          int subsampling_y, const uint8_t inter_round_bits[2],
+                          int subsampling_y,
+                          const uint8_t inter_round_bits_vertical,
                           int block_start_x, int block_start_y, int block_width,
                           int block_height, int16_t alpha, int16_t beta,
                           int16_t gamma, int16_t delta, uint16_t* dest,
@@ -576,10 +592,10 @@
   LoopRestorationFuncs loop_restorations;
   ConvolveFuncs convolve;
   ConvolveScaleFuncs convolve_scale;
-  AverageBlendingFunc average_blend;
+  AverageBlendFunc average_blend;
   DistanceWeightedBlendFunc distance_weighted_blend;
-  MaskBlendFunc mask_blend;
-  ObmcBlendFunc obmc_blend;
+  MaskBlendFuncs mask_blend;
+  ObmcBlendFuncs obmc_blend;
   WarpFunc warp;
   FilmGrainSynthesisFunc film_grain_synthesis;
 };
diff --git a/libgav1/src/dsp/film_grain.cc b/libgav1/src/dsp/film_grain.cc
index cce73d3..b26b517 100644
--- a/libgav1/src/dsp/film_grain.cc
+++ b/libgav1/src/dsp/film_grain.cc
@@ -240,14 +240,28 @@
 void Init8bpp() {
   Dsp* const dsp = dsp_internal::GetWritableDspTable(8);
   assert(dsp != nullptr);
+#if LIBGAV1_ENABLE_ALL_DSP_FUNCTIONS
   dsp->film_grain_synthesis = FilmGrainSynthesis_C<8>;
+#else  // !LIBGAV1_ENABLE_ALL_DSP_FUNCTIONS
+  static_cast<void>(dsp);
+#ifndef LIBGAV1_Dsp8bpp_FilmGrainSynthesis
+  dsp->film_grain_synthesis = FilmGrainSynthesis_C<8>;
+#endif
+#endif  // LIBGAV1_ENABLE_ALL_DSP_FUNCTIONS
 }
 
 #if LIBGAV1_MAX_BITDEPTH >= 10
 void Init10bpp() {
   Dsp* const dsp = dsp_internal::GetWritableDspTable(10);
   assert(dsp != nullptr);
+#if LIBGAV1_ENABLE_ALL_DSP_FUNCTIONS
   dsp->film_grain_synthesis = FilmGrainSynthesis_C<10>;
+#else  // !LIBGAV1_ENABLE_ALL_DSP_FUNCTIONS
+  static_cast<void>(dsp);
+#ifndef LIBGAV1_Dsp10bpp_FilmGrainSynthesis
+  dsp->film_grain_synthesis = FilmGrainSynthesis_C<10>;
+#endif
+#endif  // LIBGAV1_ENABLE_ALL_DSP_FUNCTIONS
 }
 #endif
 
@@ -297,34 +311,41 @@
   const int grain_center = 128 << (bitdepth - 8);
   grain_min_ = -grain_center;
   grain_max_ = grain_center - 1;
+}
 
+template <int bitdepth>
+bool FilmGrain<bitdepth>::Init() {
   // Section 7.18.3.3. Generate grain process.
-  GenerateLumaGrain(params, luma_grain_);
-  ApplyAutoRegressiveFilterToLumaGrain(params, grain_min_, grain_max_,
+  GenerateLumaGrain(params_, luma_grain_);
+  ApplyAutoRegressiveFilterToLumaGrain(params_, grain_min_, grain_max_,
                                        luma_grain_);
-  if (!is_monochrome) {
-    GenerateChromaGrains(params, chroma_width_, chroma_height_, u_grain_,
+  if (!is_monochrome_) {
+    GenerateChromaGrains(params_, chroma_width_, chroma_height_, u_grain_,
                          v_grain_);
     ApplyAutoRegressiveFilterToChromaGrains(
-        params, grain_min_, grain_max_, luma_grain_, subsampling_x,
-        subsampling_y, chroma_width_, chroma_height_, u_grain_, v_grain_);
+        params_, grain_min_, grain_max_, luma_grain_, subsampling_x_,
+        subsampling_y_, chroma_width_, chroma_height_, u_grain_, v_grain_);
   }
 
   // Section 7.18.3.4. Scaling lookup initialization process.
-  InitializeScalingLookupTable(params.num_y_points, params.point_y_value,
-                               params.point_y_scaling, scaling_lut_y_);
-  if (!is_monochrome) {
-    if (params.chroma_scaling_from_luma) {
-      static_assert(sizeof(scaling_lut_y_) == 256, "");
-      memcpy(scaling_lut_u_, scaling_lut_y_, sizeof(scaling_lut_y_));
-      memcpy(scaling_lut_v_, scaling_lut_y_, sizeof(scaling_lut_y_));
+  InitializeScalingLookupTable(params_.num_y_points, params_.point_y_value,
+                               params_.point_y_scaling, scaling_lut_y_);
+  if (!is_monochrome_) {
+    if (params_.chroma_scaling_from_luma) {
+      scaling_lut_u_ = scaling_lut_y_;
+      scaling_lut_v_ = scaling_lut_y_;
     } else {
-      InitializeScalingLookupTable(params.num_u_points, params.point_u_value,
-                                   params.point_u_scaling, scaling_lut_u_);
-      InitializeScalingLookupTable(params.num_v_points, params.point_v_value,
-                                   params.point_v_scaling, scaling_lut_v_);
+      scaling_lut_chroma_buffer_.reset(new (std::nothrow) uint8_t[256 * 2]);
+      if (scaling_lut_chroma_buffer_ == nullptr) return false;
+      scaling_lut_u_ = &scaling_lut_chroma_buffer_[0];
+      scaling_lut_v_ = &scaling_lut_chroma_buffer_[256];
+      InitializeScalingLookupTable(params_.num_u_points, params_.point_u_value,
+                                   params_.point_u_scaling, scaling_lut_u_);
+      InitializeScalingLookupTable(params_.num_v_points, params_.point_v_value,
+                                   params_.point_v_scaling, scaling_lut_v_);
     }
   }
+  return true;
 }
 
 // Section 7.18.3.2.
@@ -816,6 +837,10 @@
     const void* source_plane_v, ptrdiff_t source_stride_v, void* dest_plane_y,
     ptrdiff_t dest_stride_y, void* dest_plane_u, ptrdiff_t dest_stride_u,
     void* dest_plane_v, ptrdiff_t dest_stride_v) {
+  if (!Init()) {
+    LIBGAV1_DLOG(ERROR, "Init() failed.");
+    return false;
+  }
   if (!AllocateNoiseStripes()) {
     LIBGAV1_DLOG(ERROR, "AllocateNoiseStripes() failed.");
     return false;
diff --git a/libgav1/src/dsp/film_grain.h b/libgav1/src/dsp/film_grain.h
index 307dbb2..abc7312 100644
--- a/libgav1/src/dsp/film_grain.h
+++ b/libgav1/src/dsp/film_grain.h
@@ -77,6 +77,8 @@
   using Pixel =
       typename std::conditional<bitdepth == 8, uint8_t, uint16_t>::type;
 
+  bool Init();
+
   // Allocates noise_stripe_, which points to memory owned by noise_buffer_.
   bool AllocateNoiseStripes();
 
@@ -127,8 +129,12 @@
   GrainType v_grain_[kMaxChromaHeight * kMaxChromaWidth];
   // Scaling lookup tables.
   uint8_t scaling_lut_y_[256];
-  uint8_t scaling_lut_u_[256];
-  uint8_t scaling_lut_v_[256];
+  uint8_t* scaling_lut_u_ = nullptr;
+  uint8_t* scaling_lut_v_ = nullptr;
+  // If allocated, this buffer is 256 * 2 bytes long and scaling_lut_u_ and
+  // scaling_lut_v_ point into this buffer. Otherwise, scaling_lut_u_ and
+  // scaling_lut_v_ point to scaling_lut_y_.
+  std::unique_ptr<uint8_t[]> scaling_lut_chroma_buffer_;
 
   // A two-dimensional array of noise data. Generated for each 32 luma sample
   // high stripe of the image. The first dimension is called luma_num. The
diff --git a/libgav1/src/dsp/intra_edge.cc b/libgav1/src/dsp/intra_edge.cc
index a29ce3c..7f0bc4c 100644
--- a/libgav1/src/dsp/intra_edge.cc
+++ b/libgav1/src/dsp/intra_edge.cc
@@ -59,6 +59,7 @@
   dsp->intra_edge_filter = IntraEdgeFilter_C<uint8_t>;
   dsp->intra_edge_upsampler = IntraEdgeUpsampler_C<8, uint8_t>;
 #else  // !LIBGAV1_ENABLE_ALL_DSP_FUNCTIONS
+  static_cast<void>(dsp);
 #ifndef LIBGAV1_Dsp8bpp_IntraEdgeFilter
   dsp->intra_edge_filter = IntraEdgeFilter_C<uint8_t>;
 #endif
@@ -76,6 +77,7 @@
   dsp->intra_edge_filter = IntraEdgeFilter_C<uint16_t>;
   dsp->intra_edge_upsampler = IntraEdgeUpsampler_C<10, uint16_t>;
 #else  // !LIBGAV1_ENABLE_ALL_DSP_FUNCTIONS
+  static_cast<void>(dsp);
 #ifndef LIBGAV1_Dsp10bpp_IntraEdgeFilter
   dsp->intra_edge_filter = IntraEdgeFilter_C<uint16_t>;
 #endif
diff --git a/libgav1/src/dsp/intra_edge.h b/libgav1/src/dsp/intra_edge.h
index 24fc96c..13664c7 100644
--- a/libgav1/src/dsp/intra_edge.h
+++ b/libgav1/src/dsp/intra_edge.h
@@ -1,6 +1,24 @@
 #ifndef LIBGAV1_SRC_DSP_INTRA_EDGE_H_
 #define LIBGAV1_SRC_DSP_INTRA_EDGE_H_
 
+// Pull in LIBGAV1_DspXXX defines representing the implementation status
+// of each function. The resulting value of each can be used by each module to
+// determine whether an implementation is needed at compile time.
+// IWYU pragma: begin_exports
+
+// ARM:
+#include "src/dsp/arm/intra_edge_neon.h"
+
+// x86:
+// Note includes should be sorted in logical order avx2/avx/sse4, etc.
+// The order of includes is important as each tests for a superior version
+// before setting the base.
+// clang-format off
+#include "src/dsp/x86/intra_edge_sse4.h"
+// clang-format on
+
+// IWYU pragma: end_exports
+
 namespace libgav1 {
 namespace dsp {
 
diff --git a/libgav1/src/dsp/intrapred.h b/libgav1/src/dsp/intrapred.h
index 7b6d801..3038372 100644
--- a/libgav1/src/dsp/intrapred.h
+++ b/libgav1/src/dsp/intrapred.h
@@ -3,18 +3,28 @@
 
 // Pull in LIBGAV1_DspXXX defines representing the implementation status
 // of each function. The resulting value of each can be used by each module to
-// determine whether an implementation is needed at compile time. The order of
-// includes is important as each tests for a superior version before setting
-// the base.
+// determine whether an implementation is needed at compile time.
+// IWYU pragma: begin_exports
 
+// ARM:
 #include "src/dsp/arm/intrapred_neon.h"
+
+// x86:
+// Note includes should be sorted in logical order avx2/avx/sse4, etc.
+// The order of includes is important as each tests for a superior version
+// before setting the base.
+// clang-format off
 #include "src/dsp/x86/intrapred_sse4.h"
+// clang-format on
+
+// IWYU pragma: end_exports
 
 namespace libgav1 {
 namespace dsp {
 
-// Initializes Dsp::intra_predictors and Dsp::filter_intra_predictor with base
-// implementations. This function is not thread-safe.
+// Initializes Dsp::intra_predictors, Dsp::directional_intra_predictor_zone*,
+// Dsp::cfl_intra_predictors, Dsp::cfl_subsamplers and
+// Dsp::filter_intra_predictor. This function is not thread-safe.
 void IntraPredInit_C();
 
 }  // namespace dsp
diff --git a/libgav1/src/dsp/inverse_transform.cc b/libgav1/src/dsp/inverse_transform.cc
index 66110af..bd9c324 100644
--- a/libgav1/src/dsp/inverse_transform.cc
+++ b/libgav1/src/dsp/inverse_transform.cc
@@ -17,8 +17,6 @@
 // Include the constants and utility functions inside the anonymous namespace.
 #include "src/dsp/inverse_transform.inc"
 
-constexpr uint8_t kTransformRowShift[kNumTransformSizes] = {
-    0, 0, 1, 0, 1, 1, 2, 1, 1, 2, 1, 2, 2, 1, 2, 1, 2, 1, 2};
 constexpr uint8_t kTransformColumnShift = 4;
 
 int32_t RangeCheckValue(int32_t value, int8_t range) {
@@ -32,7 +30,7 @@
                  value, range);
     assert(min <= value && value <= max);
   }
-#endif  // LIBGAV1_ENABLE_TRANSFORM_RANGE_CHECKING
+#endif  // LIBGAV1_ENABLE_TRANSFORM_RANGE_CHECK
   static_cast<void>(range);
   return value;
 }
@@ -599,9 +597,6 @@
 // because the multiplicative factor of inverse identity transforms is at most
 // 4 (2 bits) and |shift| is always 4.
 
-constexpr int32_t kIdentity4Multiplier /* round(2^12 * sqrt(2)) */ = 5793;
-constexpr int32_t kIdentity16Multiplier /* 2 * round(2^12 * sqrt(2)) */ = 11586;
-
 template <typename Residual>
 void Identity4Row_C(void* dest, const void* source, int8_t shift) {
   assert(shift == 0 || shift == 1);
@@ -732,32 +727,34 @@
   temp[1] = src[3] >> shift;
   temp[0] += temp[2];
   temp[3] -= temp[1];
+  // This signed right shift must be an arithmetic shift.
   Residual e = (temp[0] - temp[3]) >> 1;
-  temp[1] = e - temp[1];
-  temp[2] = e - temp[2];
-  temp[0] -= temp[1];
-  temp[3] += temp[2];
-  memcpy(dst, temp, sizeof(temp));
+  dst[1] = e - temp[1];
+  dst[2] = e - temp[2];
+  dst[0] = temp[0] - dst[1];
+  dst[3] = temp[3] + dst[2];
 }
 
 //------------------------------------------------------------------------------
 // row/column transform loop
-constexpr int16_t kTransformRowMultiplier /* round(2^12 / sqrt(2)) */ = 2896;
 
 using InverseTransform1DFunc = void (*)(void* dst, const void* src,
                                         int8_t range);
 
-template <typename Residual, typename Pixel, Transform1D transform1d_type,
+template <int bitdepth, typename Residual, typename Pixel,
+          Transform1D transform1d_type,
           InverseTransform1DFunc row_transform1d_func,
           InverseTransform1DFunc column_transform1d_func = row_transform1d_func>
 void TransformLoop_C(TransformType tx_type, TransformSize tx_size,
-                     int8_t bitdepth, void* src_buffer, int start_x,
-                     int start_y, void* dst_frame, bool is_row,
-                     int non_zero_coeff_count) {
+                     void* src_buffer, int start_x, int start_y,
+                     void* dst_frame, bool is_row, int non_zero_coeff_count) {
   constexpr bool lossless = transform1d_type == k1DTransformWht;
   constexpr bool is_identity = transform1d_type == k1DTransformIdentity;
-  const int tx_width = kTransformWidth[tx_size];
-  const int tx_height = kTransformHeight[tx_size];
+  // The transform size of the WHT is always 4x4. Setting tx_width and
+  // tx_height to the constant 4 for the WHT speeds the code up.
+  assert(!lossless || tx_size == kTransformSize4x4);
+  const int tx_width = lossless ? 4 : kTransformWidth[tx_size];
+  const int tx_height = lossless ? 4 : kTransformHeight[tx_size];
   const int tx_width_log2 = kTransformWidthLog2[tx_size];
   const int tx_height_log2 = kTransformHeightLog2[tx_size];
   auto* frame = reinterpret_cast<Array2DView<Pixel>*>(dst_frame);
@@ -825,9 +822,9 @@
   // transforms, this will be equal to the clamping range.
   const int8_t column_clamp_range = lossless ? 0 : std::max(bitdepth + 6, 16);
   const bool flip_rows = transform1d_type == k1DTransformAdst &&
-                         ((1U << tx_type) & kTransformFlipRowsMask) != 0;
+                         kTransformFlipRowsMask.Contains(tx_type);
   const bool flip_columns =
-      !lossless && ((1U << tx_type) & kTransformFlipColumnsMask) != 0;
+      !lossless && kTransformFlipColumnsMask.Contains(tx_type);
   const int min_value = 0;
   const int max_value = (1 << bitdepth) - 1;
   // Note: 64 is the maximum size of a 1D transform buffer (the largest
@@ -858,45 +855,54 @@
 
 //------------------------------------------------------------------------------
 
-template <typename Residual, typename Pixel>
+template <int bitdepth, typename Residual, typename Pixel>
 void InitAll(Dsp* const dsp) {
   // Maximum transform size for Dct is 64.
   dsp->inverse_transforms[k1DTransformSize4][k1DTransformDct] =
-      TransformLoop_C<Residual, Pixel, k1DTransformDct, Dct_C<Residual, 2>>;
+      TransformLoop_C<bitdepth, Residual, Pixel, k1DTransformDct,
+                      Dct_C<Residual, 2>>;
   dsp->inverse_transforms[k1DTransformSize8][k1DTransformDct] =
-      TransformLoop_C<Residual, Pixel, k1DTransformDct, Dct_C<Residual, 3>>;
+      TransformLoop_C<bitdepth, Residual, Pixel, k1DTransformDct,
+                      Dct_C<Residual, 3>>;
   dsp->inverse_transforms[k1DTransformSize16][k1DTransformDct] =
-      TransformLoop_C<Residual, Pixel, k1DTransformDct, Dct_C<Residual, 4>>;
+      TransformLoop_C<bitdepth, Residual, Pixel, k1DTransformDct,
+                      Dct_C<Residual, 4>>;
   dsp->inverse_transforms[k1DTransformSize32][k1DTransformDct] =
-      TransformLoop_C<Residual, Pixel, k1DTransformDct, Dct_C<Residual, 5>>;
+      TransformLoop_C<bitdepth, Residual, Pixel, k1DTransformDct,
+                      Dct_C<Residual, 5>>;
   dsp->inverse_transforms[k1DTransformSize64][k1DTransformDct] =
-      TransformLoop_C<Residual, Pixel, k1DTransformDct, Dct_C<Residual, 6>>;
+      TransformLoop_C<bitdepth, Residual, Pixel, k1DTransformDct,
+                      Dct_C<Residual, 6>>;
 
   // Maximum transform size for Adst is 16.
   dsp->inverse_transforms[k1DTransformSize4][k1DTransformAdst] =
-      TransformLoop_C<Residual, Pixel, k1DTransformAdst, Adst4_C<Residual>>;
+      TransformLoop_C<bitdepth, Residual, Pixel, k1DTransformAdst,
+                      Adst4_C<Residual>>;
   dsp->inverse_transforms[k1DTransformSize8][k1DTransformAdst] =
-      TransformLoop_C<Residual, Pixel, k1DTransformAdst, Adst8_C<Residual>>;
+      TransformLoop_C<bitdepth, Residual, Pixel, k1DTransformAdst,
+                      Adst8_C<Residual>>;
   dsp->inverse_transforms[k1DTransformSize16][k1DTransformAdst] =
-      TransformLoop_C<Residual, Pixel, k1DTransformAdst, Adst16_C<Residual>>;
+      TransformLoop_C<bitdepth, Residual, Pixel, k1DTransformAdst,
+                      Adst16_C<Residual>>;
 
   // Maximum transform size for Identity transform is 32.
   dsp->inverse_transforms[k1DTransformSize4][k1DTransformIdentity] =
-      TransformLoop_C<Residual, Pixel, k1DTransformIdentity,
+      TransformLoop_C<bitdepth, Residual, Pixel, k1DTransformIdentity,
                       Identity4Row_C<Residual>, Identity4Column_C<Residual>>;
   dsp->inverse_transforms[k1DTransformSize8][k1DTransformIdentity] =
-      TransformLoop_C<Residual, Pixel, k1DTransformIdentity,
+      TransformLoop_C<bitdepth, Residual, Pixel, k1DTransformIdentity,
                       Identity8Row_C<Residual>, Identity8Column_C<Residual>>;
   dsp->inverse_transforms[k1DTransformSize16][k1DTransformIdentity] =
-      TransformLoop_C<Residual, Pixel, k1DTransformIdentity,
+      TransformLoop_C<bitdepth, Residual, Pixel, k1DTransformIdentity,
                       Identity16Row_C<Residual>, Identity16Column_C<Residual>>;
   dsp->inverse_transforms[k1DTransformSize32][k1DTransformIdentity] =
-      TransformLoop_C<Residual, Pixel, k1DTransformIdentity,
+      TransformLoop_C<bitdepth, Residual, Pixel, k1DTransformIdentity,
                       Identity32Row_C<Residual>, Identity32Column_C<Residual>>;
 
   // Maximum transform size for Wht is 4.
   dsp->inverse_transforms[k1DTransformSize4][k1DTransformWht] =
-      TransformLoop_C<Residual, Pixel, k1DTransformWht, Wht4_C<Residual>>;
+      TransformLoop_C<bitdepth, Residual, Pixel, k1DTransformWht,
+                      Wht4_C<Residual>>;
 }
 
 void Init8bpp() {
@@ -908,63 +914,63 @@
     }
   }
 #if LIBGAV1_ENABLE_ALL_DSP_FUNCTIONS
-  InitAll<int16_t, uint8_t>(dsp);
+  InitAll<8, int16_t, uint8_t>(dsp);
 #else  // !LIBGAV1_ENABLE_ALL_DSP_FUNCTIONS
 #ifndef LIBGAV1_Dsp8bpp_1DTransformSize4_1DTransformDct
   dsp->inverse_transforms[k1DTransformSize4][k1DTransformDct] =
-      TransformLoop_C<int16_t, uint8_t, k1DTransformDct, Dct_C<int16_t, 2>>;
+      TransformLoop_C<8, int16_t, uint8_t, k1DTransformDct, Dct_C<int16_t, 2>>;
 #endif
 #ifndef LIBGAV1_Dsp8bpp_1DTransformSize8_1DTransformDct
   dsp->inverse_transforms[k1DTransformSize8][k1DTransformDct] =
-      TransformLoop_C<int16_t, uint8_t, k1DTransformDct, Dct_C<int16_t, 3>>;
+      TransformLoop_C<8, int16_t, uint8_t, k1DTransformDct, Dct_C<int16_t, 3>>;
 #endif
 #ifndef LIBGAV1_Dsp8bpp_1DTransformSize16_1DTransformDct
   dsp->inverse_transforms[k1DTransformSize16][k1DTransformDct] =
-      TransformLoop_C<int16_t, uint8_t, k1DTransformDct, Dct_C<int16_t, 4>>;
+      TransformLoop_C<8, int16_t, uint8_t, k1DTransformDct, Dct_C<int16_t, 4>>;
 #endif
 #ifndef LIBGAV1_Dsp8bpp_1DTransformSize32_1DTransformDct
   dsp->inverse_transforms[k1DTransformSize32][k1DTransformDct] =
-      TransformLoop_C<int16_t, uint8_t, k1DTransformDct, Dct_C<int16_t, 5>>;
+      TransformLoop_C<8, int16_t, uint8_t, k1DTransformDct, Dct_C<int16_t, 5>>;
 #endif
 #ifndef LIBGAV1_Dsp8bpp_1DTransformSize64_1DTransformDct
   dsp->inverse_transforms[k1DTransformSize64][k1DTransformDct] =
-      TransformLoop_C<int16_t, uint8_t, k1DTransformDct, Dct_C<int16_t, 6>>;
+      TransformLoop_C<8, int16_t, uint8_t, k1DTransformDct, Dct_C<int16_t, 6>>;
 #endif
 #ifndef LIBGAV1_Dsp8bpp_1DTransformSize4_1DTransformAdst
   dsp->inverse_transforms[k1DTransformSize4][k1DTransformAdst] =
-      TransformLoop_C<int16_t, uint8_t, k1DTransformAdst, Adst4_C<int16_t>>;
+      TransformLoop_C<8, int16_t, uint8_t, k1DTransformAdst, Adst4_C<int16_t>>;
 #endif
 #ifndef LIBGAV1_Dsp8bpp_1DTransformSize8_1DTransformAdst
   dsp->inverse_transforms[k1DTransformSize8][k1DTransformAdst] =
-      TransformLoop_C<int16_t, uint8_t, k1DTransformAdst, Adst8_C<int16_t>>;
+      TransformLoop_C<8, int16_t, uint8_t, k1DTransformAdst, Adst8_C<int16_t>>;
 #endif
 #ifndef LIBGAV1_Dsp8bpp_1DTransformSize16_1DTransformAdst
   dsp->inverse_transforms[k1DTransformSize16][k1DTransformAdst] =
-      TransformLoop_C<int16_t, uint8_t, k1DTransformAdst, Adst16_C<int16_t>>;
+      TransformLoop_C<8, int16_t, uint8_t, k1DTransformAdst, Adst16_C<int16_t>>;
 #endif
 #ifndef LIBGAV1_Dsp8bpp_1DTransformSize4_1DTransformIdentity
   dsp->inverse_transforms[k1DTransformSize4][k1DTransformIdentity] =
-      TransformLoop_C<int16_t, uint8_t, k1DTransformIdentity,
+      TransformLoop_C<8, int16_t, uint8_t, k1DTransformIdentity,
                       Identity4Row_C<int16_t>, Identity4Column_C<int16_t>>;
 #endif
 #ifndef LIBGAV1_Dsp8bpp_1DTransformSize8_1DTransformIdentity
   dsp->inverse_transforms[k1DTransformSize8][k1DTransformIdentity] =
-      TransformLoop_C<int16_t, uint8_t, k1DTransformIdentity,
+      TransformLoop_C<8, int16_t, uint8_t, k1DTransformIdentity,
                       Identity8Row_C<int16_t>, Identity8Column_C<int16_t>>;
 #endif
 #ifndef LIBGAV1_Dsp8bpp_1DTransformSize16_1DTransformIdentity
   dsp->inverse_transforms[k1DTransformSize16][k1DTransformIdentity] =
-      TransformLoop_C<int16_t, uint8_t, k1DTransformIdentity,
+      TransformLoop_C<8, int16_t, uint8_t, k1DTransformIdentity,
                       Identity16Row_C<int16_t>, Identity16Column_C<int16_t>>;
 #endif
 #ifndef LIBGAV1_Dsp8bpp_1DTransformSize32_1DTransformIdentity
   dsp->inverse_transforms[k1DTransformSize32][k1DTransformIdentity] =
-      TransformLoop_C<int16_t, uint8_t, k1DTransformIdentity,
+      TransformLoop_C<8, int16_t, uint8_t, k1DTransformIdentity,
                       Identity32Row_C<int16_t>, Identity32Column_C<int16_t>>;
 #endif
 #ifndef LIBGAV1_Dsp8bpp_1DTransformSize4_1DTransformWht
   dsp->inverse_transforms[k1DTransformSize4][k1DTransformWht] =
-      TransformLoop_C<int16_t, uint8_t, k1DTransformWht, Wht4_C<int16_t>>;
+      TransformLoop_C<8, int16_t, uint8_t, k1DTransformWht, Wht4_C<int16_t>>;
 #endif
 #endif  // LIBGAV1_ENABLE_ALL_DSP_FUNCTIONS
 }
@@ -979,63 +985,71 @@
     }
   }
 #if LIBGAV1_ENABLE_ALL_DSP_FUNCTIONS
-  InitAll<int32_t, uint16_t>(dsp);
+  InitAll<10, int32_t, uint16_t>(dsp);
 #else  // !LIBGAV1_ENABLE_ALL_DSP_FUNCTIONS
 #ifndef LIBGAV1_Dsp10bpp_1DTransformSize4_1DTransformDct
   dsp->inverse_transforms[k1DTransformSize4][k1DTransformDct] =
-      TransformLoop_C<int32_t, uint16_t, k1DTransformDct, Dct_C<int32_t, 2>>;
+      TransformLoop_C<10, int32_t, uint16_t, k1DTransformDct,
+                      Dct_C<int32_t, 2>>;
 #endif
 #ifndef LIBGAV1_Dsp10bpp_1DTransformSize8_1DTransformDct
   dsp->inverse_transforms[k1DTransformSize8][k1DTransformDct] =
-      TransformLoop_C<int32_t, uint16_t, k1DTransformDct, Dct_C<int32_t, 3>>;
+      TransformLoop_C<10, int32_t, uint16_t, k1DTransformDct,
+                      Dct_C<int32_t, 3>>;
 #endif
 #ifndef LIBGAV1_Dsp10bpp_1DTransformSize16_1DTransformDct
   dsp->inverse_transforms[k1DTransformSize16][k1DTransformDct] =
-      TransformLoop_C<int32_t, uint16_t, k1DTransformDct, Dct_C<int32_t, 4>>;
+      TransformLoop_C<10, int32_t, uint16_t, k1DTransformDct,
+                      Dct_C<int32_t, 4>>;
 #endif
 #ifndef LIBGAV1_Dsp10bpp_1DTransformSize32_1DTransformDct
   dsp->inverse_transforms[k1DTransformSize32][k1DTransformDct] =
-      TransformLoop_C<int32_t, uint16_t, k1DTransformDct, Dct_C<int32_t, 5>>;
+      TransformLoop_C<10, int32_t, uint16_t, k1DTransformDct,
+                      Dct_C<int32_t, 5>>;
 #endif
 #ifndef LIBGAV1_Dsp10bpp_1DTransformSize64_1DTransformDct
   dsp->inverse_transforms[k1DTransformSize64][k1DTransformDct] =
-      TransformLoop_C<int32_t, uint16_t, k1DTransformDct, Dct_C<int32_t, 6>>;
+      TransformLoop_C<10, int32_t, uint16_t, k1DTransformDct,
+                      Dct_C<int32_t, 6>>;
 #endif
 #ifndef LIBGAV1_Dsp10bpp_1DTransformSize4_1DTransformAdst
   dsp->inverse_transforms[k1DTransformSize4][k1DTransformAdst] =
-      TransformLoop_C<int32_t, uint16_t, k1DTransformAdst, Adst4_C<int32_t>>;
+      TransformLoop_C<10, int32_t, uint16_t, k1DTransformAdst,
+                      Adst4_C<int32_t>>;
 #endif
 #ifndef LIBGAV1_Dsp10bpp_1DTransformSize8_1DTransformAdst
   dsp->inverse_transforms[k1DTransformSize8][k1DTransformAdst] =
-      TransformLoop_C<int32_t, uint16_t, k1DTransformAdst, Adst8_C<int32_t>>;
+      TransformLoop_C<10, int32_t, uint16_t, k1DTransformAdst,
+                      Adst8_C<int32_t>>;
 #endif
 #ifndef LIBGAV1_Dsp10bpp_1DTransformSize16_1DTransformAdst
   dsp->inverse_transforms[k1DTransformSize16][k1DTransformAdst] =
-      TransformLoop_C<int32_t, uint16_t, k1DTransformAdst, Adst16_C<int32_t>>;
+      TransformLoop_C<10, int32_t, uint16_t, k1DTransformAdst,
+                      Adst16_C<int32_t>>;
 #endif
 #ifndef LIBGAV1_Dsp10bpp_1DTransformSize4_1DTransformIdentity
   dsp->inverse_transforms[k1DTransformSize4][k1DTransformIdentity] =
-      TransformLoop_C<int32_t, uint16_t, k1DTransformIdentity,
+      TransformLoop_C<10, int32_t, uint16_t, k1DTransformIdentity,
                       Identity4Row_C<int32_t>, Identity4Column_C<int32_t>>;
 #endif
 #ifndef LIBGAV1_Dsp10bpp_1DTransformSize8_1DTransformIdentity
   dsp->inverse_transforms[k1DTransformSize8][k1DTransformIdentity] =
-      TransformLoop_C<int32_t, uint16_t, k1DTransformIdentity,
+      TransformLoop_C<10, int32_t, uint16_t, k1DTransformIdentity,
                       Identity8Row_C<int32_t>, Identity8Column_C<int32_t>>;
 #endif
 #ifndef LIBGAV1_Dsp10bpp_1DTransformSize16_1DTransformIdentity
   dsp->inverse_transforms[k1DTransformSize16][k1DTransformIdentity] =
-      TransformLoop_C<int32_t, uint16_t, k1DTransformIdentity,
+      TransformLoop_C<10, int32_t, uint16_t, k1DTransformIdentity,
                       Identity16Row_C<int32_t>, Identity16Column_C<int32_t>>;
 #endif
 #ifndef LIBGAV1_Dsp10bpp_1DTransformSize32_1DTransformIdentity
   dsp->inverse_transforms[k1DTransformSize32][k1DTransformIdentity] =
-      TransformLoop_C<int32_t, uint16_t, k1DTransformIdentity,
+      TransformLoop_C<10, int32_t, uint16_t, k1DTransformIdentity,
                       Identity32Row_C<int32_t>, Identity32Column_C<int32_t>>;
 #endif
 #ifndef LIBGAV1_Dsp10bpp_1DTransformSize4_1DTransformWht
   dsp->inverse_transforms[k1DTransformSize4][k1DTransformWht] =
-      TransformLoop_C<int32_t, uint16_t, k1DTransformWht, Wht4_C<int32_t>>;
+      TransformLoop_C<10, int32_t, uint16_t, k1DTransformWht, Wht4_C<int32_t>>;
 #endif
 #endif  // LIBGAV1_ENABLE_ALL_DSP_FUNCTIONS
 }
@@ -1048,6 +1062,11 @@
 #if LIBGAV1_MAX_BITDEPTH >= 10
   Init10bpp();
 #endif
+
+  // Local functions that may be unused depending on the optimizations
+  // available.
+  static_cast<void>(RangeCheckValue);
+  static_cast<void>(kBitReverseLookup);
 }
 
 }  // namespace dsp
diff --git a/libgav1/src/dsp/inverse_transform.h b/libgav1/src/dsp/inverse_transform.h
index 63bc3c8..28faec7 100644
--- a/libgav1/src/dsp/inverse_transform.h
+++ b/libgav1/src/dsp/inverse_transform.h
@@ -1,6 +1,24 @@
 #ifndef LIBGAV1_SRC_DSP_INVERSE_TRANSFORM_H_
 #define LIBGAV1_SRC_DSP_INVERSE_TRANSFORM_H_
 
+// Pull in LIBGAV1_DspXXX defines representing the implementation status
+// of each function. The resulting value of each can be used by each module to
+// determine whether an implementation is needed at compile time.
+// IWYU pragma: begin_exports
+
+// ARM:
+#include "src/dsp/arm/inverse_transform_neon.h"
+
+// x86:
+// Note includes should be sorted in logical order avx2/avx/sse4, etc.
+// The order of includes is important as each tests for a superior version
+// before setting the base.
+// clang-format off
+#include "src/dsp/x86/inverse_transform_sse4.h"
+// clang-format on
+
+// IWYU pragma: end_exports
+
 namespace libgav1 {
 namespace dsp {
 
diff --git a/libgav1/src/dsp/inverse_transform.inc b/libgav1/src/dsp/inverse_transform.inc
index 3b5b052..3bffa23 100644
--- a/libgav1/src/dsp/inverse_transform.inc
+++ b/libgav1/src/dsp/inverse_transform.inc
@@ -35,3 +35,16 @@
 // The value for index i is derived as:
 // round(sqrt(2) * sin(i * pi / 9) * 2 / 3 * (1 << 12)).
 constexpr int16_t kAdst4Multiplier[4] = {1321, 2482, 3344, 3803};
+
+constexpr uint8_t kTransformRowShift[kNumTransformSizes] = {
+    0, 0, 1, 0, 1, 1, 2, 1, 1, 2, 1, 2, 2, 1, 2, 1, 2, 1, 2};
+
+constexpr bool kShouldRound[kNumTransformSizes] = {
+    false, true,  false, true, false, true, false, false, true, false,
+    true,  false, false, true, false, true, false, true,  false};
+
+constexpr int16_t kIdentity4Multiplier /* round(2^12 * sqrt(2)) */ = 0x16A1;
+constexpr int16_t kIdentity4MultiplierFraction /* round(2^12 * (sqrt(2) - 1))*/
+    = 0x6A1;
+constexpr int16_t kIdentity16Multiplier /* 2 * round(2^12 * sqrt(2)) */ = 11586;
+constexpr int16_t kTransformRowMultiplier /* round(2^12 / sqrt(2)) */ = 2896;
diff --git a/libgav1/src/dsp/loop_filter.h b/libgav1/src/dsp/loop_filter.h
index ee33b91..c9e602c 100644
--- a/libgav1/src/dsp/loop_filter.h
+++ b/libgav1/src/dsp/loop_filter.h
@@ -3,18 +3,26 @@
 
 // Pull in LIBGAV1_DspXXX defines representing the implementation status
 // of each function. The resulting value of each can be used by each module to
-// determine whether an implementation is needed at compile time. The order of
-// includes is important as each tests for a superior version before setting
-// the base.
+// determine whether an implementation is needed at compile time.
+// IWYU pragma: begin_exports
 
+// ARM:
 #include "src/dsp/arm/loop_filter_neon.h"
+
+// x86:
+// Note includes should be sorted in logical order avx2/avx/sse4, etc.
+// The order of includes is important as each tests for a superior version
+// before setting the base.
+// clang-format off
 #include "src/dsp/x86/loop_filter_sse4.h"
+// clang-format on
+
+// IWYU pragma: end_exports
 
 namespace libgav1 {
 namespace dsp {
 
-// Initializes Dsp::loop_filters with base implementations. This function
-// is not thread-safe.
+// Initializes Dsp::loop_filters. This function is not thread-safe.
 void LoopFilterInit_C();
 
 }  // namespace dsp
diff --git a/libgav1/src/dsp/loop_restoration.cc b/libgav1/src/dsp/loop_restoration.cc
index 5f092c5..0a07c0e 100644
--- a/libgav1/src/dsp/loop_restoration.cc
+++ b/libgav1/src/dsp/loop_restoration.cc
@@ -90,15 +90,14 @@
     const RestorationUnitInfo& restoration_info, ptrdiff_t source_stride,
     ptrdiff_t dest_stride, int width, int height,
     RestorationBuffer* const buffer) {
-  const int* const inter_round_bits = buffer->inter_round_bits;
-  if (bitdepth == 12) {
-    assert(inter_round_bits[0] == 5 && inter_round_bits[1] == 9);
-  } else {
-    assert(inter_round_bits[0] == 3 && inter_round_bits[1] == 11);
-  }
+  constexpr int kRoundBitsHorizontal = (bitdepth == 12)
+                                           ? kInterRoundBitsHorizontal12bpp
+                                           : kInterRoundBitsHorizontal;
+  constexpr int kRoundBitsVertical =
+      (bitdepth == 12) ? kInterRoundBitsVertical12bpp : kInterRoundBitsVertical;
   int16_t filter[kSubPixelTaps - 1];
   const int limit =
-      (1 << (bitdepth + 1 + kWienerFilterBits - inter_round_bits[0])) - 1;
+      (1 << (bitdepth + 1 + kWienerFilterBits - kRoundBitsHorizontal)) - 1;
   const auto* src = static_cast<const Pixel*>(source);
   auto* dst = static_cast<Pixel*>(dest);
   source_stride /= sizeof(Pixel);
@@ -117,10 +116,7 @@
       for (int k = 0; k < kSubPixelTaps - 1; ++k) {
         sum += filter[k] * src[x + k];
       }
-      const int rounded_sum = RightShiftWithRounding(sum, inter_round_bits[0]);
-      // TODO(chengchen): make sure the horizontal and vertical rounding offset
-      // is correct and whether they ensure rounded_sum is non-negative.
-      // If yes, replace Clip3() with std::min().
+      const int rounded_sum = RightShiftWithRounding(sum, kRoundBitsHorizontal);
       wiener_buffer[x] = static_cast<uint16_t>(Clip3(rounded_sum, 0, limit));
     }
     src += source_stride;
@@ -129,7 +125,7 @@
   wiener_buffer = buffer->wiener_buffer;
   // vertical filtering.
   PopulateWienerCoefficients(restoration_info, WienerInfo::kVertical, filter);
-  const int vertical_rounding = -(1 << (bitdepth + inter_round_bits[1] - 1));
+  const int vertical_rounding = -(1 << (bitdepth + kRoundBitsVertical - 1));
   for (int y = 0; y < height; ++y) {
     for (int x = 0; x < width; ++x) {
       // sum needs 32 bits.
@@ -137,7 +133,7 @@
       for (int k = 0; k < kSubPixelTaps - 1; ++k) {
         sum += filter[k] * wiener_buffer[k * buffer_stride + x];
       }
-      const int rounded_sum = RightShiftWithRounding(sum, inter_round_bits[1]);
+      const int rounded_sum = RightShiftWithRounding(sum, kRoundBitsVertical);
       dst[x] = static_cast<Pixel>(Clip3(rounded_sum, 0, (1 << bitdepth) - 1));
     }
     dst += dest_stride;
diff --git a/libgav1/src/dsp/loop_restoration.h b/libgav1/src/dsp/loop_restoration.h
index 59c51a3..0fa4e0f 100644
--- a/libgav1/src/dsp/loop_restoration.h
+++ b/libgav1/src/dsp/loop_restoration.h
@@ -1,14 +1,28 @@
 #ifndef LIBGAV1_SRC_DSP_LOOP_RESTORATION_H_
 #define LIBGAV1_SRC_DSP_LOOP_RESTORATION_H_
 
+// Pull in LIBGAV1_DspXXX defines representing the implementation status
+// of each function. The resulting value of each can be used by each module to
+// determine whether an implementation is needed at compile time.
+// IWYU pragma: begin_exports
+
+// ARM:
 #include "src/dsp/arm/loop_restoration_neon.h"
+
+// x86:
+// Note includes should be sorted in logical order avx2/avx/sse4, etc.
+// The order of includes is important as each tests for a superior version
+// before setting the base.
+// clang-format off
 #include "src/dsp/x86/loop_restoration_sse4.h"
+// clang-format on
+
+// IWYU pragma: end_exports
 
 namespace libgav1 {
 namespace dsp {
 
-// Initializes Dsp::loop_restorations with base implementations. This function
-// is not thread-safe.
+// Initializes Dsp::loop_restorations. This function is not thread-safe.
 void LoopRestorationInit_C();
 
 }  // namespace dsp
diff --git a/libgav1/src/dsp/mask_blend.cc b/libgav1/src/dsp/mask_blend.cc
new file mode 100644
index 0000000..a6e6805
--- /dev/null
+++ b/libgav1/src/dsp/mask_blend.cc
@@ -0,0 +1,161 @@
+#include "src/dsp/mask_blend.h"
+
+#include <cassert>
+#include <cstddef>
+#include <cstdint>
+
+#include "src/dsp/dsp.h"
+#include "src/utils/common.h"
+
+namespace libgav1 {
+namespace dsp {
+namespace {
+
+template <int bitdepth, typename Pixel, bool is_inter_intra, int subsampling_x,
+          int subsampling_y>
+void MaskBlend_C(const uint16_t* prediction_0,
+                 const ptrdiff_t prediction_stride_0,
+                 const uint16_t* prediction_1,
+                 const ptrdiff_t prediction_stride_1, const uint8_t* mask,
+                 const ptrdiff_t mask_stride, const int width, const int height,
+                 void* dest, const ptrdiff_t dest_stride) {
+  assert(mask != nullptr);
+  auto* dst = static_cast<Pixel*>(dest);
+  const ptrdiff_t dst_stride = dest_stride / sizeof(Pixel);
+  constexpr int step_y = subsampling_y ? 2 : 1;
+  const uint8_t* mask_next_row = mask + mask_stride;
+  // An offset to cancel offsets used in single predictor generation that
+  // make intermediate computations non negative.
+  const int single_round_offset = (1 << bitdepth) + (1 << (bitdepth - 1));
+  // An offset to cancel offsets used in compound predictor generation that
+  // make intermediate computations non negative.
+  const int compound_round_offset =
+      (1 << (bitdepth + 4)) + (1 << (bitdepth + 3));
+  // 7.11.3.2 Rounding variables derivation process
+  //   2 * FILTER_BITS(7) - (InterRound0(3|5) + InterRound1(7))
+  constexpr int inter_post_round_bits = (bitdepth == 12) ? 2 : 4;
+  for (int y = 0; y < height; ++y) {
+    for (int x = 0; x < width; ++x) {
+      uint8_t mask_value;
+      if ((subsampling_x | subsampling_y) == 0) {
+        mask_value = mask[x];
+      } else if (subsampling_x == 1 && subsampling_y == 0) {
+        mask_value = static_cast<uint8_t>(RightShiftWithRounding(
+            mask[MultiplyBy2(x)] + mask[MultiplyBy2(x) + 1], 1));
+      } else {
+        assert(subsampling_x == 1 && subsampling_y == 1);
+        mask_value = static_cast<uint8_t>(RightShiftWithRounding(
+            mask[MultiplyBy2(x)] + mask[MultiplyBy2(x) + 1] +
+                mask_next_row[MultiplyBy2(x)] +
+                mask_next_row[MultiplyBy2(x) + 1],
+            2));
+      }
+
+      if (is_inter_intra) {
+        // In inter intra prediction mode, the intra prediction (prediction_1)
+        // values are valid pixel values: [0, (1 << bitdepth) - 1].
+        // While the inter prediction values come from subpixel prediction
+        // from another frame, which involves interpolation and rounding.
+        // Therefore prediction_0 has to be clipped.
+        dst[x] = static_cast<Pixel>(RightShiftWithRounding(
+            mask_value * prediction_1[x] +
+                (64 - mask_value) * Clip3(prediction_0[x] - single_round_offset,
+                                          0, (1 << bitdepth) - 1),
+            6));
+      } else {
+        int res = (mask_value * prediction_0[x] +
+                   (64 - mask_value) * prediction_1[x]) >>
+                  6;
+        res -= compound_round_offset;
+        dst[x] = static_cast<Pixel>(
+            Clip3(RightShiftWithRounding(res, inter_post_round_bits), 0,
+                  (1 << bitdepth) - 1));
+      }
+    }
+    dst += dst_stride;
+    mask += mask_stride * step_y;
+    mask_next_row += mask_stride * step_y;
+    prediction_0 += prediction_stride_0;
+    prediction_1 += prediction_stride_1;
+  }
+}
+
+void Init8bpp() {
+  Dsp* const dsp = dsp_internal::GetWritableDspTable(8);
+  assert(dsp != nullptr);
+#if LIBGAV1_ENABLE_ALL_DSP_FUNCTIONS
+  dsp->mask_blend[0][0] = MaskBlend_C<8, uint8_t, false, 0, 0>;
+  dsp->mask_blend[1][0] = MaskBlend_C<8, uint8_t, false, 1, 0>;
+  dsp->mask_blend[2][0] = MaskBlend_C<8, uint8_t, false, 1, 1>;
+  dsp->mask_blend[0][1] = MaskBlend_C<8, uint8_t, true, 0, 0>;
+  dsp->mask_blend[1][1] = MaskBlend_C<8, uint8_t, true, 1, 0>;
+  dsp->mask_blend[2][1] = MaskBlend_C<8, uint8_t, true, 1, 1>;
+#else  // !LIBGAV1_ENABLE_ALL_DSP_FUNCTIONS
+  static_cast<void>(dsp);
+#ifndef LIBGAV1_Dsp8bpp_MaskBlend444
+  dsp->mask_blend[0][0] = MaskBlend_C<8, uint8_t, false, 0, 0>;
+#endif
+#ifndef LIBGAV1_Dsp8bpp_MaskBlend422
+  dsp->mask_blend[1][0] = MaskBlend_C<8, uint8_t, false, 1, 0>;
+#endif
+#ifndef LIBGAV1_Dsp8bpp_MaskBlend420
+  dsp->mask_blend[2][0] = MaskBlend_C<8, uint8_t, false, 1, 1>;
+#endif
+#ifndef LIBGAV1_Dsp8bpp_MaskBlendInterIntra444
+  dsp->mask_blend[0][1] = MaskBlend_C<8, uint8_t, true, 0, 0>;
+#endif
+#ifndef LIBGAV1_Dsp8bpp_MaskBlendInterIntra422
+  dsp->mask_blend[1][1] = MaskBlend_C<8, uint8_t, true, 1, 0>;
+#endif
+#ifndef LIBGAV1_Dsp8bpp_MaskBlendInterIntra420
+  dsp->mask_blend[2][1] = MaskBlend_C<8, uint8_t, true, 1, 1>;
+#endif
+#endif  // LIBGAV1_ENABLE_ALL_DSP_FUNCTIONS
+}
+
+#if LIBGAV1_MAX_BITDEPTH >= 10
+void Init10bpp() {
+  Dsp* const dsp = dsp_internal::GetWritableDspTable(10);
+  assert(dsp != nullptr);
+#if LIBGAV1_ENABLE_ALL_DSP_FUNCTIONS
+  dsp->mask_blend[0][0] = MaskBlend_C<10, uint16_t, false, 0, 0>;
+  dsp->mask_blend[1][0] = MaskBlend_C<10, uint16_t, false, 1, 0>;
+  dsp->mask_blend[2][0] = MaskBlend_C<10, uint16_t, false, 1, 1>;
+  dsp->mask_blend[0][1] = MaskBlend_C<10, uint16_t, true, 0, 0>;
+  dsp->mask_blend[1][1] = MaskBlend_C<10, uint16_t, true, 1, 0>;
+  dsp->mask_blend[2][1] = MaskBlend_C<10, uint16_t, true, 1, 1>;
+#else  // !LIBGAV1_ENABLE_ALL_DSP_FUNCTIONS
+  static_cast<void>(dsp);
+#ifndef LIBGAV1_Dsp10bpp_MaskBlend444
+  dsp->mask_blend[0][0] = MaskBlend_C<10, uint16_t, false, 0, 0>;
+#endif
+#ifndef LIBGAV1_Dsp10bpp_MaskBlend422
+  dsp->mask_blend[1][0] = MaskBlend_C<10, uint16_t, false, 1, 0>;
+#endif
+#ifndef LIBGAV1_Dsp10bpp_MaskBlend420
+  dsp->mask_blend[2][0] = MaskBlend_C<10, uint16_t, false, 1, 1>;
+#endif
+#ifndef LIBGAV1_Dsp10bpp_MaskBlendInterIntra444
+  dsp->mask_blend[0][1] = MaskBlend_C<10, uint16_t, true, 0, 0>;
+#endif
+#ifndef LIBGAV1_Dsp10bpp_MaskBlendInterIntra422
+  dsp->mask_blend[1][1] = MaskBlend_C<10, uint16_t, true, 1, 0>;
+#endif
+#ifndef LIBGAV1_Dsp10bpp_MaskBlendInterIntra420
+  dsp->mask_blend[2][1] = MaskBlend_C<10, uint16_t, true, 1, 1>;
+#endif
+#endif  // LIBGAV1_ENABLE_ALL_DSP_FUNCTIONS
+}
+#endif
+
+}  // namespace
+
+void MaskBlendInit_C() {
+  Init8bpp();
+#if LIBGAV1_MAX_BITDEPTH >= 10
+  Init10bpp();
+#endif
+}
+
+}  // namespace dsp
+}  // namespace libgav1
diff --git a/libgav1/src/dsp/mask_blend.h b/libgav1/src/dsp/mask_blend.h
new file mode 100644
index 0000000..5c53270
--- /dev/null
+++ b/libgav1/src/dsp/mask_blend.h
@@ -0,0 +1,23 @@
+#ifndef LIBGAV1_SRC_DSP_MASK_BLEND_H_
+#define LIBGAV1_SRC_DSP_MASK_BLEND_H_
+
+// Pull in LIBGAV1_DspXXX defines representing the implementation status
+// of each function. The resulting value of each can be used by each module to
+// determine whether an implementation is needed at compile time.
+// IWYU pragma: begin_exports
+
+// ARM:
+#include "src/dsp/arm/mask_blend_neon.h"
+
+// IWYU pragma: end_exports
+
+namespace libgav1 {
+namespace dsp {
+
+// Initializes Dsp::mask_blend. This function is not thread-safe.
+void MaskBlendInit_C();
+
+}  // namespace dsp
+}  // namespace libgav1
+
+#endif  // LIBGAV1_SRC_DSP_MASK_BLEND_H_
diff --git a/libgav1/src/dsp/mask_blending.cc b/libgav1/src/dsp/mask_blending.cc
deleted file mode 100644
index b250d7c..0000000
--- a/libgav1/src/dsp/mask_blending.cc
+++ /dev/null
@@ -1,108 +0,0 @@
-#include "src/dsp/mask_blending.h"
-
-#include <cassert>
-#include <cstddef>
-#include <cstdint>
-
-#include "src/dsp/dsp.h"
-#include "src/utils/common.h"
-
-namespace libgav1 {
-namespace dsp {
-namespace {
-
-template <int bitdepth, typename Pixel>
-void MaskBlending_C(
-    const uint16_t* prediction_0, const ptrdiff_t prediction_stride_0,
-    const uint16_t* prediction_1, const ptrdiff_t prediction_stride_1,
-    const uint8_t* mask, const ptrdiff_t mask_stride, const int width,
-    const int height, const int subsampling_x, const int subsampling_y,
-    const bool is_inter_intra, const bool is_wedge_inter_intra,
-    const int inter_post_round_bits, void* dest, const ptrdiff_t dest_stride) {
-  auto* dst = static_cast<Pixel*>(dest);
-  const ptrdiff_t dst_stride = dest_stride / sizeof(Pixel);
-  const int step_y = subsampling_y ? 2 : 1;
-  const int mask_step_y =
-      (is_inter_intra && !is_wedge_inter_intra) ? 1 : step_y;
-  const uint8_t* mask_next_row = mask + mask_stride;
-  // An offset to cancel offsets used in single predictor generation that
-  // make intermediate computations non negative.
-  const int single_round_offset = (1 << bitdepth) + (1 << (bitdepth - 1));
-  // An offset to cancel offsets used in compound predictor generation that
-  // make intermediate computations non negative.
-  const int compound_round_offset =
-      (1 << (bitdepth + 4)) + (1 << (bitdepth + 3));
-  for (int y = 0; y < height; ++y) {
-    for (int x = 0; x < width; ++x) {
-      uint8_t mask_value;
-      if (((subsampling_x | subsampling_y) == 0) ||
-          (is_inter_intra && !is_wedge_inter_intra)) {
-        mask_value = mask[x];
-      } else if (subsampling_x == 1 && subsampling_y == 0) {
-        mask_value = static_cast<uint8_t>(RightShiftWithRounding(
-            mask[MultiplyBy2(x)] + mask[MultiplyBy2(x) + 1], 1));
-      } else if (subsampling_x == 0 && subsampling_y == 1) {
-        mask_value = static_cast<uint8_t>(
-            RightShiftWithRounding(mask[x] + mask_next_row[x], 1));
-      } else {
-        mask_value = static_cast<uint8_t>(RightShiftWithRounding(
-            mask[MultiplyBy2(x)] + mask[MultiplyBy2(x) + 1] +
-                mask_next_row[MultiplyBy2(x)] +
-                mask_next_row[MultiplyBy2(x) + 1],
-            2));
-      }
-
-      if (is_inter_intra) {
-        // In inter intra prediction mode, the intra prediction (prediction_1)
-        // values are valid pixel values: [0, (1 << bitdepth) - 1].
-        // While the inter prediction values come from subpixel prediction
-        // from another frame, which involves interpolation and rounding.
-        // Therefore prediction_0 has to be clipped.
-        dst[x] = static_cast<Pixel>(RightShiftWithRounding(
-            mask_value * prediction_1[x] +
-                (64 - mask_value) * Clip3(prediction_0[x] - single_round_offset,
-                                          0, (1 << bitdepth) - 1),
-            6));
-      } else {
-        int res = (mask_value * prediction_0[x] +
-                   (64 - mask_value) * prediction_1[x]) >>
-                  6;
-        res -= compound_round_offset;
-        dst[x] = static_cast<Pixel>(
-            Clip3(RightShiftWithRounding(res, inter_post_round_bits), 0,
-                  (1 << bitdepth) - 1));
-      }
-    }
-    dst += dst_stride;
-    mask += mask_stride * mask_step_y;
-    mask_next_row += mask_stride * step_y;
-    prediction_0 += prediction_stride_0;
-    prediction_1 += prediction_stride_1;
-  }
-}
-
-void Init8bpp() {
-  Dsp* const dsp = dsp_internal::GetWritableDspTable(8);
-  assert(dsp != nullptr);
-  dsp->mask_blend = MaskBlending_C<8, uint8_t>;
-}
-
-#if LIBGAV1_MAX_BITDEPTH >= 10
-void Init10bpp() {
-  Dsp* const dsp = dsp_internal::GetWritableDspTable(10);
-  assert(dsp != nullptr);
-  dsp->mask_blend = MaskBlending_C<10, uint16_t>;
-}
-#endif
-
-}  // namespace
-
-void MaskBlendingInit_C() {
-  Init8bpp();
-#if LIBGAV1_MAX_BITDEPTH >= 10
-  Init10bpp();
-#endif
-}
-
-}  // namespace dsp
-}  // namespace libgav1
diff --git a/libgav1/src/dsp/mask_blending.h b/libgav1/src/dsp/mask_blending.h
deleted file mode 100644
index 2d339db..0000000
--- a/libgav1/src/dsp/mask_blending.h
+++ /dev/null
@@ -1,12 +0,0 @@
-#ifndef LIBGAV1_SRC_DSP_MASK_BLENDING_H_
-#define LIBGAV1_SRC_DSP_MASK_BLENDING_H_
-
-namespace libgav1 {
-namespace dsp {
-
-void MaskBlendingInit_C();
-
-}  // namespace dsp
-}  // namespace libgav1
-
-#endif  // LIBGAV1_SRC_DSP_MASK_BLENDING_H_
diff --git a/libgav1/src/dsp/obmc.cc b/libgav1/src/dsp/obmc.cc
index ed0ca42..6c81852 100644
--- a/libgav1/src/dsp/obmc.cc
+++ b/libgav1/src/dsp/obmc.cc
@@ -6,29 +6,52 @@
 
 #include "src/dsp/dsp.h"
 #include "src/utils/common.h"
+#include "src/utils/constants.h"
 
 namespace libgav1 {
 namespace dsp {
 namespace {
 
-// 7.11.3.10.
+#include "src/dsp/obmc.inc"
+
+// 7.11.3.10 (from top samples).
 template <typename Pixel>
-void OverlapBlending_C(void* const prediction,
-                       const ptrdiff_t prediction_stride, const int width,
-                       const int height, const int blending_direction,
-                       const uint8_t* const mask,
-                       const void* const obmc_prediction,
-                       const ptrdiff_t obmc_prediction_stride) {
-  // 0 == kBlendFromAbove, 1 == kBlendFromLeft.
-  assert(blending_direction == 0 || blending_direction == 1);
+void OverlapBlendVertical_C(void* const prediction,
+                            const ptrdiff_t prediction_stride, const int width,
+                            const int height, const void* const obmc_prediction,
+                            const ptrdiff_t obmc_prediction_stride) {
   auto* pred = static_cast<Pixel*>(prediction);
   const ptrdiff_t pred_stride = prediction_stride / sizeof(Pixel);
   const auto* obmc_pred = static_cast<const Pixel*>(obmc_prediction);
   const ptrdiff_t obmc_pred_stride = obmc_prediction_stride / sizeof(Pixel);
+  const uint8_t* const mask = kObmcMask + height - 2;
 
   for (int y = 0; y < height; ++y) {
+    const uint8_t mask_value = mask[y];
     for (int x = 0; x < width; ++x) {
-      const uint8_t mask_value = (blending_direction == 0) ? mask[y] : mask[x];
+      pred[x] = static_cast<Pixel>(RightShiftWithRounding(
+          mask_value * pred[x] + (64 - mask_value) * obmc_pred[x], 6));
+    }
+    pred += pred_stride;
+    obmc_pred += obmc_pred_stride;
+  }
+}
+
+// 7.11.3.10 (from left samples).
+template <typename Pixel>
+void OverlapBlendHorizontal_C(void* const prediction,
+                              const ptrdiff_t prediction_stride,
+                              const int width, const int height,
+                              const void* const obmc_prediction,
+                              const ptrdiff_t obmc_prediction_stride) {
+  auto* pred = static_cast<Pixel*>(prediction);
+  const ptrdiff_t pred_stride = prediction_stride / sizeof(Pixel);
+  const auto* obmc_pred = static_cast<const Pixel*>(obmc_prediction);
+  const ptrdiff_t obmc_pred_stride = obmc_prediction_stride / sizeof(Pixel);
+  const uint8_t* const mask = kObmcMask + width - 2;
+  for (int y = 0; y < height; ++y) {
+    for (int x = 0; x < width; ++x) {
+      const uint8_t mask_value = mask[x];
       pred[x] = static_cast<Pixel>(RightShiftWithRounding(
           mask_value * pred[x] + (64 - mask_value) * obmc_pred[x], 6));
     }
@@ -40,14 +63,38 @@
 void Init8bpp() {
   Dsp* const dsp = dsp_internal::GetWritableDspTable(8);
   assert(dsp != nullptr);
-  dsp->obmc_blend = OverlapBlending_C<uint8_t>;
+#if LIBGAV1_ENABLE_ALL_DSP_FUNCTIONS
+  dsp->obmc_blend[kObmcDirectionVertical] = OverlapBlendVertical_C<uint8_t>;
+  dsp->obmc_blend[kObmcDirectionHorizontal] = OverlapBlendHorizontal_C<uint8_t>;
+#else  // !LIBGAV1_ENABLE_ALL_DSP_FUNCTIONS
+  static_cast<void>(dsp);
+#ifndef LIBGAV1_Dsp8bpp_ObmcVertical
+  dsp->obmc_blend[kObmcDirectionVertical] = OverlapBlendVertical_C<uint8_t>;
+#endif
+#ifndef LIBGAV1_Dsp8bpp_ObmcHorizontal
+  dsp->obmc_blend[kObmcDirectionHorizontal] = OverlapBlendHorizontal_C<uint8_t>;
+#endif
+#endif  // LIBGAV1_ENABLE_ALL_DSP_FUNCTIONS
 }
 
 #if LIBGAV1_MAX_BITDEPTH >= 10
 void Init10bpp() {
   Dsp* const dsp = dsp_internal::GetWritableDspTable(10);
   assert(dsp != nullptr);
-  dsp->obmc_blend = OverlapBlending_C<uint16_t>;
+#if LIBGAV1_ENABLE_ALL_DSP_FUNCTIONS
+  dsp->obmc_blend[kObmcDirectionVertical] = OverlapBlendVertical_C<uint16_t>;
+  dsp->obmc_blend[kObmcDirectionHorizontal] =
+      OverlapBlendHorizontal_C<uint16_t>;
+#else  // !LIBGAV1_ENABLE_ALL_DSP_FUNCTIONS
+  static_cast<void>(dsp);
+#ifndef LIBGAV1_Dsp10bpp_ObmcVertical
+  dsp->obmc_blend[kObmcDirectionVertical] = OverlapBlendVertical_C<uint16_t>;
+#endif
+#ifndef LIBGAV1_Dsp10bpp_ObmcHorizontal
+  dsp->obmc_blend[kObmcDirectionHorizontal] =
+      OverlapBlendHorizontal_C<uint16_t>;
+#endif
+#endif  // LIBGAV1_ENABLE_ALL_DSP_FUNCTIONS
 }
 #endif
 
diff --git a/libgav1/src/dsp/obmc.h b/libgav1/src/dsp/obmc.h
index 97e4b69..4a6d9cb 100644
--- a/libgav1/src/dsp/obmc.h
+++ b/libgav1/src/dsp/obmc.h
@@ -1,9 +1,28 @@
 #ifndef LIBGAV1_SRC_DSP_OBMC_H_
 #define LIBGAV1_SRC_DSP_OBMC_H_
 
+// Pull in LIBGAV1_DspXXX defines representing the implementation status
+// of each function. The resulting value of each can be used by each module to
+// determine whether an implementation is needed at compile time.
+// IWYU pragma: begin_exports
+
+// ARM:
+#include "src/dsp/arm/obmc_neon.h"
+
+// x86:
+// Note includes should be sorted in logical order avx2/avx/sse4, etc.
+// The order of includes is important as each tests for a superior version
+// before setting the base.
+// clang-format off
+#include "src/dsp/x86/obmc_sse4.h"
+// clang-format on
+
+// IWYU pragma: end_exports
+
 namespace libgav1 {
 namespace dsp {
 
+// Initializes Dsp::obmc_blend. This function is not thread-safe.
 void ObmcInit_C();
 
 }  // namespace dsp
diff --git a/libgav1/src/dsp/obmc.inc b/libgav1/src/dsp/obmc.inc
new file mode 100644
index 0000000..6da0001
--- /dev/null
+++ b/libgav1/src/dsp/obmc.inc
@@ -0,0 +1,18 @@
+// Constants and utility functions used for overlap blend implementations.
+// This will be included inside an anonymous namespace on files where these are
+// necessary.
+
+// This is a flat array of masks for each block dimension from 2 to 32. The
+// starting index for each length is length-2.
+constexpr uint8_t kObmcMask[62] = {
+    // Obmc Mask 2
+    45, 64,
+    // Obmc Mask 4
+    39, 50, 59, 64,
+    // Obmc Mask 8
+    36, 42, 48, 53, 57, 61, 64, 64,
+    // Obmc Mask 16
+    34, 37, 40, 43, 46, 49, 52, 54, 56, 58, 60, 61, 64, 64, 64, 64,
+    // Obmc Mask 32
+    33, 35, 36, 38, 40, 41, 43, 44, 45, 47, 48, 50, 51, 52, 53, 55, 56, 57,
+    58, 59, 60, 60, 61, 62, 64, 64, 64, 64, 64, 64, 64, 64};
diff --git a/libgav1/src/dsp/warp.cc b/libgav1/src/dsp/warp.cc
index 12eaa71..60aab23 100644
--- a/libgav1/src/dsp/warp.cc
+++ b/libgav1/src/dsp/warp.cc
@@ -22,20 +22,26 @@
 void Warp_C(const void* const source, ptrdiff_t source_stride,
             const int source_width, const int source_height,
             const int* const warp_params, const int subsampling_x,
-            const int subsampling_y, const uint8_t inter_round_bits[2],
+            const int subsampling_y, const uint8_t inter_round_bits_vertical,
             const int block_start_x, const int block_start_y,
             const int block_width, const int block_height, const int16_t alpha,
             const int16_t beta, const int16_t gamma, const int16_t delta,
             uint16_t* dest, const ptrdiff_t dest_stride) {
-  // Intermediate_result is the output of the horizontal filtering.
-  // The range is within 16 bits.
+  constexpr int kRoundBitsHorizontal = (bitdepth == 12)
+                                           ? kInterRoundBitsHorizontal12bpp
+                                           : kInterRoundBitsHorizontal;
+  // Intermediate_result is the output of the horizontal filtering and rounding.
+  // The range is within 16 bits (unsigned).
   uint16_t intermediate_result[15][8];  // 15 rows, 8 columns.
-  const int horizontal_offset_bits = bitdepth + kFilterBits - 1;
-  const int vertical_offset_bits =
-      bitdepth + 2 * kFilterBits - inter_round_bits[0];
+  const int horizontal_offset = 1 << (bitdepth + kFilterBits - 1);
+  const int vertical_offset =
+      1 << (bitdepth + 2 * kFilterBits - kRoundBitsHorizontal);
   const auto* const src = static_cast<const Pixel*>(source);
   source_stride /= sizeof(Pixel);
 
+  assert(block_width >= 8);
+  assert(block_height >= 8);
+
   // Warp process applies for each 8x8 block (or smaller).
   for (int start_y = block_start_y; start_y < block_start_y + block_height;
        start_y += 8) {
@@ -50,28 +56,32 @@
       const int x4 = dst_x >> subsampling_x;
       const int y4 = dst_y >> subsampling_y;
       const int ix4 = x4 >> kWarpedModelPrecisionBits;
-      const int sx4 = x4 & ((1 << kWarpedModelPrecisionBits) - 1);
       const int iy4 = y4 >> kWarpedModelPrecisionBits;
-      const int sy4 = y4 & ((1 << kWarpedModelPrecisionBits) - 1);
 
       // Horizontal filter.
+      int sx4 = (x4 & ((1 << kWarpedModelPrecisionBits) - 1)) - beta * 7;
       for (int y = -7; y < 8; ++y) {
-        // TODO(chenghchen):
+        // TODO(chengchen):
         // Because of warping, the index could be out of frame boundary. Thus
         // clip is needed. However, can we remove or reduce usage of clip?
         // Besides, special cases exist, for example,
-        // if iy4 - 7 >= source_height, there's no need to do the filtering.
+        // if iy4 - 7 >= source_height or iy4 + 7 < 0, there's no need to do the
+        // filtering.
         const int row = Clip3(iy4 + y, 0, source_height - 1);
         const Pixel* const src_row = src + row * source_stride;
+        int sx = sx4 - MultiplyBy4(alpha);
         for (int x = -4; x < 4; ++x) {
-          const int sx = sx4 + alpha * x + beta * y;
           const int offset =
               RightShiftWithRounding(sx, kWarpedDiffPrecisionBits) +
               kWarpedPixelPrecisionShifts;
+          // Since alpha and beta have been validated by SetupShear(), one can
+          // prove that 0 <= offset <= 3 * 2^6.
+          assert(offset >= 0);
+          assert(offset < 3 * kWarpedPixelPrecisionShifts + 1);
           // For SIMD optimization:
           // For 8 bit, the range of sum is within uint16_t, if we add an
           // horizontal offset:
-          int sum = 1 << horizontal_offset_bits;
+          int sum = horizontal_offset;
           // Horizontal_offset guarantees sum is non negative.
           // If horizontal_offset is used, intermediate_result needs to be
           // uint16_t.
@@ -80,37 +90,64 @@
             const int column = Clip3(ix4 + x + k - 3, 0, source_width - 1);
             sum += kWarpedFilters[offset][k] * src_row[column];
           }
-          assert(sum >= 0 && sum < (1 << (horizontal_offset_bits + 2)));
+          assert(sum >= 0 && sum < (horizontal_offset << 2));
           intermediate_result[y + 7][x + 4] = static_cast<uint16_t>(
-              RightShiftWithRounding(sum, inter_round_bits[0]));
+              RightShiftWithRounding(sum, kRoundBitsHorizontal));
+          sx += alpha;
         }
+        sx4 += beta;
       }
 
       // Vertical filter.
       uint16_t* dst_row = dest + start_x - block_start_x;
-      for (int y = -4;
-           y < std::min(4, block_start_y + block_height - start_y - 4); ++y) {
-        for (int x = -4;
-             x < std::min(4, block_start_x + block_width - start_x - 4); ++x) {
-          const int sy = sy4 + gamma * x + delta * y;
+      int sy4 =
+          (y4 & ((1 << kWarpedModelPrecisionBits) - 1)) - MultiplyBy4(delta);
+      // The spec says we should use the following loop condition:
+      //   y < std::min(4, block_start_y + block_height - start_y - 4);
+      // We can prove that block_start_y + block_height - start_y >= 8, which
+      // implies std::min(4, block_start_y + block_height - start_y - 4) = 4.
+      // So the loop condition is simply y < 4.
+      //
+      // Proof:
+      //    start_y < block_start_y + block_height
+      // => block_start_y + block_height - start_y > 0
+      // => block_height - (start_y - block_start_y) > 0
+      //
+      // Since block_height >= 8 and is a power of 2, it follows that
+      // block_height is a multiple of 8. start_y - block_start_y is also a
+      // multiple of 8. Therefore their difference is a multiple of 8. Since
+      // their difference is > 0, their difference must be >= 8.
+      for (int y = -4; y < 4; ++y) {
+        int sy = sy4 - MultiplyBy4(gamma);
+        // The spec says we should use the following loop condition:
+        //   x < std::min(4, block_start_x + block_width - start_x - 4);
+        // Similar to the above, we can prove that the loop condition can be
+        // simplified to x < 4.
+        for (int x = -4; x < 4; ++x) {
           const int offset =
               RightShiftWithRounding(sy, kWarpedDiffPrecisionBits) +
               kWarpedPixelPrecisionShifts;
+          // Since gamma and delta have been validated by SetupShear(), one can
+          // prove that 0 <= offset <= 3 * 2^6.
+          assert(offset >= 0);
+          assert(offset < 3 * kWarpedPixelPrecisionShifts + 1);
           // Similar to horizontal_offset, vertical_offset guarantees sum
           // before shifting is non negative:
-          int sum = 1 << vertical_offset_bits;
+          int sum = vertical_offset;
           for (int k = 0; k < 8; ++k) {
             sum += kWarpedFilters[offset][k] *
-                   intermediate_result[y + k + 4][x + 4];
+                   intermediate_result[y + 4 + k][x + 4];
           }
-          assert(sum >= 0 && sum < (1 << (vertical_offset_bits + 2)));
-          sum = RightShiftWithRounding(sum, inter_round_bits[1]);
+          assert(sum >= 0 && sum < (vertical_offset << 2));
+          sum = RightShiftWithRounding(sum, inter_round_bits_vertical);
           // Warp output is a predictor, whose type is uint16_t.
           // Do not clip it here. The clipping is applied at the stage of
           // final pixel value output.
           dst_row[x + 4] = static_cast<uint16_t>(sum);
+          sy += gamma;
         }
         dst_row += dest_stride;
+        sy4 += delta;
       }
     }
     dest += 8 * dest_stride;
@@ -120,14 +157,28 @@
 void Init8bpp() {
   Dsp* const dsp = dsp_internal::GetWritableDspTable(8);
   assert(dsp != nullptr);
+#if LIBGAV1_ENABLE_ALL_DSP_FUNCTIONS
   dsp->warp = Warp_C<8, uint8_t>;
+#else  // !LIBGAV1_ENABLE_ALL_DSP_FUNCTIONS
+  static_cast<void>(dsp);
+#ifndef LIBGAV1_Dsp8bpp_Warp
+  dsp->warp = Warp_C<8, uint8_t>;
+#endif
+#endif  // LIBGAV1_ENABLE_ALL_DSP_FUNCTIONS
 }
 
 #if LIBGAV1_MAX_BITDEPTH >= 10
 void Init10bpp() {
   Dsp* const dsp = dsp_internal::GetWritableDspTable(10);
   assert(dsp != nullptr);
+#if LIBGAV1_ENABLE_ALL_DSP_FUNCTIONS
   dsp->warp = Warp_C<10, uint16_t>;
+#else  // !LIBGAV1_ENABLE_ALL_DSP_FUNCTIONS
+  static_cast<void>(dsp);
+#ifndef LIBGAV1_Dsp10bpp_Warp
+  dsp->warp = Warp_C<10, uint16_t>;
+#endif
+#endif  // LIBGAV1_ENABLE_ALL_DSP_FUNCTIONS
 }
 #endif
 
diff --git a/libgav1/src/dsp/warp.h b/libgav1/src/dsp/warp.h
index e2cbf60..cd5f177 100644
--- a/libgav1/src/dsp/warp.h
+++ b/libgav1/src/dsp/warp.h
@@ -1,9 +1,20 @@
 #ifndef LIBGAV1_SRC_DSP_WARP_H_
 #define LIBGAV1_SRC_DSP_WARP_H_
 
+// Pull in LIBGAV1_DspXXX defines representing the implementation status
+// of each function. The resulting value of each can be used by each module to
+// determine whether an implementation is needed at compile time.
+// IWYU pragma: begin_exports
+
+// ARM:
+#include "src/dsp/arm/warp_neon.h"
+
+// IWYU pragma: end_exports
+
 namespace libgav1 {
 namespace dsp {
 
+// Initializes Dsp::warp. This function is not thread-safe.
 void WarpInit_C();
 
 }  // namespace dsp
diff --git a/libgav1/src/dsp/x86/average_blend_sse4.cc b/libgav1/src/dsp/x86/average_blend_sse4.cc
new file mode 100644
index 0000000..0ad0954
--- /dev/null
+++ b/libgav1/src/dsp/x86/average_blend_sse4.cc
@@ -0,0 +1,148 @@
+#include "src/dsp/average_blend.h"
+#include "src/dsp/dsp.h"
+
+#if LIBGAV1_ENABLE_SSE4_1
+
+#include <xmmintrin.h>
+
+#include <cassert>
+#include <cstddef>
+#include <cstdint>
+
+#include "src/dsp/x86/common_sse4.h"
+#include "src/utils/common.h"
+
+namespace libgav1 {
+namespace dsp {
+namespace {
+
+constexpr int kBitdepth8 = 8;
+constexpr int kInterPostRoundBit = 4;
+// An offset to cancel offsets used in compound predictor generation that
+// make intermediate computations non negative.
+const __m128i kCompoundRoundOffset =
+    _mm_set1_epi16((2 << (kBitdepth8 + 4)) + (2 << (kBitdepth8 + 3)));
+
+inline void AverageBlend4Row(const uint16_t* prediction_0,
+                             const uint16_t* prediction_1, uint8_t* dest) {
+  const __m128i pred_0 = LoadLo8(prediction_0);
+  const __m128i pred_1 = LoadLo8(prediction_1);
+  __m128i res = _mm_add_epi16(pred_0, pred_1);
+  res = _mm_sub_epi16(res, kCompoundRoundOffset);
+  res = RightShiftWithRounding_S16(res, kInterPostRoundBit + 1);
+  Store4(dest, _mm_packus_epi16(res, res));
+}
+
+inline void AverageBlend8Row(const uint16_t* prediction_0,
+                             const uint16_t* prediction_1, uint8_t* dest) {
+  const __m128i pred_0 = LoadUnaligned16(prediction_0);
+  const __m128i pred_1 = LoadUnaligned16(prediction_1);
+  __m128i res = _mm_add_epi16(pred_0, pred_1);
+  res = _mm_sub_epi16(res, kCompoundRoundOffset);
+  res = RightShiftWithRounding_S16(res, kInterPostRoundBit + 1);
+  StoreLo8(dest, _mm_packus_epi16(res, res));
+}
+
+inline void AverageBlendLargeRow(const uint16_t* prediction_0,
+                                 const uint16_t* prediction_1, const int width,
+                                 uint8_t* dest) {
+  int x = 0;
+  do {
+    const __m128i pred_00 = LoadUnaligned16(&prediction_0[x]);
+    const __m128i pred_01 = LoadUnaligned16(&prediction_1[x]);
+    __m128i res0 = _mm_add_epi16(pred_00, pred_01);
+    res0 = _mm_sub_epi16(res0, kCompoundRoundOffset);
+    res0 = RightShiftWithRounding_S16(res0, kInterPostRoundBit + 1);
+    const __m128i pred_10 = LoadUnaligned16(&prediction_0[x + 8]);
+    const __m128i pred_11 = LoadUnaligned16(&prediction_1[x + 8]);
+    __m128i res1 = _mm_add_epi16(pred_10, pred_11);
+    res1 = _mm_sub_epi16(res1, kCompoundRoundOffset);
+    res1 = RightShiftWithRounding_S16(res1, kInterPostRoundBit + 1);
+    StoreUnaligned16(dest + x, _mm_packus_epi16(res0, res1));
+    x += 16;
+  } while (x < width);
+}
+
+void AverageBlend_SSE4_1(const uint16_t* prediction_0,
+                         const ptrdiff_t prediction_stride_0,
+                         const uint16_t* prediction_1,
+                         const ptrdiff_t prediction_stride_1, const int width,
+                         const int height, void* const dest,
+                         const ptrdiff_t dest_stride) {
+  auto* dst = static_cast<uint8_t*>(dest);
+  int y = height;
+
+  if (width == 4) {
+    do {
+      AverageBlend4Row(prediction_0, prediction_1, dst);
+      dst += dest_stride;
+      prediction_0 += prediction_stride_0;
+      prediction_1 += prediction_stride_1;
+
+      AverageBlend4Row(prediction_0, prediction_1, dst);
+      dst += dest_stride;
+      prediction_0 += prediction_stride_0;
+      prediction_1 += prediction_stride_1;
+
+      y -= 2;
+    } while (y != 0);
+    return;
+  }
+
+  if (width == 8) {
+    do {
+      AverageBlend8Row(prediction_0, prediction_1, dst);
+      dst += dest_stride;
+      prediction_0 += prediction_stride_0;
+      prediction_1 += prediction_stride_1;
+
+      AverageBlend8Row(prediction_0, prediction_1, dst);
+      dst += dest_stride;
+      prediction_0 += prediction_stride_0;
+      prediction_1 += prediction_stride_1;
+
+      y -= 2;
+    } while (y != 0);
+    return;
+  }
+
+  do {
+    AverageBlendLargeRow(prediction_0, prediction_1, width, dst);
+    dst += dest_stride;
+    prediction_0 += prediction_stride_0;
+    prediction_1 += prediction_stride_1;
+
+    AverageBlendLargeRow(prediction_0, prediction_1, width, dst);
+    dst += dest_stride;
+    prediction_0 += prediction_stride_0;
+    prediction_1 += prediction_stride_1;
+
+    y -= 2;
+  } while (y != 0);
+}
+
+void Init8bpp() {
+  Dsp* const dsp = dsp_internal::GetWritableDspTable(8);
+  assert(dsp != nullptr);
+#if DSP_ENABLED_8BPP_SSE4_1(AverageBlend)
+  dsp->average_blend = AverageBlend_SSE4_1;
+#endif
+}
+
+}  // namespace
+
+void AverageBlendInit_SSE4_1() { Init8bpp(); }
+
+}  // namespace dsp
+}  // namespace libgav1
+
+#else   // !LIBGAV1_ENABLE_SSE4_1
+
+namespace libgav1 {
+namespace dsp {
+
+void AverageBlendInit_SSE4_1() {}
+
+}  // namespace dsp
+}  // namespace libgav1
+#endif  // LIBGAV1_ENABLE_SSE4_1
diff --git a/libgav1/src/dsp/x86/average_blend_sse4.h b/libgav1/src/dsp/x86/average_blend_sse4.h
new file mode 100644
index 0000000..26449d1
--- /dev/null
+++ b/libgav1/src/dsp/x86/average_blend_sse4.h
@@ -0,0 +1,25 @@
+#ifndef LIBGAV1_SRC_DSP_X86_AVERAGE_BLEND_SSE4_H_
+#define LIBGAV1_SRC_DSP_X86_AVERAGE_BLEND_SSE4_H_
+
+#include "src/dsp/cpu.h"
+#include "src/dsp/dsp.h"
+
+namespace libgav1 {
+namespace dsp {
+
+// Initializes Dsp::average_blend. This function is not thread-safe.
+void AverageBlendInit_SSE4_1();
+
+}  // namespace dsp
+}  // namespace libgav1
+
+// If sse4 is enabled and the baseline isn't set due to a higher level of
+// optimization being enabled, signal the sse4 implementation should be used.
+#if LIBGAV1_ENABLE_SSE4_1
+#ifndef LIBGAV1_Dsp8bpp_AverageBlend
+#define LIBGAV1_Dsp8bpp_AverageBlend LIBGAV1_DSP_SSE4_1
+#endif
+
+#endif  // LIBGAV1_ENABLE_SSE4_1
+
+#endif  // LIBGAV1_SRC_DSP_X86_AVERAGE_BLEND_SSE4_H_
diff --git a/libgav1/src/dsp/x86/common_sse4.h b/libgav1/src/dsp/x86/common_sse4.h
index e584bcc..305837a 100644
--- a/libgav1/src/dsp/x86/common_sse4.h
+++ b/libgav1/src/dsp/x86/common_sse4.h
@@ -12,10 +12,8 @@
 #include <cstdint>
 #include <cstring>
 
-namespace libgav1 {
-namespace dsp {
-
 #if 0
+#include <cinttypes>
 #include <cstdio>
 
 // Quite useful macro for debugging. Left here for convenience.
@@ -37,8 +35,8 @@
   } else if (size == 32) {
     for (n = 0; n < 4; ++n) fprintf(stderr, "%.8x ", tmp.i32[n]);
   } else {
-    for (n = 0; n < 2; ++n) fprintf(
-        stderr, "%.16llx ", static_cast<uint64_t>(tmp.i64[n]));
+    for (n = 0; n < 2; ++n)
+      fprintf(stderr, "%.16" PRIx64 " ", static_cast<uint64_t>(tmp.i64[n]));
   }
   fprintf(stderr, "\n");
 }
@@ -56,10 +54,33 @@
 #define PX(var) PrintRegX(var, #var);
 #endif  // 0
 
+namespace libgav1 {
+namespace dsp {
+
 //------------------------------------------------------------------------------
 // Load functions.
 
+inline __m128i Load2(const void* src) {
+  int16_t val;
+  memcpy(&val, src, sizeof(val));
+  return _mm_cvtsi32_si128(val);
+}
+
+inline __m128i Load2x2(const void* src1, const void* src2) {
+  uint16_t val1;
+  uint16_t val2;
+  memcpy(&val1, src1, sizeof(val1));
+  memcpy(&val2, src2, sizeof(val2));
+  return _mm_cvtsi32_si128(val1 | (val2 << 16));
+}
+
 inline __m128i Load4(const void* src) {
+  // With new compilers such as clang 8.0.0 we can use the new _mm_loadu_si32
+  // intrinsic. Both _mm_loadu_si32(src) and the code here are compiled into a
+  // movss instruction.
+  //
+  // Until compiler support of _mm_loadu_si32 is widespread, use of
+  // _mm_loadu_si32 is banned.
   int val;
   memcpy(&val, src, sizeof(val));
   return _mm_cvtsi32_si128(val);
@@ -82,6 +103,11 @@
 //------------------------------------------------------------------------------
 // Store functions.
 
+inline void Store2(void* dst, const __m128i x) {
+  const int val = _mm_cvtsi128_si32(x);
+  memcpy(dst, &val, 2);
+}
+
 inline void Store4(void* dst, const __m128i x) {
   const int val = _mm_cvtsi128_si32(x);
   memcpy(dst, &val, sizeof(val));
diff --git a/libgav1/src/dsp/x86/convolve_sse4.cc b/libgav1/src/dsp/x86/convolve_sse4.cc
new file mode 100644
index 0000000..b945b09
--- /dev/null
+++ b/libgav1/src/dsp/x86/convolve_sse4.cc
@@ -0,0 +1,21 @@
+#include "src/dsp/convolve.h"
+#include "src/dsp/dsp.h"
+
+#if LIBGAV1_ENABLE_SSE4_1
+namespace libgav1 {
+namespace dsp {
+
+void ConvolveInit_SSE4_1() {}
+
+}  // namespace dsp
+}  // namespace libgav1
+
+#else   // !LIBGAV1_ENABLE_SSE4_1
+namespace libgav1 {
+namespace dsp {
+
+void ConvolveInit_SSE4_1() {}
+
+}  // namespace dsp
+}  // namespace libgav1
+#endif  // LIBGAV1_ENABLE_SSE4_1
diff --git a/libgav1/src/dsp/x86/convolve_sse4.h b/libgav1/src/dsp/x86/convolve_sse4.h
new file mode 100644
index 0000000..1cc240d
--- /dev/null
+++ b/libgav1/src/dsp/x86/convolve_sse4.h
@@ -0,0 +1,27 @@
+#ifndef LIBGAV1_SRC_DSP_X86_CONVOLVE_SSE4_H_
+#define LIBGAV1_SRC_DSP_X86_CONVOLVE_SSE4_H_
+
+#include "src/dsp/cpu.h"
+#include "src/dsp/dsp.h"
+
+namespace libgav1 {
+namespace dsp {
+
+// Initializes Dsp::convolve, see the defines below for specifics. This
+// function is not thread-safe.
+void ConvolveInit_SSE4_1();
+
+}  // namespace dsp
+}  // namespace libgav1
+
+// If sse4 is enabled and the baseline isn't set due to a higher level of
+// optimization being enabled, signal the sse4 implementation should be used.
+#if LIBGAV1_ENABLE_SSE4_1
+
+#ifndef LIBGAV1_Dsp8bpp_Convolve2D
+// #define LIBGAV1_Dsp8bpp_Convolve2D LIBGAV1_DSP_SSE4_1
+#endif
+
+#endif  // LIBGAV1_ENABLE_SSE4_1
+
+#endif  // LIBGAV1_SRC_DSP_X86_CONVOLVE_SSE4_H_
diff --git a/libgav1/src/dsp/x86/distance_weighted_blend_sse4.cc b/libgav1/src/dsp/x86/distance_weighted_blend_sse4.cc
new file mode 100644
index 0000000..3a9673a
--- /dev/null
+++ b/libgav1/src/dsp/x86/distance_weighted_blend_sse4.cc
@@ -0,0 +1,233 @@
+#include "src/dsp/distance_weighted_blend.h"
+#include "src/dsp/dsp.h"
+
+#if LIBGAV1_ENABLE_SSE4_1
+
+#include <xmmintrin.h>
+
+#include <cassert>
+#include <cstddef>
+#include <cstdint>
+
+#include "src/dsp/x86/common_sse4.h"
+#include "src/utils/common.h"
+
+namespace libgav1 {
+namespace dsp {
+namespace {
+
+constexpr int kBitdepth8 = 8;
+constexpr int kInterPostRoundBit = 4;
+
+inline __m128i ComputeWeightedAverage8(const __m128i& pred0,
+                                       const __m128i& pred1,
+                                       const __m128i& weights) {
+  const __m128i compound_round_offset32 =
+      _mm_set1_epi32((16 << (kBitdepth8 + 4)) + (16 << (kBitdepth8 + 3)));
+  const __m128i preds_lo = _mm_unpacklo_epi16(pred0, pred1);
+  const __m128i mult_lo =
+      _mm_sub_epi32(_mm_madd_epi16(preds_lo, weights), compound_round_offset32);
+  const __m128i result_lo =
+      RightShiftWithRounding_S32(mult_lo, kInterPostRoundBit + 4);
+
+  const __m128i preds_hi = _mm_unpackhi_epi16(pred0, pred1);
+  const __m128i mult_hi =
+      _mm_sub_epi32(_mm_madd_epi16(preds_hi, weights), compound_round_offset32);
+  const __m128i result_hi =
+      RightShiftWithRounding_S32(mult_hi, kInterPostRoundBit + 4);
+
+  return _mm_packs_epi32(result_lo, result_hi);
+}
+
+template <int height>
+inline void DistanceWeightedBlend4xH_SSE4_1(
+    const uint16_t* prediction_0, const ptrdiff_t prediction_stride_0,
+    const uint16_t* prediction_1, const ptrdiff_t prediction_stride_1,
+    const uint8_t weight_0, const uint8_t weight_1, void* const dest,
+    const ptrdiff_t dest_stride) {
+  auto* dst = static_cast<uint8_t*>(dest);
+  const uint16_t* pred_0 = prediction_0;
+  const uint16_t* pred_1 = prediction_1;
+  const __m128i weights = _mm_set1_epi32(weight_0 | (weight_1 << 16));
+
+  for (int y = 0; y < height; y += 4) {
+    const __m128i src_00 = LoadLo8(pred_0);
+    const __m128i src_10 = LoadLo8(pred_1);
+    pred_0 += prediction_stride_0;
+    pred_1 += prediction_stride_1;
+    __m128i src_0 = LoadHi8(src_00, pred_0);
+    __m128i src_1 = LoadHi8(src_10, pred_1);
+    pred_0 += prediction_stride_0;
+    pred_1 += prediction_stride_1;
+    const __m128i res0 = ComputeWeightedAverage8(src_0, src_1, weights);
+
+    const __m128i src_01 = LoadLo8(pred_0);
+    const __m128i src_11 = LoadLo8(pred_1);
+    pred_0 += prediction_stride_0;
+    pred_1 += prediction_stride_1;
+    src_0 = LoadHi8(src_01, pred_0);
+    src_1 = LoadHi8(src_11, pred_1);
+    pred_0 += prediction_stride_0;
+    pred_1 += prediction_stride_1;
+    const __m128i res1 = ComputeWeightedAverage8(src_0, src_1, weights);
+
+    const __m128i result_pixels = _mm_packus_epi16(res0, res1);
+    Store4(dst, result_pixels);
+    dst += dest_stride;
+    const int result_1 = _mm_extract_epi32(result_pixels, 1);
+    memcpy(dst, &result_1, sizeof(result_1));
+    dst += dest_stride;
+    const int result_2 = _mm_extract_epi32(result_pixels, 2);
+    memcpy(dst, &result_2, sizeof(result_2));
+    dst += dest_stride;
+    const int result_3 = _mm_extract_epi32(result_pixels, 3);
+    memcpy(dst, &result_3, sizeof(result_3));
+    dst += dest_stride;
+  }
+}
+
+template <int height>
+inline void DistanceWeightedBlend8xH_SSE4_1(
+    const uint16_t* prediction_0, const ptrdiff_t prediction_stride_0,
+    const uint16_t* prediction_1, const ptrdiff_t prediction_stride_1,
+    const uint8_t weight_0, const uint8_t weight_1, void* const dest,
+    const ptrdiff_t dest_stride) {
+  auto* dst = static_cast<uint8_t*>(dest);
+  const uint16_t* pred_0 = prediction_0;
+  const uint16_t* pred_1 = prediction_1;
+  const __m128i weights = _mm_set1_epi32(weight_0 | (weight_1 << 16));
+
+  for (int y = 0; y < height; y += 2) {
+    const __m128i src_00 = LoadUnaligned16(pred_0);
+    const __m128i src_10 = LoadUnaligned16(pred_1);
+    pred_0 += prediction_stride_0;
+    pred_1 += prediction_stride_1;
+    const __m128i res0 = ComputeWeightedAverage8(src_00, src_10, weights);
+
+    const __m128i src_01 = LoadUnaligned16(pred_0);
+    const __m128i src_11 = LoadUnaligned16(pred_1);
+    pred_0 += prediction_stride_0;
+    pred_1 += prediction_stride_1;
+    const __m128i res1 = ComputeWeightedAverage8(src_01, src_11, weights);
+
+    const __m128i result_pixels = _mm_packus_epi16(res0, res1);
+    StoreLo8(dst, result_pixels);
+    dst += dest_stride;
+    StoreHi8(dst, result_pixels);
+    dst += dest_stride;
+  }
+}
+
+inline void DistanceWeightedBlendLarge_SSE4_1(
+    const uint16_t* prediction_0, const ptrdiff_t prediction_stride_0,
+    const uint16_t* prediction_1, const ptrdiff_t prediction_stride_1,
+    const uint8_t weight_0, const uint8_t weight_1, const int width,
+    const int height, void* const dest, const ptrdiff_t dest_stride) {
+  auto* dst = static_cast<uint8_t*>(dest);
+  const uint16_t* pred_0 = prediction_0;
+  const uint16_t* pred_1 = prediction_1;
+  const __m128i weights = _mm_set1_epi32(weight_0 | (weight_1 << 16));
+
+  int y = height;
+  do {
+    int x = 0;
+    do {
+      const __m128i src_0_lo = LoadUnaligned16(pred_0 + x);
+      const __m128i src_1_lo = LoadUnaligned16(pred_1 + x);
+      const __m128i res_lo =
+          ComputeWeightedAverage8(src_0_lo, src_1_lo, weights);
+
+      const __m128i src_0_hi = LoadUnaligned16(pred_0 + x + 8);
+      const __m128i src_1_hi = LoadUnaligned16(pred_1 + x + 8);
+      const __m128i res_hi =
+          ComputeWeightedAverage8(src_0_hi, src_1_hi, weights);
+
+      StoreUnaligned16(dst + x, _mm_packus_epi16(res_lo, res_hi));
+      x += 16;
+    } while (x < width);
+    dst += dest_stride;
+    pred_0 += prediction_stride_0;
+    pred_1 += prediction_stride_1;
+  } while (--y != 0);
+}
+
+void DistanceWeightedBlend_SSE4_1(
+    const uint16_t* prediction_0, const ptrdiff_t prediction_stride_0,
+    const uint16_t* prediction_1, const ptrdiff_t prediction_stride_1,
+    const uint8_t weight_0, const uint8_t weight_1, const int width,
+    const int height, void* const dest, const ptrdiff_t dest_stride) {
+  if (width == 4) {
+    if (height == 4) {
+      DistanceWeightedBlend4xH_SSE4_1<4>(prediction_0, prediction_stride_0,
+                                         prediction_1, prediction_stride_1,
+                                         weight_0, weight_1, dest, dest_stride);
+    } else if (height == 8) {
+      DistanceWeightedBlend4xH_SSE4_1<8>(prediction_0, prediction_stride_0,
+                                         prediction_1, prediction_stride_1,
+                                         weight_0, weight_1, dest, dest_stride);
+    } else {
+      assert(height == 16);
+      DistanceWeightedBlend4xH_SSE4_1<16>(
+          prediction_0, prediction_stride_0, prediction_1, prediction_stride_1,
+          weight_0, weight_1, dest, dest_stride);
+    }
+    return;
+  }
+
+  if (width == 8) {
+    switch (height) {
+      case 4:
+        DistanceWeightedBlend8xH_SSE4_1<4>(
+            prediction_0, prediction_stride_0, prediction_1,
+            prediction_stride_1, weight_0, weight_1, dest, dest_stride);
+        return;
+      case 8:
+        DistanceWeightedBlend8xH_SSE4_1<8>(
+            prediction_0, prediction_stride_0, prediction_1,
+            prediction_stride_1, weight_0, weight_1, dest, dest_stride);
+        return;
+      case 16:
+        DistanceWeightedBlend8xH_SSE4_1<16>(
+            prediction_0, prediction_stride_0, prediction_1,
+            prediction_stride_1, weight_0, weight_1, dest, dest_stride);
+        return;
+      default:
+        assert(height == 32);
+        DistanceWeightedBlend8xH_SSE4_1<32>(
+            prediction_0, prediction_stride_0, prediction_1,
+            prediction_stride_1, weight_0, weight_1, dest, dest_stride);
+
+        return;
+    }
+  }
+
+  DistanceWeightedBlendLarge_SSE4_1(prediction_0, prediction_stride_0,
+                                    prediction_1, prediction_stride_1, weight_0,
+                                    weight_1, width, height, dest, dest_stride);
+}
+
+void Init8bpp() {
+  Dsp* const dsp = dsp_internal::GetWritableDspTable(8);
+  assert(dsp != nullptr);
+#if DSP_ENABLED_8BPP_SSE4_1(DistanceWeightedBlend)
+  dsp->distance_weighted_blend = DistanceWeightedBlend_SSE4_1;
+#endif
+}
+
+}  // namespace
+
+void DistanceWeightedBlendInit_SSE4_1() { Init8bpp(); }
+
+}  // namespace dsp
+}  // namespace libgav1
+
+#else   // !LIBGAV1_ENABLE_SSE4_1
+
+namespace libgav1 {
+namespace dsp {
+
+void DistanceWeightedBlendInit_SSE4_1() {}
+
+}  // namespace dsp
+}  // namespace libgav1
+#endif  // LIBGAV1_ENABLE_SSE4_1
diff --git a/libgav1/src/dsp/x86/distance_weighted_blend_sse4.h b/libgav1/src/dsp/x86/distance_weighted_blend_sse4.h
new file mode 100644
index 0000000..831853d
--- /dev/null
+++ b/libgav1/src/dsp/x86/distance_weighted_blend_sse4.h
@@ -0,0 +1,25 @@
+#ifndef LIBGAV1_SRC_DSP_X86_DISTANCE_WEIGHTED_BLEND_SSE4_H_
+#define LIBGAV1_SRC_DSP_X86_DISTANCE_WEIGHTED_BLEND_SSE4_H_
+
+#include "src/dsp/cpu.h"
+#include "src/dsp/dsp.h"
+
+namespace libgav1 {
+namespace dsp {
+
+// Initializes Dsp::distance_weighted_blend. This function is not thread-safe.
+void DistanceWeightedBlendInit_SSE4_1();
+
+}  // namespace dsp
+}  // namespace libgav1
+
+// If sse4 is enabled and the baseline isn't set due to a higher level of
+// optimization being enabled, signal the sse4 implementation should be used.
+#if LIBGAV1_ENABLE_SSE4_1
+#ifndef LIBGAV1_Dsp8bpp_DistanceWeightedBlend
+#define LIBGAV1_Dsp8bpp_DistanceWeightedBlend LIBGAV1_DSP_SSE4_1
+#endif
+
+#endif  // LIBGAV1_ENABLE_SSE4_1
+
+#endif  // LIBGAV1_SRC_DSP_X86_DISTANCE_WEIGHTED_BLEND_SSE4_H_
diff --git a/libgav1/src/dsp/x86/intra_edge_sse4.cc b/libgav1/src/dsp/x86/intra_edge_sse4.cc
index 7772234..7a20927 100644
--- a/libgav1/src/dsp/x86/intra_edge_sse4.cc
+++ b/libgav1/src/dsp/x86/intra_edge_sse4.cc
@@ -1,6 +1,5 @@
-#include "src/dsp/x86/intra_edge_sse4.h"
-
 #include "src/dsp/dsp.h"
+#include "src/dsp/intra_edge.h"
 
 #if LIBGAV1_ENABLE_SSE4_1
 
@@ -143,6 +142,9 @@
   memcpy(edge, buffer, size);
   auto* dst_buffer = static_cast<uint8_t*>(buffer);
 
+  // Only process |size| - 1 elements. Nothing to do in this case.
+  if (size == 1) return;
+
   int i = 0;
   switch (strength) {
     case 1:
diff --git a/libgav1/src/dsp/x86/intra_edge_sse4.h b/libgav1/src/dsp/x86/intra_edge_sse4.h
index abaad40..7803ce9 100644
--- a/libgav1/src/dsp/x86/intra_edge_sse4.h
+++ b/libgav1/src/dsp/x86/intra_edge_sse4.h
@@ -3,13 +3,12 @@
 
 #include "src/dsp/cpu.h"
 #include "src/dsp/dsp.h"
-#include "src/dsp/intra_edge.h"
 
 namespace libgav1 {
 namespace dsp {
 
-// Initializes intra edge functions with sse4 implementations. This function
-// is not thread-safe.
+// Initializes Dsp::intra_edge_filter and Dsp::intra_edge_upsampler. This
+// function is not thread-safe.
 void IntraEdgeInit_SSE4_1();
 
 }  // namespace dsp
diff --git a/libgav1/src/dsp/x86/intrapred_cfl_sse4.cc b/libgav1/src/dsp/x86/intrapred_cfl_sse4.cc
index 3744fc6..4d97fb0 100644
--- a/libgav1/src/dsp/x86/intrapred_cfl_sse4.cc
+++ b/libgav1/src/dsp/x86/intrapred_cfl_sse4.cc
@@ -1,12 +1,11 @@
 #include "src/dsp/dsp.h"
-#include "src/dsp/x86/intrapred_sse4.h"
+#include "src/dsp/intrapred.h"
 
 #if LIBGAV1_ENABLE_SSE4_1
 
-#include <emmintrin.h>
 #include <smmintrin.h>
-#include <xmmintrin.h>
 
+#include <algorithm>
 #include <cassert>
 #include <cstddef>
 #include <cstdint>
@@ -168,6 +167,7 @@
     int16_t luma[kCflLumaBufferStride][kCflLumaBufferStride],
     const int /*max_luma_width*/, const int /*max_luma_height*/,
     const void* const source, ptrdiff_t stride) {
+  static_assert(block_width_log2 == 4 || block_width_log2 == 5, "");
   const int block_height = 1 << block_height_log2;
   const int block_width = 1 << block_width_log2;
   const __m128i dup16 = _mm_set1_epi32(0x01000100);
@@ -214,9 +214,434 @@
   }
 }
 
+// Takes in two sums of input row pairs, and completes the computation for two
+// output rows.
+inline __m128i StoreLumaResults4_420(const __m128i vertical_sum0,
+                                     const __m128i vertical_sum1,
+                                     int16_t* luma_ptr) {
+  __m128i result = _mm_hadd_epi16(vertical_sum0, vertical_sum1);
+  result = _mm_slli_epi16(result, 1);
+  StoreLo8(luma_ptr, result);
+  StoreHi8(luma_ptr + kCflLumaBufferStride, result);
+  return result;
+}
+
+// Takes two halves of a vertically added pair of rows and completes the
+// computation for one output row.
+inline __m128i StoreLumaResults8_420(const __m128i vertical_sum0,
+                                     const __m128i vertical_sum1,
+                                     int16_t* luma_ptr) {
+  __m128i result = _mm_hadd_epi16(vertical_sum0, vertical_sum1);
+  result = _mm_slli_epi16(result, 1);
+  StoreUnaligned16(luma_ptr, result);
+  return result;
+}
+
+template <int block_height_log2>
+void CflSubsampler420_4xH_SSE4_1(
+    int16_t luma[kCflLumaBufferStride][kCflLumaBufferStride],
+    const int /*max_luma_width*/, const int max_luma_height,
+    const void* const source, ptrdiff_t stride) {
+  const int block_height = 1 << block_height_log2;
+  const auto* src = static_cast<const uint8_t*>(source);
+  int16_t* luma_ptr = luma[0];
+  const __m128i zero = _mm_setzero_si128();
+  __m128i final_sum = zero;
+  const int luma_height = std::min(block_height, max_luma_height >> 1);
+  int y = 0;
+  do {
+    // Note that double sampling and converting to 16bit makes a row fill the
+    // vector.
+    const __m128i samples_row0 = _mm_cvtepu8_epi16(LoadLo8(src));
+    src += stride;
+    const __m128i samples_row1 = _mm_cvtepu8_epi16(LoadLo8(src));
+    src += stride;
+    const __m128i luma_sum01 = _mm_add_epi16(samples_row0, samples_row1);
+
+    const __m128i samples_row2 = _mm_cvtepu8_epi16(LoadLo8(src));
+    src += stride;
+    const __m128i samples_row3 = _mm_cvtepu8_epi16(LoadLo8(src));
+    src += stride;
+    const __m128i luma_sum23 = _mm_add_epi16(samples_row2, samples_row3);
+    __m128i sum = StoreLumaResults4_420(luma_sum01, luma_sum23, luma_ptr);
+    luma_ptr += kCflLumaBufferStride << 1;
+
+    const __m128i samples_row4 = _mm_cvtepu8_epi16(LoadLo8(src));
+    src += stride;
+    const __m128i samples_row5 = _mm_cvtepu8_epi16(LoadLo8(src));
+    src += stride;
+    const __m128i luma_sum45 = _mm_add_epi16(samples_row4, samples_row5);
+
+    const __m128i samples_row6 = _mm_cvtepu8_epi16(LoadLo8(src));
+    src += stride;
+    const __m128i samples_row7 = _mm_cvtepu8_epi16(LoadLo8(src));
+    src += stride;
+    const __m128i luma_sum67 = _mm_add_epi16(samples_row6, samples_row7);
+    sum = _mm_add_epi16(
+        sum, StoreLumaResults4_420(luma_sum45, luma_sum67, luma_ptr));
+    luma_ptr += kCflLumaBufferStride << 1;
+
+    final_sum = _mm_add_epi32(final_sum, _mm_cvtepu16_epi32(sum));
+    final_sum = _mm_add_epi32(final_sum, _mm_unpackhi_epi16(sum, zero));
+    y += 4;
+  } while (y < luma_height);
+  const __m128i final_fill = LoadLo8(luma_ptr - kCflLumaBufferStride);
+  const __m128i final_fill_to_sum = _mm_cvtepu16_epi32(final_fill);
+  for (; y < block_height; ++y) {
+    StoreLo8(luma_ptr, final_fill);
+    luma_ptr += kCflLumaBufferStride;
+
+    final_sum = _mm_add_epi32(final_sum, final_fill_to_sum);
+  }
+  final_sum = _mm_add_epi32(final_sum, _mm_srli_si128(final_sum, 8));
+  final_sum = _mm_add_epi32(final_sum, _mm_srli_si128(final_sum, 4));
+
+  __m128i averages = RightShiftWithRounding_U32(
+      final_sum, block_height_log2 + 2 /*log2 of width 4*/);
+
+  averages = _mm_shufflelo_epi16(averages, 0);
+  luma_ptr = luma[0];
+  for (int y = 0; y < block_height; ++y, luma_ptr += kCflLumaBufferStride) {
+    const __m128i samples = LoadLo8(luma_ptr);
+    StoreLo8(luma_ptr, _mm_sub_epi16(samples, averages));
+  }
+}
+
+// This duplicates the last two 16-bit values in |row|.
+inline __m128i LastRowSamples(const __m128i row) {
+  return _mm_shuffle_epi32(row, 0xFF);
+}
+
+// This duplicates the last 16-bit value in |row|.
+inline __m128i LastRowResult(const __m128i row) {
+  const __m128i dup_row = _mm_shufflehi_epi16(row, 0xFF);
+  return _mm_shuffle_epi32(dup_row, 0xFF);
+}
+
+template <int block_height_log2, int max_luma_width>
+inline void CflSubsampler420Impl_8xH_SSE4_1(
+    int16_t luma[kCflLumaBufferStride][kCflLumaBufferStride],
+    const int /*max_luma_width*/, const int max_luma_height,
+    const void* const source, ptrdiff_t stride) {
+  const int block_height = 1 << block_height_log2;
+  const auto* src = static_cast<const uint8_t*>(source);
+  const __m128i zero = _mm_setzero_si128();
+  __m128i final_sum = zero;
+  int16_t* luma_ptr = luma[0];
+  const int luma_height = std::min(block_height, max_luma_height >> 1);
+  int y = 0;
+
+  do {
+    const __m128i samples_row00 = _mm_cvtepu8_epi16(LoadLo8(src));
+    const __m128i samples_row01 = (max_luma_width == 16)
+                                      ? _mm_cvtepu8_epi16(LoadLo8(src + 8))
+                                      : LastRowSamples(samples_row00);
+    src += stride;
+    const __m128i samples_row10 = _mm_cvtepu8_epi16(LoadLo8(src));
+    const __m128i samples_row11 = (max_luma_width == 16)
+                                      ? _mm_cvtepu8_epi16(LoadLo8(src + 8))
+                                      : LastRowSamples(samples_row10);
+    src += stride;
+    const __m128i luma_sum00 = _mm_add_epi16(samples_row00, samples_row10);
+    const __m128i luma_sum01 = _mm_add_epi16(samples_row01, samples_row11);
+    __m128i sum = StoreLumaResults8_420(luma_sum00, luma_sum01, luma_ptr);
+    luma_ptr += kCflLumaBufferStride;
+
+    const __m128i samples_row20 = _mm_cvtepu8_epi16(LoadLo8(src));
+    const __m128i samples_row21 = (max_luma_width == 16)
+                                      ? _mm_cvtepu8_epi16(LoadLo8(src + 8))
+                                      : LastRowSamples(samples_row20);
+    src += stride;
+    const __m128i samples_row30 = _mm_cvtepu8_epi16(LoadLo8(src));
+    const __m128i samples_row31 = (max_luma_width == 16)
+                                      ? _mm_cvtepu8_epi16(LoadLo8(src + 8))
+                                      : LastRowSamples(samples_row30);
+    src += stride;
+    const __m128i luma_sum10 = _mm_add_epi16(samples_row20, samples_row30);
+    const __m128i luma_sum11 = _mm_add_epi16(samples_row21, samples_row31);
+    sum = _mm_add_epi16(
+        sum, StoreLumaResults8_420(luma_sum10, luma_sum11, luma_ptr));
+    luma_ptr += kCflLumaBufferStride;
+
+    const __m128i samples_row40 = _mm_cvtepu8_epi16(LoadLo8(src));
+    const __m128i samples_row41 = (max_luma_width == 16)
+                                      ? _mm_cvtepu8_epi16(LoadLo8(src + 8))
+                                      : LastRowSamples(samples_row40);
+    src += stride;
+    const __m128i samples_row50 = _mm_cvtepu8_epi16(LoadLo8(src));
+    const __m128i samples_row51 = (max_luma_width == 16)
+                                      ? _mm_cvtepu8_epi16(LoadLo8(src + 8))
+                                      : LastRowSamples(samples_row50);
+    src += stride;
+    const __m128i luma_sum20 = _mm_add_epi16(samples_row40, samples_row50);
+    const __m128i luma_sum21 = _mm_add_epi16(samples_row41, samples_row51);
+    sum = _mm_add_epi16(
+        sum, StoreLumaResults8_420(luma_sum20, luma_sum21, luma_ptr));
+    luma_ptr += kCflLumaBufferStride;
+
+    const __m128i samples_row60 = _mm_cvtepu8_epi16(LoadLo8(src));
+    const __m128i samples_row61 = (max_luma_width == 16)
+                                      ? _mm_cvtepu8_epi16(LoadLo8(src + 8))
+                                      : LastRowSamples(samples_row60);
+    src += stride;
+    const __m128i samples_row70 = _mm_cvtepu8_epi16(LoadLo8(src));
+    const __m128i samples_row71 = (max_luma_width == 16)
+                                      ? _mm_cvtepu8_epi16(LoadLo8(src + 8))
+                                      : LastRowSamples(samples_row70);
+    src += stride;
+    const __m128i luma_sum30 = _mm_add_epi16(samples_row60, samples_row70);
+    const __m128i luma_sum31 = _mm_add_epi16(samples_row61, samples_row71);
+    sum = _mm_add_epi16(
+        sum, StoreLumaResults8_420(luma_sum30, luma_sum31, luma_ptr));
+    luma_ptr += kCflLumaBufferStride;
+
+    final_sum = _mm_add_epi32(final_sum, _mm_cvtepu16_epi32(sum));
+    final_sum = _mm_add_epi32(final_sum, _mm_unpackhi_epi16(sum, zero));
+    y += 4;
+  } while (y < luma_height);
+  // Duplicate the final row downward to the end after max_luma_height.
+  const __m128i final_fill = LoadUnaligned16(luma_ptr - kCflLumaBufferStride);
+  const __m128i final_fill_to_sum0 = _mm_cvtepi16_epi32(final_fill);
+  const __m128i final_fill_to_sum1 =
+      _mm_cvtepi16_epi32(_mm_srli_si128(final_fill, 8));
+  const __m128i final_fill_to_sum =
+      _mm_add_epi32(final_fill_to_sum0, final_fill_to_sum1);
+  for (; y < block_height; ++y) {
+    StoreUnaligned16(luma_ptr, final_fill);
+    luma_ptr += kCflLumaBufferStride;
+
+    final_sum = _mm_add_epi32(final_sum, final_fill_to_sum);
+  }
+  final_sum = _mm_add_epi32(final_sum, _mm_srli_si128(final_sum, 8));
+  final_sum = _mm_add_epi32(final_sum, _mm_srli_si128(final_sum, 4));
+
+  __m128i averages = RightShiftWithRounding_S32(
+      final_sum, block_height_log2 + 3 /*log2 of width 8*/);
+
+  averages = _mm_shufflelo_epi16(averages, 0);
+  averages = _mm_shuffle_epi32(averages, 0);
+  luma_ptr = luma[0];
+  for (int y = 0; y < block_height; ++y, luma_ptr += kCflLumaBufferStride) {
+    const __m128i samples = LoadUnaligned16(luma_ptr);
+    StoreUnaligned16(luma_ptr, _mm_sub_epi16(samples, averages));
+  }
+}
+
+template <int block_height_log2>
+void CflSubsampler420_8xH_SSE4_1(
+    int16_t luma[kCflLumaBufferStride][kCflLumaBufferStride],
+    const int max_luma_width, const int max_luma_height,
+    const void* const source, ptrdiff_t stride) {
+  if (max_luma_width == 8) {
+    CflSubsampler420Impl_8xH_SSE4_1<block_height_log2, 8>(
+        luma, max_luma_width, max_luma_height, source, stride);
+  } else {
+    CflSubsampler420Impl_8xH_SSE4_1<block_height_log2, 16>(
+        luma, max_luma_width, max_luma_height, source, stride);
+  }
+}
+
+template <int block_width_log2, int block_height_log2, int max_luma_width>
+inline void CflSubsampler420Impl_WxH_SSE4_1(
+    int16_t luma[kCflLumaBufferStride][kCflLumaBufferStride],
+    const int /*max_luma_width*/, const int max_luma_height,
+    const void* const source, ptrdiff_t stride) {
+  const auto* src = static_cast<const uint8_t*>(source);
+  const __m128i zero = _mm_setzero_si128();
+  __m128i final_sum = zero;
+  const int block_height = 1 << block_height_log2;
+  const int luma_height = std::min(block_height, max_luma_height >> 1);
+
+  int16_t* luma_ptr = luma[0];
+  __m128i final_row_result;
+  // Begin first y section, covering width up to 16.
+  int y = 0;
+  do {
+    const uint8_t* src_next = src + stride;
+    const __m128i samples_row0_lo = LoadUnaligned16(src);
+    const __m128i samples_row00 = _mm_cvtepu8_epi16(samples_row0_lo);
+    const __m128i samples_row01 = (max_luma_width >= 16)
+                                      ? _mm_unpackhi_epi8(samples_row0_lo, zero)
+                                      : LastRowSamples(samples_row00);
+    const __m128i samples_row0_hi = LoadUnaligned16(src + 16);
+    const __m128i samples_row02 = (max_luma_width >= 24)
+                                      ? _mm_cvtepu8_epi16(samples_row0_hi)
+                                      : LastRowSamples(samples_row01);
+    const __m128i samples_row03 = (max_luma_width == 32)
+                                      ? _mm_unpackhi_epi8(samples_row0_hi, zero)
+                                      : LastRowSamples(samples_row02);
+    const __m128i samples_row1_lo = LoadUnaligned16(src_next);
+    const __m128i samples_row10 = _mm_cvtepu8_epi16(samples_row1_lo);
+    const __m128i samples_row11 = (max_luma_width >= 16)
+                                      ? _mm_unpackhi_epi8(samples_row1_lo, zero)
+                                      : LastRowSamples(samples_row10);
+    const __m128i samples_row1_hi = LoadUnaligned16(src_next + 16);
+    const __m128i samples_row12 = (max_luma_width >= 24)
+                                      ? _mm_cvtepu8_epi16(samples_row1_hi)
+                                      : LastRowSamples(samples_row11);
+    const __m128i samples_row13 = (max_luma_width == 32)
+                                      ? _mm_unpackhi_epi8(samples_row1_hi, zero)
+                                      : LastRowSamples(samples_row12);
+    const __m128i luma_sum0 = _mm_add_epi16(samples_row00, samples_row10);
+    const __m128i luma_sum1 = _mm_add_epi16(samples_row01, samples_row11);
+    const __m128i luma_sum2 = _mm_add_epi16(samples_row02, samples_row12);
+    const __m128i luma_sum3 = _mm_add_epi16(samples_row03, samples_row13);
+    __m128i sum = StoreLumaResults8_420(luma_sum0, luma_sum1, luma_ptr);
+    final_row_result =
+        StoreLumaResults8_420(luma_sum2, luma_sum3, luma_ptr + 8);
+    sum = _mm_add_epi16(sum, final_row_result);
+    final_sum = _mm_add_epi32(final_sum, _mm_cvtepu16_epi32(sum));
+    final_sum = _mm_add_epi32(final_sum, _mm_unpackhi_epi16(sum, zero));
+    src += stride << 1;
+    luma_ptr += kCflLumaBufferStride;
+  } while (++y < luma_height);
+
+  // Because max_luma_width is at most 32, any values beyond x=16 will
+  // necessarily be duplicated.
+  if (block_width_log2 == 5) {
+    const __m128i wide_fill = LastRowResult(final_row_result);
+    // Multiply duplicated value by number of occurrences, height * 4, since
+    // there are 16 in each row and the value appears in the vector 4 times.
+    final_sum = _mm_add_epi32(
+        final_sum,
+        _mm_slli_epi32(_mm_cvtepi16_epi32(wide_fill), block_height_log2 + 2));
+  }
+
+  // Begin second y section.
+  if (y < block_height) {
+    const __m128i final_fill0 =
+        LoadUnaligned16(luma_ptr - kCflLumaBufferStride);
+    const __m128i final_fill1 =
+        LoadUnaligned16(luma_ptr - kCflLumaBufferStride + 8);
+    const __m128i final_inner_sum = _mm_add_epi16(final_fill0, final_fill1);
+    const __m128i final_inner_sum0 = _mm_cvtepu16_epi32(final_inner_sum);
+    const __m128i final_inner_sum1 = _mm_unpackhi_epi16(final_inner_sum, zero);
+    const __m128i final_fill_to_sum =
+        _mm_add_epi32(final_inner_sum0, final_inner_sum1);
+
+    do {
+      StoreUnaligned16(luma_ptr, final_fill0);
+      StoreUnaligned16(luma_ptr + 8, final_fill1);
+      luma_ptr += kCflLumaBufferStride;
+
+      final_sum = _mm_add_epi32(final_sum, final_fill_to_sum);
+    } while (++y < block_height);
+  }  // End second y section.
+
+  final_sum = _mm_add_epi32(final_sum, _mm_srli_si128(final_sum, 8));
+  final_sum = _mm_add_epi32(final_sum, _mm_srli_si128(final_sum, 4));
+
+  __m128i averages = RightShiftWithRounding_S32(
+      final_sum, block_width_log2 + block_height_log2);
+  averages = _mm_shufflelo_epi16(averages, 0);
+  averages = _mm_shuffle_epi32(averages, 0);
+
+  luma_ptr = luma[0];
+  for (int y = 0; y < block_height; ++y, luma_ptr += kCflLumaBufferStride) {
+    const __m128i samples0 = LoadUnaligned16(luma_ptr);
+    StoreUnaligned16(luma_ptr, _mm_sub_epi16(samples0, averages));
+    const __m128i samples1 = LoadUnaligned16(luma_ptr + 8);
+    final_row_result = _mm_sub_epi16(samples1, averages);
+    StoreUnaligned16(luma_ptr + 8, final_row_result);
+  }
+  if (block_width_log2 == 5) {
+    int16_t* wide_luma_ptr = luma[0] + 16;
+    const __m128i wide_fill = LastRowResult(final_row_result);
+    for (int i = 0; i < block_height;
+         ++i, wide_luma_ptr += kCflLumaBufferStride) {
+      StoreUnaligned16(wide_luma_ptr, wide_fill);
+      StoreUnaligned16(wide_luma_ptr + 8, wide_fill);
+    }
+  }
+}
+
+template <int block_width_log2, int block_height_log2>
+void CflSubsampler420_WxH_SSE4_1(
+    int16_t luma[kCflLumaBufferStride][kCflLumaBufferStride],
+    const int max_luma_width, const int max_luma_height,
+    const void* const source, ptrdiff_t stride) {
+  switch (max_luma_width) {
+    case 8:
+      CflSubsampler420Impl_WxH_SSE4_1<block_width_log2, block_height_log2, 8>(
+          luma, max_luma_width, max_luma_height, source, stride);
+      return;
+    case 16:
+      CflSubsampler420Impl_WxH_SSE4_1<block_width_log2, block_height_log2, 16>(
+          luma, max_luma_width, max_luma_height, source, stride);
+      return;
+    case 24:
+      CflSubsampler420Impl_WxH_SSE4_1<block_width_log2, block_height_log2, 24>(
+          luma, max_luma_width, max_luma_height, source, stride);
+      return;
+    default:
+      assert(max_luma_width == 32);
+      CflSubsampler420Impl_WxH_SSE4_1<block_width_log2, block_height_log2, 32>(
+          luma, max_luma_width, max_luma_height, source, stride);
+      return;
+  }
+}
+
 void Init8bpp() {
   Dsp* const dsp = dsp_internal::GetWritableDspTable(8);
   assert(dsp != nullptr);
+#if DSP_ENABLED_8BPP_SSE4_1(TransformSize4x4_CflSubsampler420)
+  dsp->cfl_subsamplers[kTransformSize4x4][kSubsamplingType420] =
+      CflSubsampler420_4xH_SSE4_1<2>;
+#endif
+#if DSP_ENABLED_8BPP_SSE4_1(TransformSize4x8_CflSubsampler420)
+  dsp->cfl_subsamplers[kTransformSize4x8][kSubsamplingType420] =
+      CflSubsampler420_4xH_SSE4_1<3>;
+#endif
+#if DSP_ENABLED_8BPP_SSE4_1(TransformSize4x16_CflSubsampler420)
+  dsp->cfl_subsamplers[kTransformSize4x16][kSubsamplingType420] =
+      CflSubsampler420_4xH_SSE4_1<4>;
+#endif
+#if DSP_ENABLED_8BPP_SSE4_1(TransformSize8x4_CflSubsampler420)
+  dsp->cfl_subsamplers[kTransformSize8x4][kSubsamplingType420] =
+      CflSubsampler420_8xH_SSE4_1<2>;
+#endif
+#if DSP_ENABLED_8BPP_SSE4_1(TransformSize8x8_CflSubsampler420)
+  dsp->cfl_subsamplers[kTransformSize8x8][kSubsamplingType420] =
+      CflSubsampler420_8xH_SSE4_1<3>;
+#endif
+#if DSP_ENABLED_8BPP_SSE4_1(TransformSize8x16_CflSubsampler420)
+  dsp->cfl_subsamplers[kTransformSize8x16][kSubsamplingType420] =
+      CflSubsampler420_8xH_SSE4_1<4>;
+#endif
+#if DSP_ENABLED_8BPP_SSE4_1(TransformSize8x32_CflSubsampler420)
+  dsp->cfl_subsamplers[kTransformSize8x32][kSubsamplingType420] =
+      CflSubsampler420_8xH_SSE4_1<5>;
+#endif
+#if DSP_ENABLED_8BPP_SSE4_1(TransformSize16x4_CflSubsampler420)
+  dsp->cfl_subsamplers[kTransformSize16x4][kSubsamplingType420] =
+      CflSubsampler420_WxH_SSE4_1<4, 2>;
+#endif
+#if DSP_ENABLED_8BPP_SSE4_1(TransformSize16x8_CflSubsampler420)
+  dsp->cfl_subsamplers[kTransformSize16x8][kSubsamplingType420] =
+      CflSubsampler420_WxH_SSE4_1<4, 3>;
+#endif
+#if DSP_ENABLED_8BPP_SSE4_1(TransformSize16x16_CflSubsampler420)
+  dsp->cfl_subsamplers[kTransformSize16x16][kSubsamplingType420] =
+      CflSubsampler420_WxH_SSE4_1<4, 4>;
+#endif
+#if DSP_ENABLED_8BPP_SSE4_1(TransformSize16x32_CflSubsampler420)
+  dsp->cfl_subsamplers[kTransformSize16x32][kSubsamplingType420] =
+      CflSubsampler420_WxH_SSE4_1<4, 5>;
+#endif
+#if DSP_ENABLED_8BPP_SSE4_1(TransformSize32x8_CflSubsampler420)
+  dsp->cfl_subsamplers[kTransformSize32x8][kSubsamplingType420] =
+      CflSubsampler420_WxH_SSE4_1<5, 3>;
+#endif
+#if DSP_ENABLED_8BPP_SSE4_1(TransformSize32x16_CflSubsampler420)
+  dsp->cfl_subsamplers[kTransformSize32x16][kSubsamplingType420] =
+      CflSubsampler420_WxH_SSE4_1<5, 4>;
+#endif
+#if DSP_ENABLED_8BPP_SSE4_1(TransformSize32x32_CflSubsampler420)
+  dsp->cfl_subsamplers[kTransformSize32x32][kSubsamplingType420] =
+      CflSubsampler420_WxH_SSE4_1<5, 5>;
+#endif
+  // TODO(b/137035169): enable these once test vector mismatches are fixed.
+#if 0
 #if DSP_ENABLED_8BPP_SSE4_1(TransformSize4x4_CflSubsampler444)
   dsp->cfl_subsamplers[kTransformSize4x4][kSubsamplingType444] =
       CflSubsampler444_4xH_SSE4_1<2>;
@@ -273,6 +698,7 @@
   dsp->cfl_subsamplers[kTransformSize32x32][kSubsamplingType444] =
       CflSubsampler444_SSE4_1<5, 5>;
 #endif
+#endif
 #if DSP_ENABLED_8BPP_SSE4_1(TransformSize4x4_CflIntraPredictor)
   dsp->cfl_intra_predictors[kTransformSize4x4] = CflIntraPredictor_SSE4_1<4, 4>;
 #endif
diff --git a/libgav1/src/dsp/x86/intrapred_smooth_sse4.cc b/libgav1/src/dsp/x86/intrapred_smooth_sse4.cc
index 86504d7..eefe8c6 100644
--- a/libgav1/src/dsp/x86/intrapred_smooth_sse4.cc
+++ b/libgav1/src/dsp/x86/intrapred_smooth_sse4.cc
@@ -1,7 +1,5 @@
-#include "src/dsp/x86/intrapred_smooth_sse4.h"
-
 #include "src/dsp/dsp.h"
-#include "src/dsp/x86/intrapred_sse4.h"
+#include "src/dsp/intrapred.h"
 
 #if LIBGAV1_ENABLE_SSE4_1
 
@@ -146,7 +144,7 @@
 // pixels[1]: left vector
 // pixels[2]: right_pred vector
 inline void LoadSmoothPixels4(const uint8_t* above, const uint8_t* left,
-                              int height, __m128i* pixels) {
+                              const int height, __m128i* pixels) {
   if (height == 4) {
     pixels[1] = Load4(left);
   } else if (height == 8) {
@@ -542,7 +540,7 @@
   }
 }
 
-void SmoothHorizontal4x4_SSE4_1(void* dest, ptrdiff_t stride,
+void SmoothHorizontal4x4_SSE4_1(void* dest, const ptrdiff_t stride,
                                 const void* top_row, const void* left_column) {
   const auto* const top_ptr = static_cast<const uint8_t*>(top_row);
   const __m128i top_right = _mm_set1_epi32(top_ptr[3]);
@@ -563,7 +561,7 @@
   WriteSmoothHorizontalSum4<0xFF>(dst, left, weights, scaled_top_right, scale);
 }
 
-void SmoothHorizontal4x8_SSE4_1(void* const dest, ptrdiff_t stride,
+void SmoothHorizontal4x8_SSE4_1(void* const dest, const ptrdiff_t stride,
                                 const void* const top_row,
                                 const void* const left_column) {
   const auto* const top = static_cast<const uint8_t*>(top_row);
@@ -595,7 +593,7 @@
   WriteSmoothHorizontalSum4<0xFF>(dst, left, weights, scaled_top_right, scale);
 }
 
-void SmoothHorizontal4x16_SSE4_1(void* const dest, ptrdiff_t stride,
+void SmoothHorizontal4x16_SSE4_1(void* const dest, const ptrdiff_t stride,
                                  const void* const top_row,
                                  const void* const left_column) {
   const auto* const top = static_cast<const uint8_t*>(top_row);
@@ -647,7 +645,7 @@
   WriteSmoothHorizontalSum4<0xFF>(dst, left, weights, scaled_top_right, scale);
 }
 
-void SmoothHorizontal8x4_SSE4_1(void* const dest, ptrdiff_t stride,
+void SmoothHorizontal8x4_SSE4_1(void* const dest, const ptrdiff_t stride,
                                 const void* const top_row,
                                 const void* const left_column) {
   const auto* const top = static_cast<const uint8_t*>(top_row);
@@ -676,7 +674,7 @@
   WriteSmoothDirectionalSum8(dst, left_y, weights, scaled_top_right, scale);
 }
 
-void SmoothHorizontal8x8_SSE4_1(void* const dest, ptrdiff_t stride,
+void SmoothHorizontal8x8_SSE4_1(void* const dest, const ptrdiff_t stride,
                                 const void* const top_row,
                                 const void* const left_column) {
   const auto* const top = static_cast<const uint8_t*>(top_row);
@@ -696,7 +694,7 @@
   }
 }
 
-void SmoothHorizontal8x16_SSE4_1(void* const dest, ptrdiff_t stride,
+void SmoothHorizontal8x16_SSE4_1(void* const dest, const ptrdiff_t stride,
                                  const void* const top_row,
                                  const void* const left_column) {
   const auto* const top = static_cast<const uint8_t*>(top_row);
@@ -724,7 +722,7 @@
   }
 }
 
-void SmoothHorizontal8x32_SSE4_1(void* const dest, ptrdiff_t stride,
+void SmoothHorizontal8x32_SSE4_1(void* const dest, const ptrdiff_t stride,
                                  const void* const top_row,
                                  const void* const left_column) {
   const auto* const top = static_cast<const uint8_t*>(top_row);
@@ -766,7 +764,7 @@
   }
 }
 
-void SmoothHorizontal16x4_SSE4_1(void* const dest, ptrdiff_t stride,
+void SmoothHorizontal16x4_SSE4_1(void* const dest, const ptrdiff_t stride,
                                  const void* const top_row,
                                  const void* const left_column) {
   const auto* const top = static_cast<const uint8_t*>(top_row);
@@ -805,7 +803,7 @@
                               scaled_top_right1, scaled_top_right2, scale);
 }
 
-void SmoothHorizontal16x8_SSE4_1(void* const dest, ptrdiff_t stride,
+void SmoothHorizontal16x8_SSE4_1(void* const dest, const ptrdiff_t stride,
                                  const void* const top_row,
                                  const void* const left_column) {
   const auto* const top = static_cast<const uint8_t*>(top_row);
@@ -832,7 +830,7 @@
   }
 }
 
-void SmoothHorizontal16x16_SSE4_1(void* const dest, ptrdiff_t stride,
+void SmoothHorizontal16x16_SSE4_1(void* const dest, const ptrdiff_t stride,
                                   const void* const top_row,
                                   const void* const left_column) {
   const auto* const top = static_cast<const uint8_t*>(top_row);
@@ -868,7 +866,7 @@
   }
 }
 
-void SmoothHorizontal16x32_SSE4_1(void* const dest, ptrdiff_t stride,
+void SmoothHorizontal16x32_SSE4_1(void* const dest, const ptrdiff_t stride,
                                   const void* const top_row,
                                   const void* const left_column) {
   const auto* const top = static_cast<const uint8_t*>(top_row);
@@ -920,7 +918,7 @@
   }
 }
 
-void SmoothHorizontal16x64_SSE4_1(void* const dest, ptrdiff_t stride,
+void SmoothHorizontal16x64_SSE4_1(void* const dest, const ptrdiff_t stride,
                                   const void* const top_row,
                                   const void* const left_column) {
   const auto* const top = static_cast<const uint8_t*>(top_row);
@@ -950,7 +948,7 @@
   }
 }
 
-void SmoothHorizontal32x8_SSE4_1(void* const dest, ptrdiff_t stride,
+void SmoothHorizontal32x8_SSE4_1(void* const dest, const ptrdiff_t stride,
                                  const void* const top_row,
                                  const void* const left_column) {
   const auto* const top = static_cast<const uint8_t*>(top_row);
@@ -988,7 +986,7 @@
   }
 }
 
-void SmoothHorizontal32x16_SSE4_1(void* const dest, ptrdiff_t stride,
+void SmoothHorizontal32x16_SSE4_1(void* const dest, const ptrdiff_t stride,
                                   const void* const top_row,
                                   const void* const left_column) {
   const auto* const top = static_cast<const uint8_t*>(top_row);
@@ -1037,7 +1035,7 @@
   }
 }
 
-void SmoothHorizontal32x32_SSE4_1(void* const dest, ptrdiff_t stride,
+void SmoothHorizontal32x32_SSE4_1(void* const dest, const ptrdiff_t stride,
                                   const void* const top_row,
                                   const void* const left_column) {
   const auto* const top = static_cast<const uint8_t*>(top_row);
@@ -1106,7 +1104,7 @@
   }
 }
 
-void SmoothHorizontal32x64_SSE4_1(void* const dest, ptrdiff_t stride,
+void SmoothHorizontal32x64_SSE4_1(void* const dest, const ptrdiff_t stride,
                                   const void* const top_row,
                                   const void* const left_column) {
   const auto* const top = static_cast<const uint8_t*>(top_row);
@@ -1147,7 +1145,7 @@
   }
 }
 
-void SmoothHorizontal64x16_SSE4_1(void* const dest, ptrdiff_t stride,
+void SmoothHorizontal64x16_SSE4_1(void* const dest, const ptrdiff_t stride,
                                   const void* const top_row,
                                   const void* const left_column) {
   const auto* const top = static_cast<const uint8_t*>(top_row);
@@ -1222,7 +1220,7 @@
   }
 }
 
-void SmoothHorizontal64x32_SSE4_1(void* const dest, ptrdiff_t stride,
+void SmoothHorizontal64x32_SSE4_1(void* const dest, const ptrdiff_t stride,
                                   const void* const top_row,
                                   const void* const left_column) {
   const auto* const top = static_cast<const uint8_t*>(top_row);
@@ -1325,7 +1323,7 @@
   }
 }
 
-void SmoothHorizontal64x64_SSE4_1(void* const dest, ptrdiff_t stride,
+void SmoothHorizontal64x64_SSE4_1(void* const dest, const ptrdiff_t stride,
                                   const void* const top_row,
                                   const void* const left_column) {
   const auto* const top = static_cast<const uint8_t*>(top_row);
@@ -1389,7 +1387,7 @@
 }
 
 inline void LoadSmoothVerticalPixels4(const uint8_t* above, const uint8_t* left,
-                                      int height, __m128i* pixels) {
+                                      const int height, __m128i* pixels) {
   __m128i top = Load4(above);
   const __m128i bottom_left = _mm_set1_epi16(left[height - 1]);
   top = _mm_cvtepu8_epi16(top);
@@ -1400,8 +1398,8 @@
 // (256-w) counterparts. This is precomputed by the compiler when the weights
 // table is visible to this module. Removing this visibility can cut speed by up
 // to half in both 4xH and 8xH transforms.
-inline void LoadSmoothVerticalWeights4(const uint8_t* weight_array, int height,
-                                       __m128i* weights) {
+inline void LoadSmoothVerticalWeights4(const uint8_t* weight_array,
+                                       const int height, __m128i* weights) {
   const __m128i inverter = _mm_set1_epi16(256);
 
   if (height == 4) {
@@ -1423,7 +1421,8 @@
 }
 
 inline void WriteSmoothVertical4xH(const __m128i* pixel, const __m128i* weight,
-                                   int height, uint8_t* dst, ptrdiff_t stride) {
+                                   const int height, uint8_t* dst,
+                                   const ptrdiff_t stride) {
   const __m128i pred_round = _mm_set1_epi32(128);
   const __m128i mask_increment = _mm_set1_epi16(0x0202);
   const __m128i cvtepu8_epi32 = _mm_set1_epi32(0xC080400);
@@ -1447,7 +1446,7 @@
   }
 }
 
-void SmoothVertical4x4_SSE4_1(void* const dest, ptrdiff_t stride,
+void SmoothVertical4x4_SSE4_1(void* const dest, const ptrdiff_t stride,
                               const void* const top_row,
                               const void* const left_column) {
   const auto* const left = static_cast<const uint8_t*>(left_column);
@@ -1462,7 +1461,7 @@
   WriteSmoothVertical4xH(&pixels, weights, 4, dst, stride);
 }
 
-void SmoothVertical4x8_SSE4_1(void* const dest, ptrdiff_t stride,
+void SmoothVertical4x8_SSE4_1(void* const dest, const ptrdiff_t stride,
                               const void* const top_row,
                               const void* const left_column) {
   const auto* const left = static_cast<const uint8_t*>(left_column);
@@ -1477,7 +1476,7 @@
   WriteSmoothVertical4xH(&pixels, weights, 8, dst, stride);
 }
 
-void SmoothVertical4x16_SSE4_1(void* const dest, ptrdiff_t stride,
+void SmoothVertical4x16_SSE4_1(void* const dest, const ptrdiff_t stride,
                                const void* const top_row,
                                const void* const left_column) {
   const auto* const left = static_cast<const uint8_t*>(left_column);
@@ -1494,7 +1493,7 @@
   WriteSmoothVertical4xH(&pixels, &weights[2], 8, dst, stride);
 }
 
-void SmoothVertical8x4_SSE4_1(void* const dest, ptrdiff_t stride,
+void SmoothVertical8x4_SSE4_1(void* const dest, const ptrdiff_t stride,
                               const void* const top_row,
                               const void* const left_column) {
   const auto* const left_ptr = static_cast<const uint8_t*>(left_column);
@@ -1529,7 +1528,7 @@
   WriteSmoothDirectionalSum8(dst, top, weights_y, scaled_bottom_left_y, scale);
 }
 
-void SmoothVertical8x8_SSE4_1(void* const dest, ptrdiff_t stride,
+void SmoothVertical8x8_SSE4_1(void* const dest, const ptrdiff_t stride,
                               const void* const top_row,
                               const void* const left_column) {
   const auto* const left_ptr = static_cast<const uint8_t*>(left_column);
@@ -1553,7 +1552,7 @@
   }
 }
 
-void SmoothVertical8x16_SSE4_1(void* const dest, ptrdiff_t stride,
+void SmoothVertical8x16_SSE4_1(void* const dest, const ptrdiff_t stride,
                                const void* const top_row,
                                const void* const left_column) {
   const auto* const left_ptr = static_cast<const uint8_t*>(left_column);
@@ -1592,7 +1591,7 @@
   }
 }
 
-void SmoothVertical8x32_SSE4_1(void* const dest, ptrdiff_t stride,
+void SmoothVertical8x32_SSE4_1(void* const dest, const ptrdiff_t stride,
                                const void* const top_row,
                                const void* const left_column) {
   const auto* const left_ptr = static_cast<const uint8_t*>(left_column);
@@ -1658,7 +1657,7 @@
   }
 }
 
-void SmoothVertical16x4_SSE4_1(void* const dest, ptrdiff_t stride,
+void SmoothVertical16x4_SSE4_1(void* const dest, const ptrdiff_t stride,
                                const void* const top_row,
                                const void* const left_column) {
   const auto* const left_ptr = static_cast<const uint8_t*>(left_column);
@@ -1703,7 +1702,7 @@
                               scale);
 }
 
-void SmoothVertical16x8_SSE4_1(void* const dest, ptrdiff_t stride,
+void SmoothVertical16x8_SSE4_1(void* const dest, const ptrdiff_t stride,
                                const void* const top_row,
                                const void* const left_column) {
   const auto* const left_ptr = static_cast<const uint8_t*>(left_column);
@@ -1731,7 +1730,7 @@
   }
 }
 
-void SmoothVertical16x16_SSE4_1(void* const dest, ptrdiff_t stride,
+void SmoothVertical16x16_SSE4_1(void* const dest, const ptrdiff_t stride,
                                 const void* const top_row,
                                 const void* const left_column) {
   const auto* const left_ptr = static_cast<const uint8_t*>(left_column);
@@ -1775,7 +1774,7 @@
   }
 }
 
-void SmoothVertical16x32_SSE4_1(void* const dest, ptrdiff_t stride,
+void SmoothVertical16x32_SSE4_1(void* const dest, const ptrdiff_t stride,
                                 const void* const top_row,
                                 const void* const left_column) {
   const auto* const left_ptr = static_cast<const uint8_t*>(left_column);
@@ -1848,7 +1847,7 @@
   }
 }
 
-void SmoothVertical16x64_SSE4_1(void* const dest, ptrdiff_t stride,
+void SmoothVertical16x64_SSE4_1(void* const dest, const ptrdiff_t stride,
                                 const void* const top_row,
                                 const void* const left_column) {
   const auto* const left_ptr = static_cast<const uint8_t*>(left_column);
@@ -1896,7 +1895,7 @@
   }
 }
 
-void SmoothVertical32x8_SSE4_1(void* const dest, ptrdiff_t stride,
+void SmoothVertical32x8_SSE4_1(void* const dest, const ptrdiff_t stride,
                                const void* const top_row,
                                const void* const left_column) {
   const auto* const left_ptr = static_cast<const uint8_t*>(left_column);
@@ -1931,7 +1930,7 @@
   }
 }
 
-void SmoothVertical32x16_SSE4_1(void* const dest, ptrdiff_t stride,
+void SmoothVertical32x16_SSE4_1(void* const dest, const ptrdiff_t stride,
                                 const void* const top_row,
                                 const void* const left_column) {
   const auto* const left_ptr = static_cast<const uint8_t*>(left_column);
@@ -1984,7 +1983,7 @@
   }
 }
 
-void SmoothVertical32x32_SSE4_1(void* const dest, ptrdiff_t stride,
+void SmoothVertical32x32_SSE4_1(void* const dest, const ptrdiff_t stride,
                                 const void* const top_row,
                                 const void* const left_column) {
   const auto* const left_ptr = static_cast<const uint8_t*>(left_column);
@@ -2072,7 +2071,7 @@
   }
 }
 
-void SmoothVertical32x64_SSE4_1(void* const dest, ptrdiff_t stride,
+void SmoothVertical32x64_SSE4_1(void* const dest, const ptrdiff_t stride,
                                 const void* const top_row,
                                 const void* const left_column) {
   const auto* const left_ptr = static_cast<const uint8_t*>(left_column);
@@ -2129,7 +2128,7 @@
   }
 }
 
-void SmoothVertical64x16_SSE4_1(void* const dest, ptrdiff_t stride,
+void SmoothVertical64x16_SSE4_1(void* const dest, const ptrdiff_t stride,
                                 const void* const top_row,
                                 const void* const left_column) {
   const auto* const left_ptr = static_cast<const uint8_t*>(left_column);
@@ -2201,7 +2200,7 @@
   }
 }
 
-void SmoothVertical64x32_SSE4_1(void* const dest, ptrdiff_t stride,
+void SmoothVertical64x32_SSE4_1(void* const dest, const ptrdiff_t stride,
                                 const void* const top_row,
                                 const void* const left_column) {
   const auto* const left_ptr = static_cast<const uint8_t*>(left_column);
@@ -2320,7 +2319,7 @@
   }
 }
 
-void SmoothVertical64x64_SSE4_1(void* const dest, ptrdiff_t stride,
+void SmoothVertical64x64_SSE4_1(void* const dest, const ptrdiff_t stride,
                                 const void* const top_row,
                                 const void* const left_column) {
   const auto* const left_ptr = static_cast<const uint8_t*>(left_column);
diff --git a/libgav1/src/dsp/x86/intrapred_smooth_sse4.h b/libgav1/src/dsp/x86/intrapred_smooth_sse4.h
deleted file mode 100644
index 35c4a4e..0000000
--- a/libgav1/src/dsp/x86/intrapred_smooth_sse4.h
+++ /dev/null
@@ -1,10 +0,0 @@
-#ifndef LIBGAV1_SRC_DSP_X86_INTRAPRED_SMOOTH_SSE4_H_
-#define LIBGAV1_SRC_DSP_X86_INTRAPRED_SMOOTH_SSE4_H_
-namespace libgav1 {
-namespace dsp {
-
-void IntraPredSmoothInit_SSE4_1();
-
-}  // namespace dsp
-}  // namespace libgav1
-#endif  // LIBGAV1_SRC_DSP_X86_INTRAPRED_SMOOTH_SSE4_H_
diff --git a/libgav1/src/dsp/x86/intrapred_sse4.cc b/libgav1/src/dsp/x86/intrapred_sse4.cc
index bf1d939..8651437 100644
--- a/libgav1/src/dsp/x86/intrapred_sse4.cc
+++ b/libgav1/src/dsp/x86/intrapred_sse4.cc
@@ -1,6 +1,5 @@
-#include "src/dsp/x86/intrapred_sse4.h"
-
-#include "src/dsp/x86/intrapred_smooth_sse4.h"
+#include "src/dsp/dsp.h"
+#include "src/dsp/intrapred.h"
 
 #if LIBGAV1_ENABLE_SSE4_1
 
@@ -1417,7 +1416,8 @@
     memcpy(dst, top + offset + 3, width);
     return;
   }
-  for (int y = 0; y < height; y += 8, offset += 8) {
+  int y = 0;
+  do {
     memcpy(dst, top + offset, width);
     dst += stride;
     memcpy(dst, top + offset + 1, width);
@@ -1434,7 +1434,10 @@
     dst += stride;
     memcpy(dst, top + offset + 7, width);
     dst += stride;
-  }
+
+    offset += 8;
+    y += 8;
+  } while (y < height);
 }
 
 inline void DirectionalZone1_4xH(uint8_t* dst, ptrdiff_t stride,
@@ -1454,8 +1457,17 @@
   const __m128i offsets =
       _mm_set_epi32(0x00080007, 0x00060005, 0x00040003, 0x00020001);
 
-  for (int y = 0, top_x = xstep; y < height;
-       ++y, dst += stride, top_x += xstep) {
+  // All rows from |min_corner_only_y| down will simply use memcpy. |max_base_x|
+  // is always greater than |height|, so clipping to 1 is enough to make the
+  // logic work.
+  const int xstep_units = std::max(xstep >> scale_bits, 1);
+  const int min_corner_only_y = std::min(max_base_x / xstep_units, height);
+
+  // Rows up to this y-value can be computed without checking for bounds.
+  int y = 0;
+  int top_x = xstep;
+
+  for (; y < min_corner_only_y; ++y, dst += stride, top_x += xstep) {
     const int top_base_x = top_x >> scale_bits;
 
     // Permit negative values of |top_x|.
@@ -1475,9 +1487,114 @@
     const __m128i past_max = _mm_cmpgt_epi16(top_index_vect, max_base_x_vect);
     __m128i prod = _mm_maddubs_epi16(sampled_values, shifts);
     prod = RightShiftWithRounding_U16(prod, rounding_bits);
+    // Replace pixels from invalid range with top-right corner.
     prod = _mm_blendv_epi8(prod, final_top_val, past_max);
     Store4(dst, _mm_packus_epi16(prod, prod));
   }
+
+  // Fill in corner-only rows.
+  for (; y < height; ++y) {
+    memset(dst, top[max_base_x], /* width */ 4);
+    dst += stride;
+  }
+}
+
+// 7.11.2.4 (7) angle < 90
+inline void DirectionalZone1_Large(uint8_t* dest, ptrdiff_t stride,
+                                   const uint8_t* const top_row,
+                                   const int width, const int height,
+                                   const int xstep, const bool upsampled) {
+  const int upsample_shift = static_cast<int>(upsampled);
+  const __m128i sampler =
+      upsampled ? _mm_set_epi32(0x0F0E0D0C, 0x0B0A0908, 0x07060504, 0x03020100)
+                : _mm_set_epi32(0x08070706, 0x06050504, 0x04030302, 0x02010100);
+  const int scale_bits = 6 - upsample_shift;
+  const int max_base_x = ((width + height) - 1) << upsample_shift;
+
+  const __m128i max_shift = _mm_set1_epi8(32);
+  const int rounding_bits = 5;
+  const int base_step = 1 << upsample_shift;
+  const int base_step8 = base_step << 3;
+
+  // All rows from |min_corner_only_y| down will simply use memcpy. |max_base_x|
+  // is always greater than |height|, so clipping to 1 is enough to make the
+  // logic work.
+  const int xstep_units = std::max(xstep >> scale_bits, 1);
+  const int min_corner_only_y = std::min(max_base_x / xstep_units, height);
+
+  // Rows up to this y-value can be computed without checking for bounds.
+  const int max_no_corner_y = std::min(
+      LeftShift((max_base_x - (base_step * width)), scale_bits) / xstep,
+      height);
+  // No need to check for exceeding |max_base_x| in the first loop.
+  int y = 0;
+  int top_x = xstep;
+  for (; y < max_no_corner_y; ++y, dest += stride, top_x += xstep) {
+    int top_base_x = top_x >> scale_bits;
+    // Permit negative values of |top_x|.
+    const int shift_val = (LeftShift(top_x, upsample_shift) & 0x3F) >> 1;
+    const __m128i shift = _mm_set1_epi8(shift_val);
+    const __m128i opposite_shift = _mm_sub_epi8(max_shift, shift);
+    const __m128i shifts = _mm_unpacklo_epi8(opposite_shift, shift);
+    int x = 0;
+    do {
+      const __m128i top_vals = LoadUnaligned16(top_row + top_base_x);
+      __m128i vals = _mm_shuffle_epi8(top_vals, sampler);
+      vals = _mm_maddubs_epi16(vals, shifts);
+      vals = RightShiftWithRounding_U16(vals, rounding_bits);
+      StoreLo8(dest + x, _mm_packus_epi16(vals, vals));
+      top_base_x += base_step8;
+      x += 8;
+    } while (x < width);
+  }
+
+  // Each 16-bit value here corresponds to a position that may exceed
+  // |max_base_x|. When added to the top_base_x, it is used to mask values
+  // that pass the end of |top|. Starting from 1 to simulate "cmpge" which is
+  // not supported for packed integers.
+  const __m128i offsets =
+      _mm_set_epi32(0x00080007, 0x00060005, 0x00040003, 0x00020001);
+
+  const __m128i max_base_x_vect = _mm_set1_epi16(max_base_x);
+  const __m128i final_top_val = _mm_set1_epi16(top_row[max_base_x]);
+  const __m128i base_step8_vect = _mm_set1_epi16(base_step8);
+  for (; y < min_corner_only_y; ++y, dest += stride, top_x += xstep) {
+    int top_base_x = top_x >> scale_bits;
+
+    const int shift_val = (LeftShift(top_x, upsample_shift) & 0x3F) >> 1;
+    const __m128i shift = _mm_set1_epi8(shift_val);
+    const __m128i opposite_shift = _mm_sub_epi8(max_shift, shift);
+    const __m128i shifts = _mm_unpacklo_epi8(opposite_shift, shift);
+    __m128i top_index_vect = _mm_set1_epi16(top_base_x);
+    top_index_vect = _mm_add_epi16(top_index_vect, offsets);
+
+    int x = 0;
+    const int min_corner_only_x =
+        std::min(width, ((max_base_x - top_base_x) >> upsample_shift) + 7) & ~7;
+    for (; x < min_corner_only_x;
+         x += 8, top_base_x += base_step8,
+         top_index_vect = _mm_add_epi16(top_index_vect, base_step8_vect)) {
+      const __m128i past_max = _mm_cmpgt_epi16(top_index_vect, max_base_x_vect);
+      // Assuming a buffer zone of 8 bytes at the end of top_row, this prevents
+      // reading out of bounds. If all indices are past max and we don't need to
+      // use the loaded bytes at all, |top_base_x| becomes 0. |top_base_x| will
+      // reset for the next |y|.
+      top_base_x &= ~_mm_cvtsi128_si32(past_max);
+      const __m128i top_vals = LoadUnaligned16(top_row + top_base_x);
+      __m128i vals = _mm_shuffle_epi8(top_vals, sampler);
+      vals = _mm_maddubs_epi16(vals, shifts);
+      vals = RightShiftWithRounding_U16(vals, rounding_bits);
+      vals = _mm_blendv_epi8(vals, final_top_val, past_max);
+      StoreLo8(dest + x, _mm_packus_epi16(vals, vals));
+    }
+    // Corner-only section of the row.
+    memset(dest + x, top_row[max_base_x], width - x);
+  }
+  // Fill in corner-only rows.
+  for (; y < height; ++y) {
+    memset(dest, top_row[max_base_x], width);
+    dest += stride;
+  }
 }
 
 // 7.11.2.4 (7) angle < 90
@@ -1494,6 +1611,11 @@
     DirectionalZone1_4xH(dest, stride, top_row, height, xstep, upsampled);
     return;
   }
+  if (width >= 32) {
+    DirectionalZone1_Large(dest, stride, top_row, width, height, xstep,
+                           upsampled);
+    return;
+  }
   const __m128i sampler =
       upsampled ? _mm_set_epi32(0x0F0E0D0C, 0x0B0A0908, 0x07060504, 0x03020100)
                 : _mm_set_epi32(0x08070706, 0x06050504, 0x04030302, 0x02010100);
@@ -1507,22 +1629,28 @@
 
   // No need to check for exceeding |max_base_x| in the loops.
   if (((xstep * height) >> scale_bits) + base_step * width < max_base_x) {
-    for (int y = 0, top_x = xstep; y < height;
-         ++y, dest += stride, top_x += xstep) {
+    int top_x = xstep;
+    int y = 0;
+    do {
       int top_base_x = top_x >> scale_bits;
       // Permit negative values of |top_x|.
       const int shift_val = (LeftShift(top_x, upsample_shift) & 0x3F) >> 1;
       const __m128i shift = _mm_set1_epi8(shift_val);
       const __m128i opposite_shift = _mm_sub_epi8(max_shift, shift);
       const __m128i shifts = _mm_unpacklo_epi8(opposite_shift, shift);
-      for (int x = 0; x < width; x += 8, top_base_x += base_step8) {
+      int x = 0;
+      do {
         const __m128i top_vals = LoadUnaligned16(top_row + top_base_x);
         __m128i vals = _mm_shuffle_epi8(top_vals, sampler);
         vals = _mm_maddubs_epi16(vals, shifts);
         vals = RightShiftWithRounding_U16(vals, rounding_bits);
         StoreLo8(dest + x, _mm_packus_epi16(vals, vals));
-      }
-    }
+        top_base_x += base_step8;
+        x += 8;
+      } while (x < width);
+      dest += stride;
+      top_x += xstep;
+    } while (++y < height);
     return;
   }
 
@@ -1536,8 +1664,9 @@
   const __m128i max_base_x_vect = _mm_set1_epi16(max_base_x);
   const __m128i final_top_val = _mm_set1_epi16(top_row[max_base_x]);
   const __m128i base_step8_vect = _mm_set1_epi16(base_step8);
-  for (int y = 0, top_x = xstep; y < height;
-       ++y, dest += stride, top_x += xstep) {
+  int top_x = xstep;
+  int y = 0;
+  do {
     int top_base_x = top_x >> scale_bits;
 
     if (top_base_x >= max_base_x) {
@@ -1585,7 +1714,9 @@
     vals = RightShiftWithRounding_U16(vals, rounding_bits);
     vals = _mm_blendv_epi8(vals, final_top_val, past_max);
     StoreLo8(dest + x, _mm_packus_epi16(vals, vals));
-  }
+    dest += stride;
+    top_x += xstep;
+  } while (++y < height);
 }
 
 void DirectionalIntraPredictorZone1_SSE4_1(void* const dest, ptrdiff_t stride,
@@ -1690,42 +1821,70 @@
   if (width == 4 || height == 4) {
     const ptrdiff_t stride4 = stride << 2;
     if (upsampled) {
-      for (int x = 0, left_y = ystep; x < width; x += 4, left_y += ystep << 2) {
+      int left_y = ystep;
+      int x = 0;
+      do {
         uint8_t* dst_x = dst + x;
-        for (int y = 0; y < height; y += 4, dst_x += stride4) {
+        int y = 0;
+        do {
           DirectionalZone3_4x4<true>(
               dst_x, stride, left_ptr + (y << upsample_shift), left_y, ystep);
-        }
-      }
+          dst_x += stride4;
+          y += 4;
+        } while (y < height);
+        left_y += ystep << 2;
+        x += 4;
+      } while (x < width);
     } else {
-      for (int x = 0, left_y = ystep; x < width; x += 4, left_y += ystep << 2) {
+      int left_y = ystep;
+      int x = 0;
+      do {
         uint8_t* dst_x = dst + x;
-        for (int y = 0; y < height; y += 4, dst_x += stride4) {
+        int y = 0;
+        do {
           DirectionalZone3_4x4<false>(dst_x, stride, left_ptr + y, left_y,
                                       ystep);
-        }
-      }
+          dst_x += stride4;
+          y += 4;
+        } while (y < height);
+        left_y += ystep << 2;
+        x += 4;
+      } while (x < width);
     }
     return;
   }
 
   const ptrdiff_t stride8 = stride << 3;
   if (upsampled) {
-    for (int x = 0, left_y = ystep; x < width; x += 8, left_y += ystep << 3) {
+    int left_y = ystep;
+    int x = 0;
+    do {
       uint8_t* dst_x = dst + x;
-      for (int y = 0; y < height; y += 8, dst_x += stride8) {
+      int y = 0;
+      do {
         DirectionalZone3_8xH<true, 8>(
             dst_x, stride, left_ptr + (y << upsample_shift), left_y, ystep);
-      }
-    }
+        dst_x += stride8;
+        y += 8;
+      } while (y < height);
+      left_y += ystep << 3;
+      x += 8;
+    } while (x < width);
   } else {
-    for (int x = 0, left_y = ystep; x < width; x += 8, left_y += ystep << 3) {
+    int left_y = ystep;
+    int x = 0;
+    do {
       uint8_t* dst_x = dst + x;
-      for (int y = 0; y < height; y += 8, dst_x += stride8) {
+      int y = 0;
+      do {
         DirectionalZone3_8xH<false, 8>(
             dst_x, stride, left_ptr + (y << upsample_shift), left_y, ystep);
-      }
-    }
+        dst_x += stride8;
+        y += 8;
+      } while (y < height);
+      left_y += ystep << 3;
+      x += 8;
+    } while (x < width);
   }
 }
 
@@ -1988,7 +2147,7 @@
   // This loop treats each set of 4 columns in 3 stages with y-value boundaries.
   // The first stage, before the first y-loop, covers blocks that are only
   // computed from the top row. The second stage, comprising two y-loops, covers
-  // blocks that have a mixture of values computer from top or left. The final
+  // blocks that have a mixture of values computed from top or left. The final
   // stage covers blocks that are only computed from the left.
   for (int left_offset = -left_base_increment; x < min_top_only_x;
        x += 8,
diff --git a/libgav1/src/dsp/x86/intrapred_sse4.h b/libgav1/src/dsp/x86/intrapred_sse4.h
index bf6e2fe..ec5dfb0 100644
--- a/libgav1/src/dsp/x86/intrapred_sse4.h
+++ b/libgav1/src/dsp/x86/intrapred_sse4.h
@@ -3,15 +3,17 @@
 
 #include "src/dsp/cpu.h"
 #include "src/dsp/dsp.h"
-#include "src/dsp/intrapred.h"
 
 namespace libgav1 {
 namespace dsp {
 
-// Initializes Dsp::intra_predictors with sse4 implementations. This function
-// is not thread-safe.
+// Initializes Dsp::intra_predictors, Dsp::directional_intra_predictor_zone*,
+// Dsp::cfl_intra_predictors, Dsp::cfl_subsamplers and
+// Dsp::filter_intra_predictor, see the defines below for specifics. These
+// functions are not thread-safe.
 void IntraPredInit_SSE4_1();
 void IntraPredCflInit_SSE4_1();
+void IntraPredSmoothInit_SSE4_1();
 
 }  // namespace dsp
 }  // namespace libgav1
@@ -120,6 +122,64 @@
   LIBGAV1_DSP_SSE4_1
 #endif
 
+#ifndef LIBGAV1_Dsp8bpp_TransformSize4x4_CflSubsampler420
+#define LIBGAV1_Dsp8bpp_TransformSize4x4_CflSubsampler420 LIBGAV1_DSP_SSE4_1
+#endif
+
+#ifndef LIBGAV1_Dsp8bpp_TransformSize4x8_CflSubsampler420
+#define LIBGAV1_Dsp8bpp_TransformSize4x8_CflSubsampler420 LIBGAV1_DSP_SSE4_1
+#endif
+
+#ifndef LIBGAV1_Dsp8bpp_TransformSize4x16_CflSubsampler420
+#define LIBGAV1_Dsp8bpp_TransformSize4x16_CflSubsampler420 LIBGAV1_DSP_SSE4_1
+#endif
+
+#ifndef LIBGAV1_Dsp8bpp_TransformSize8x4_CflSubsampler420
+#define LIBGAV1_Dsp8bpp_TransformSize8x4_CflSubsampler420 LIBGAV1_DSP_SSE4_1
+#endif
+
+#ifndef LIBGAV1_Dsp8bpp_TransformSize8x8_CflSubsampler420
+#define LIBGAV1_Dsp8bpp_TransformSize8x8_CflSubsampler420 LIBGAV1_DSP_SSE4_1
+#endif
+
+#ifndef LIBGAV1_Dsp8bpp_TransformSize8x16_CflSubsampler420
+#define LIBGAV1_Dsp8bpp_TransformSize8x16_CflSubsampler420 LIBGAV1_DSP_SSE4_1
+#endif
+
+#ifndef LIBGAV1_Dsp8bpp_TransformSize8x32_CflSubsampler420
+#define LIBGAV1_Dsp8bpp_TransformSize8x32_CflSubsampler420 LIBGAV1_DSP_SSE4_1
+#endif
+
+#ifndef LIBGAV1_Dsp8bpp_TransformSize16x4_CflSubsampler420
+#define LIBGAV1_Dsp8bpp_TransformSize16x4_CflSubsampler420 LIBGAV1_DSP_SSE4_1
+#endif
+
+#ifndef LIBGAV1_Dsp8bpp_TransformSize16x8_CflSubsampler420
+#define LIBGAV1_Dsp8bpp_TransformSize16x8_CflSubsampler420 LIBGAV1_DSP_SSE4_1
+#endif
+
+#ifndef LIBGAV1_Dsp8bpp_TransformSize16x16_CflSubsampler420
+#define LIBGAV1_Dsp8bpp_TransformSize16x16_CflSubsampler420 LIBGAV1_DSP_SSE4_1
+#endif
+
+#ifndef LIBGAV1_Dsp8bpp_TransformSize16x32_CflSubsampler420
+#define LIBGAV1_Dsp8bpp_TransformSize16x32_CflSubsampler420 LIBGAV1_DSP_SSE4_1
+#endif
+
+#ifndef LIBGAV1_Dsp8bpp_TransformSize32x8_CflSubsampler420
+#define LIBGAV1_Dsp8bpp_TransformSize32x8_CflSubsampler420 LIBGAV1_DSP_SSE4_1
+#endif
+
+#ifndef LIBGAV1_Dsp8bpp_TransformSize32x16_CflSubsampler420
+#define LIBGAV1_Dsp8bpp_TransformSize32x16_CflSubsampler420 LIBGAV1_DSP_SSE4_1
+#endif
+
+#ifndef LIBGAV1_Dsp8bpp_TransformSize32x32_CflSubsampler420
+#define LIBGAV1_Dsp8bpp_TransformSize32x32_CflSubsampler420 LIBGAV1_DSP_SSE4_1
+#endif
+
+// TODO(b/137035169): enable these once test vector mismatches are fixed.
+#if 0
 #ifndef LIBGAV1_Dsp8bpp_TransformSize4x4_CflSubsampler444
 #define LIBGAV1_Dsp8bpp_TransformSize4x4_CflSubsampler444 LIBGAV1_DSP_SSE4_1
 #endif
@@ -175,6 +235,7 @@
 #ifndef LIBGAV1_Dsp8bpp_TransformSize32x32_CflSubsampler444
 #define LIBGAV1_Dsp8bpp_TransformSize32x32_CflSubsampler444 LIBGAV1_DSP_SSE4_1
 #endif
+#endif
 
 #ifndef LIBGAV1_Dsp8bpp_TransformSize4x4_CflIntraPredictor
 #define LIBGAV1_Dsp8bpp_TransformSize4x4_CflIntraPredictor LIBGAV1_DSP_SSE4_1
diff --git a/libgav1/src/dsp/x86/inverse_transform_sse4.cc b/libgav1/src/dsp/x86/inverse_transform_sse4.cc
index 65bc2f3..81f6244 100644
--- a/libgav1/src/dsp/x86/inverse_transform_sse4.cc
+++ b/libgav1/src/dsp/x86/inverse_transform_sse4.cc
@@ -1,4 +1,5 @@
-#include "src/dsp/x86/inverse_transform_sse4.h"
+#include "src/dsp/dsp.h"
+#include "src/dsp/inverse_transform.h"
 
 #if LIBGAV1_ENABLE_SSE4_1
 
@@ -10,6 +11,7 @@
 #include <cstring>
 
 #include "src/dsp/x86/common_sse4.h"
+#include "src/dsp/x86/transpose_sse4.h"
 #include "src/utils/array_2d.h"
 #include "src/utils/common.h"
 #include "src/utils/compiler_attributes.h"
@@ -22,176 +24,6 @@
 // Include the constants and utility functions inside the anonymous namespace.
 #include "src/dsp/inverse_transform.inc"
 
-// TODO(slavarnway): move to transpose_sse4.h
-LIBGAV1_ALWAYS_INLINE void Transpose16_4x4(const __m128i* in, __m128i* out) {
-  const __m128i ba = _mm_unpacklo_epi16(in[0], in[1]);
-  const __m128i dc = _mm_unpacklo_epi16(in[2], in[3]);
-  const __m128i dcba_lo = _mm_unpacklo_epi32(ba, dc);
-  const __m128i dcba_hi = _mm_unpackhi_epi32(ba, dc);
-  out[0] = dcba_lo;
-  out[1] = _mm_srli_si128(dcba_lo, 8);
-  out[2] = dcba_hi;
-  out[3] = _mm_srli_si128(dcba_hi, 8);
-}
-
-// TODO(slavarnway): move to transpose_sse4.h
-LIBGAV1_ALWAYS_INLINE void Transpose16_8x4To4x8(const __m128i* in,
-                                                __m128i* out) {
-  // Unpack 16 bit elements. Goes from:
-  // in[0]: 00 01 02 03  04 05 06 07
-  // in[1]: 10 11 12 13  14 15 16 17
-  // in[2]: 20 21 22 23  24 25 26 27
-  // in[3]: 30 31 32 33  34 35 36 37
-
-  // to:
-  // a0:    00 10 01 11  02 12 03 13
-  // a1:    20 30 21 31  22 32 23 33
-  // a4:    04 14 05 15  06 16 07 17
-  // a5:    24 34 25 35  26 36 27 37
-  const __m128i a0 = _mm_unpacklo_epi16(in[0], in[1]);
-  const __m128i a1 = _mm_unpacklo_epi16(in[2], in[3]);
-  const __m128i a4 = _mm_unpackhi_epi16(in[0], in[1]);
-  const __m128i a5 = _mm_unpackhi_epi16(in[2], in[3]);
-
-  // Unpack 32 bit elements resulting in:
-  // b0: 00 10 20 30  01 11 21 31
-  // b2: 04 14 24 34  05 15 25 35
-  // b4: 02 12 22 32  03 13 23 33
-  // b6: 06 16 26 36  07 17 27 37
-  const __m128i b0 = _mm_unpacklo_epi32(a0, a1);
-  const __m128i b2 = _mm_unpacklo_epi32(a4, a5);
-  const __m128i b4 = _mm_unpackhi_epi32(a0, a1);
-  const __m128i b6 = _mm_unpackhi_epi32(a4, a5);
-
-  // Unpack 64 bit elements resulting in:
-  // out[0]: 00 10 20 30  XX XX XX XX
-  // out[1]: 01 11 21 31  XX XX XX XX
-  // out[2]: 02 12 22 32  XX XX XX XX
-  // out[3]: 03 13 23 33  XX XX XX XX
-  // out[4]: 04 14 24 34  XX XX XX XX
-  // out[5]: 05 15 25 35  XX XX XX XX
-  // out[6]: 06 16 26 36  XX XX XX XX
-  // out[7]: 07 17 27 37  XX XX XX XX
-  const __m128i zeros = _mm_setzero_si128();
-  out[0] = _mm_unpacklo_epi64(b0, zeros);
-  out[1] = _mm_unpackhi_epi64(b0, zeros);
-  out[2] = _mm_unpacklo_epi64(b4, zeros);
-  out[3] = _mm_unpackhi_epi64(b4, zeros);
-  out[4] = _mm_unpacklo_epi64(b2, zeros);
-  out[5] = _mm_unpackhi_epi64(b2, zeros);
-  out[6] = _mm_unpacklo_epi64(b6, zeros);
-  out[7] = _mm_unpackhi_epi64(b6, zeros);
-}
-
-// TODO(slavarnway): move to transpose_sse4.h
-LIBGAV1_ALWAYS_INLINE void Transpose16_4x8To8x4(const __m128i* in,
-                                                __m128i* out) {
-  // Unpack 16 bit elements. Goes from:
-  // in[0]: 00 01 02 03  XX XX XX XX
-  // in[1]: 10 11 12 13  XX XX XX XX
-  // in[2]: 20 21 22 23  XX XX XX XX
-  // in[3]: 30 31 32 33  XX XX XX XX
-  // in[4]: 40 41 42 43  XX XX XX XX
-  // in[5]: 50 51 52 53  XX XX XX XX
-  // in[6]: 60 61 62 63  XX XX XX XX
-  // in[7]: 70 71 72 73  XX XX XX XX
-  // to:
-  // a0:    00 10 01 11  02 12 03 13
-  // a1:    20 30 21 31  22 32 23 33
-  // a2:    40 50 41 51  42 52 43 53
-  // a3:    60 70 61 71  62 72 63 73
-  const __m128i a0 = _mm_unpacklo_epi16(in[0], in[1]);
-  const __m128i a1 = _mm_unpacklo_epi16(in[2], in[3]);
-  const __m128i a2 = _mm_unpacklo_epi16(in[4], in[5]);
-  const __m128i a3 = _mm_unpacklo_epi16(in[6], in[7]);
-
-  // Unpack 32 bit elements resulting in:
-  // b0: 00 10 20 30  01 11 21 31
-  // b1: 40 50 60 70  41 51 61 71
-  // b2: 02 12 22 32  03 13 23 33
-  // b3: 42 52 62 72  43 53 63 73
-  const __m128i b0 = _mm_unpacklo_epi32(a0, a1);
-  const __m128i b1 = _mm_unpacklo_epi32(a2, a3);
-  const __m128i b2 = _mm_unpackhi_epi32(a0, a1);
-  const __m128i b3 = _mm_unpackhi_epi32(a2, a3);
-
-  // Unpack 64 bit elements resulting in:
-  // out[0]: 00 10 20 30  40 50 60 70
-  // out[1]: 01 11 21 31  41 51 61 71
-  // out[2]: 02 12 22 32  42 52 62 72
-  // out[3]: 03 13 23 33  43 53 63 73
-  out[0] = _mm_unpacklo_epi64(b0, b1);
-  out[1] = _mm_unpackhi_epi64(b0, b1);
-  out[2] = _mm_unpacklo_epi64(b2, b3);
-  out[3] = _mm_unpackhi_epi64(b2, b3);
-}
-
-// TODO(slavarnway): move to transpose_sse4.h
-LIBGAV1_ALWAYS_INLINE void Transpose16_8x8(const __m128i* in, __m128i* out) {
-  // Unpack 16 bit elements. Goes from:
-  // in[0]: 00 01 02 03  04 05 06 07
-  // in[1]: 10 11 12 13  14 15 16 17
-  // in[2]: 20 21 22 23  24 25 26 27
-  // in[3]: 30 31 32 33  34 35 36 37
-  // in[4]: 40 41 42 43  44 45 46 47
-  // in[5]: 50 51 52 53  54 55 56 57
-  // in[6]: 60 61 62 63  64 65 66 67
-  // in[7]: 70 71 72 73  74 75 76 77
-  // to:
-  // a0:    00 10 01 11  02 12 03 13
-  // a1:    20 30 21 31  22 32 23 33
-  // a2:    40 50 41 51  42 52 43 53
-  // a3:    60 70 61 71  62 72 63 73
-  // a4:    04 14 05 15  06 16 07 17
-  // a5:    24 34 25 35  26 36 27 37
-  // a6:    44 54 45 55  46 56 47 57
-  // a7:    64 74 65 75  66 76 67 77
-  const __m128i a0 = _mm_unpacklo_epi16(in[0], in[1]);
-  const __m128i a1 = _mm_unpacklo_epi16(in[2], in[3]);
-  const __m128i a2 = _mm_unpacklo_epi16(in[4], in[5]);
-  const __m128i a3 = _mm_unpacklo_epi16(in[6], in[7]);
-  const __m128i a4 = _mm_unpackhi_epi16(in[0], in[1]);
-  const __m128i a5 = _mm_unpackhi_epi16(in[2], in[3]);
-  const __m128i a6 = _mm_unpackhi_epi16(in[4], in[5]);
-  const __m128i a7 = _mm_unpackhi_epi16(in[6], in[7]);
-
-  // Unpack 32 bit elements resulting in:
-  // b0: 00 10 20 30  01 11 21 31
-  // b1: 40 50 60 70  41 51 61 71
-  // b2: 04 14 24 34  05 15 25 35
-  // b3: 44 54 64 74  45 55 65 75
-  // b4: 02 12 22 32  03 13 23 33
-  // b5: 42 52 62 72  43 53 63 73
-  // b6: 06 16 26 36  07 17 27 37
-  // b7: 46 56 66 76  47 57 67 77
-  const __m128i b0 = _mm_unpacklo_epi32(a0, a1);
-  const __m128i b1 = _mm_unpacklo_epi32(a2, a3);
-  const __m128i b2 = _mm_unpacklo_epi32(a4, a5);
-  const __m128i b3 = _mm_unpacklo_epi32(a6, a7);
-  const __m128i b4 = _mm_unpackhi_epi32(a0, a1);
-  const __m128i b5 = _mm_unpackhi_epi32(a2, a3);
-  const __m128i b6 = _mm_unpackhi_epi32(a4, a5);
-  const __m128i b7 = _mm_unpackhi_epi32(a6, a7);
-
-  // Unpack 64 bit elements resulting in:
-  // out[0]: 00 10 20 30  40 50 60 70
-  // out[1]: 01 11 21 31  41 51 61 71
-  // out[2]: 02 12 22 32  42 52 62 72
-  // out[3]: 03 13 23 33  43 53 63 73
-  // out[4]: 04 14 24 34  44 54 64 74
-  // out[5]: 05 15 25 35  45 55 65 75
-  // out[6]: 06 16 26 36  46 56 66 76
-  // out[7]: 07 17 27 37  47 57 67 77
-  out[0] = _mm_unpacklo_epi64(b0, b1);
-  out[1] = _mm_unpackhi_epi64(b0, b1);
-  out[2] = _mm_unpacklo_epi64(b4, b5);
-  out[3] = _mm_unpackhi_epi64(b4, b5);
-  out[4] = _mm_unpacklo_epi64(b2, b3);
-  out[5] = _mm_unpackhi_epi64(b2, b3);
-  out[6] = _mm_unpacklo_epi64(b6, b7);
-  out[7] = _mm_unpackhi_epi64(b6, b7);
-}
-
 template <int store_width, int store_count>
 LIBGAV1_ALWAYS_INLINE void StoreDst(int16_t* dst, int32_t stride, int32_t idx,
                                     const __m128i* s) {
@@ -385,14 +217,14 @@
     if (transpose) {
       __m128i input[8];
       LoadSrc<8, 8>(src, step, 0, input);
-      Transpose16_4x8To8x4(input, x);
+      Transpose4x8To8x4_U16(input, x);
     } else {
       LoadSrc<16, 4>(src, step, 0, x);
     }
   } else {
     LoadSrc<8, 4>(src, step, 0, x);
     if (transpose) {
-      Transpose16_4x4(x, x);
+      Transpose4x4_U16(x, x);
     }
   }
   // stage 1.
@@ -407,14 +239,14 @@
   if (stage_is_rectangular) {
     if (transpose) {
       __m128i output[8];
-      Transpose16_8x4To4x8(s, output);
+      Transpose8x4To4x8_U16(s, output);
       StoreDst<8, 8>(dst, step, 0, output);
     } else {
       StoreDst<16, 4>(dst, step, 0, s);
     }
   } else {
     if (transpose) {
-      Transpose16_4x4(s, s);
+      Transpose4x4_U16(s, s);
     }
     StoreDst<8, 4>(dst, step, 0, s);
   }
@@ -458,7 +290,7 @@
     if (transpose) {
       __m128i input[4];
       LoadSrc<16, 4>(src, step, 0, input);
-      Transpose16_8x4To4x8(input, x);
+      Transpose8x4To4x8_U16(input, x);
     } else {
       LoadSrc<8, 8>(src, step, 0, x);
     }
@@ -466,7 +298,7 @@
     if (transpose) {
       __m128i input[8];
       LoadSrc<16, 8>(src, step, 0, input);
-      Transpose16_8x8(input, x);
+      Transpose8x8_U16(input, x);
     } else {
       LoadSrc<16, 8>(src, step, 0, x);
     }
@@ -489,7 +321,7 @@
   if (stage_is_rectangular) {
     if (transpose) {
       __m128i output[4];
-      Transpose16_4x8To8x4(s, output);
+      Transpose4x8To8x4_U16(s, output);
       StoreDst<16, 4>(dst, step, 0, output);
     } else {
       StoreDst<8, 8>(dst, step, 0, s);
@@ -497,7 +329,7 @@
   } else {
     if (transpose) {
       __m128i output[8];
-      Transpose16_8x8(s, output);
+      Transpose8x8_U16(s, output);
       StoreDst<16, 8>(dst, step, 0, output);
     } else {
       StoreDst<16, 8>(dst, step, 0, s);
@@ -564,9 +396,9 @@
     if (transpose) {
       __m128i input[4];
       LoadSrc<16, 4>(src, step, 0, input);
-      Transpose16_8x4To4x8(input, x);
+      Transpose8x4To4x8_U16(input, x);
       LoadSrc<16, 4>(src, step, 8, input);
-      Transpose16_8x4To4x8(input, &x[8]);
+      Transpose8x4To4x8_U16(input, &x[8]);
     } else {
       LoadSrc<8, 16>(src, step, 0, x);
     }
@@ -575,7 +407,7 @@
       for (int idx = 0; idx < 16; idx += 8) {
         __m128i input[8];
         LoadSrc<16, 8>(src, step, idx, input);
-        Transpose16_8x8(input, &x[idx]);
+        Transpose8x8_U16(input, &x[idx]);
       }
     } else {
       LoadSrc<16, 16>(src, step, 0, x);
@@ -608,9 +440,9 @@
   if (stage_is_rectangular) {
     if (transpose) {
       __m128i output[4];
-      Transpose16_4x8To8x4(s, output);
+      Transpose4x8To8x4_U16(s, output);
       StoreDst<16, 4>(dst, step, 0, output);
-      Transpose16_4x8To8x4(&s[8], output);
+      Transpose4x8To8x4_U16(&s[8], output);
       StoreDst<16, 4>(dst, step, 8, output);
     } else {
       StoreDst<8, 16>(dst, step, 0, s);
@@ -619,7 +451,7 @@
     if (transpose) {
       for (int idx = 0; idx < 16; idx += 8) {
         __m128i output[8];
-        Transpose16_8x8(&s[idx], output);
+        Transpose8x8_U16(&s[idx], output);
         StoreDst<16, 8>(dst, step, idx, output);
       }
     } else {
@@ -730,7 +562,7 @@
     for (int idx = 0; idx < 32; idx += 8) {
       __m128i input[8];
       LoadSrc<16, 8>(src, step, idx, input);
-      Transpose16_8x8(input, &x[idx]);
+      Transpose8x8_U16(input, &x[idx]);
     }
   } else {
     LoadSrc<16, 32>(src, step, 0, x);
@@ -782,7 +614,7 @@
   if (transpose) {
     for (int idx = 0; idx < 32; idx += 8) {
       __m128i output[8];
-      Transpose16_8x8(&s[idx], output);
+      Transpose8x8_U16(&s[idx], output);
       StoreDst<16, 8>(dst, step, idx, output);
     }
   } else {
@@ -804,7 +636,7 @@
     for (int idx = 0; idx < 32; idx += 8) {
       __m128i input[8];
       LoadSrc<16, 8>(src, step, idx, input);
-      Transpose16_8x8(input, &x[idx]);
+      Transpose8x8_U16(input, &x[idx]);
     }
   } else {
     // The last 32 values of every column are always zero if the |tx_height| is
@@ -1002,7 +834,7 @@
   if (transpose) {
     for (int idx = 0; idx < 64; idx += 8) {
       __m128i output[8];
-      Transpose16_8x8(&s[idx], output);
+      Transpose8x8_U16(&s[idx], output);
       StoreDst<16, 8>(dst, step, idx, output);
     }
   } else {
@@ -1024,14 +856,14 @@
     if (transpose) {
       __m128i input[8];
       LoadSrc<8, 8>(src, step, 0, input);
-      Transpose16_4x8To8x4(input, x);
+      Transpose4x8To8x4_U16(input, x);
     } else {
       LoadSrc<16, 4>(src, step, 0, x);
     }
   } else {
     LoadSrc<8, 4>(src, step, 0, x);
     if (transpose) {
-      Transpose16_4x4(x, x);
+      Transpose4x4_U16(x, x);
     }
   }
 
@@ -1090,14 +922,14 @@
   if (stage_is_rectangular) {
     if (transpose) {
       __m128i output[8];
-      Transpose16_8x4To4x8(x, output);
+      Transpose8x4To4x8_U16(x, output);
       StoreDst<8, 8>(dst, step, 0, output);
     } else {
       StoreDst<16, 4>(dst, step, 0, x);
     }
   } else {
     if (transpose) {
-      Transpose16_4x4(x, x);
+      Transpose4x4_U16(x, x);
     }
     StoreDst<8, 4>(dst, step, 0, x);
   }
@@ -1114,7 +946,7 @@
     if (transpose) {
       __m128i input[4];
       LoadSrc<16, 4>(src, step, 0, input);
-      Transpose16_8x4To4x8(input, x);
+      Transpose8x4To4x8_U16(input, x);
     } else {
       LoadSrc<8, 8>(src, step, 0, x);
     }
@@ -1122,7 +954,7 @@
     if (transpose) {
       __m128i input[8];
       LoadSrc<16, 8>(src, step, 0, input);
-      Transpose16_8x8(input, x);
+      Transpose8x8_U16(input, x);
     } else {
       LoadSrc<16, 8>(src, step, 0, x);
     }
@@ -1178,7 +1010,7 @@
   if (stage_is_rectangular) {
     if (transpose) {
       __m128i output[4];
-      Transpose16_4x8To8x4(x, output);
+      Transpose4x8To8x4_U16(x, output);
       StoreDst<16, 4>(dst, step, 0, output);
     } else {
       StoreDst<8, 8>(dst, step, 0, x);
@@ -1186,7 +1018,7 @@
   } else {
     if (transpose) {
       __m128i output[8];
-      Transpose16_8x8(x, output);
+      Transpose8x8_U16(x, output);
       StoreDst<16, 8>(dst, step, 0, output);
     } else {
       StoreDst<16, 8>(dst, step, 0, x);
@@ -1205,9 +1037,9 @@
     if (transpose) {
       __m128i input[4];
       LoadSrc<16, 4>(src, step, 0, input);
-      Transpose16_8x4To4x8(input, x);
+      Transpose8x4To4x8_U16(input, x);
       LoadSrc<16, 4>(src, step, 8, input);
-      Transpose16_8x4To4x8(input, &x[8]);
+      Transpose8x4To4x8_U16(input, &x[8]);
     } else {
       LoadSrc<8, 16>(src, step, 0, x);
     }
@@ -1216,7 +1048,7 @@
       for (int idx = 0; idx < 16; idx += 8) {
         __m128i input[8];
         LoadSrc<16, 8>(src, step, idx, input);
-        Transpose16_8x8(input, &x[idx]);
+        Transpose8x8_U16(input, &x[idx]);
       }
     } else {
       LoadSrc<16, 16>(src, step, 0, x);
@@ -1321,9 +1153,9 @@
   if (stage_is_rectangular) {
     if (transpose) {
       __m128i output[4];
-      Transpose16_4x8To8x4(x, output);
+      Transpose4x8To8x4_U16(x, output);
       StoreDst<16, 4>(dst, step, 0, output);
-      Transpose16_4x8To8x4(&x[8], output);
+      Transpose4x8To8x4_U16(&x[8], output);
       StoreDst<16, 4>(dst, step, 8, output);
     } else {
       StoreDst<8, 16>(dst, step, 0, x);
@@ -1332,7 +1164,7 @@
     if (transpose) {
       for (int idx = 0; idx < 16; idx += 8) {
         __m128i output[8];
-        Transpose16_8x8(&x[idx], output);
+        Transpose8x8_U16(&x[idx], output);
         StoreDst<16, 8>(dst, step, idx, output);
       }
     } else {
@@ -1343,11 +1175,6 @@
 
 //------------------------------------------------------------------------------
 // Identity Transforms.
-constexpr int16_t kIdentity4Multiplier /* round(2^12 * sqrt(2)) */ = 0x16A1;
-constexpr int16_t kIdentity4MultiplierFraction /* round(2^12 * (sqrt(2) - 1))*/
-    = 0x6a1;
-constexpr int16_t kIdentity16Multiplier /* 2 * round(2^12 * sqrt(2)) */ = 11586;
-constexpr int16_t kTransformRowMultiplier /* round(2^12 / sqrt(2)) */ = 2896;
 
 template <bool is_row_shift>
 LIBGAV1_ALWAYS_INLINE void Identity4_SSE4_1(void* dest, const void* source,
@@ -1408,7 +1235,8 @@
   } else {
     for (int i = 0; i < tx_height; ++i) {
       const int row = i * tx_width;
-      for (int j = 0; j < tx_width; j += 8) {
+      int j = 0;
+      do {
         const __m128i v_src = LoadUnaligned16(&source[row + j]);
         const __m128i v_src_mult =
             _mm_mulhrs_epi16(v_src, v_multiplier_fraction);
@@ -1419,7 +1247,8 @@
         const __m128i c = _mm_cvtepu8_epi16(frame_data);
         const __m128i d = _mm_adds_epi16(c, b);
         StoreLo8(dst + j, _mm_packus_epi16(d, d));
-      }
+        j += 8;
+      } while (j < tx_width);
       dst += stride;
     }
   }
@@ -1456,7 +1285,8 @@
   } else {
     for (int i = 0; i < tx_height; ++i) {
       const int row = i * tx_width;
-      for (int j = 0; j < tx_width; j += 8) {
+      int j = 0;
+      do {
         const __m128i v_src = LoadUnaligned16(&source[row + j]);
         const __m128i v_src_round =
             _mm_mulhrs_epi16(v_src, v_kTransformRowMultiplier);
@@ -1470,7 +1300,8 @@
         const __m128i b = _mm_srai_epi16(a, 4);
         const __m128i c = _mm_adds_epi16(frame_data16, b);
         StoreLo8(dst + j, _mm_packus_epi16(c, c));
-      }
+        j += 8;
+      } while (j < tx_width);
       dst += stride;
     }
   }
@@ -1528,7 +1359,8 @@
   } else {
     for (int i = 0; i < tx_height; ++i) {
       const int row = i * tx_width;
-      for (int j = 0; j < tx_width; j += 8) {
+      int j = 0;
+      do {
         const __m128i v_src = LoadUnaligned16(&source[row + j]);
         const __m128i v_dst_i = _mm_adds_epi16(v_src, v_src);
         const __m128i frame_data = LoadLo8(dst + j);
@@ -1537,7 +1369,8 @@
         const __m128i c = _mm_cvtepu8_epi16(frame_data);
         const __m128i d = _mm_adds_epi16(c, b);
         StoreLo8(dst + j, _mm_packus_epi16(d, d));
-      }
+        j += 8;
+      } while (j < tx_width);
       dst += stride;
     }
   }
@@ -1599,7 +1432,8 @@
   } else {
     for (int i = 0; i < tx_height; ++i) {
       const int row = i * tx_width;
-      for (int j = 0; j < tx_width; j += 8) {
+      int j = 0;
+      do {
         const __m128i v_src = LoadUnaligned16(&source[row + j]);
         const __m128i v_src_mult = _mm_mulhrs_epi16(v_src, v_multiplier);
         const __m128i frame_data = LoadLo8(dst + j);
@@ -1610,7 +1444,8 @@
         const __m128i c = _mm_cvtepu8_epi16(frame_data);
         const __m128i d = _mm_adds_epi16(c, b);
         StoreLo8(dst + j, _mm_packus_epi16(d, d));
-      }
+        j += 8;
+      } while (j < tx_width);
       dst += stride;
     }
   }
@@ -1645,7 +1480,8 @@
 
   for (int i = 0; i < tx_height; ++i) {
     const int row = i * tx_width;
-    for (int j = 0; j < tx_width; j += 8) {
+    int j = 0;
+    do {
       const __m128i v_dst_i = LoadUnaligned16(&source[row + j]);
       const __m128i frame_data = LoadLo8(dst + j);
       const __m128i a = _mm_adds_epi16(v_dst_i, v_two);
@@ -1653,7 +1489,88 @@
       const __m128i c = _mm_cvtepu8_epi16(frame_data);
       const __m128i d = _mm_adds_epi16(c, b);
       StoreLo8(dst + j, _mm_packus_epi16(d, d));
-    }
+      j += 8;
+    } while (j < tx_width);
+    dst += stride;
+  }
+}
+
+//------------------------------------------------------------------------------
+// Walsh Hadamard Transform.
+
+// Process 4 wht4 rows and columns.
+LIBGAV1_ALWAYS_INLINE void Wht4_SSE4_1(Array2DView<uint8_t> frame,
+                                       const int start_x, const int start_y,
+                                       const void* source,
+                                       const int non_zero_coeff_count) {
+  const auto* const src = static_cast<const int16_t*>(source);
+  __m128i s[4], x[4];
+
+  if (non_zero_coeff_count == 1) {
+    // Special case: only src[0] is nonzero.
+    //   src[0]  0   0   0
+    //       0   0   0   0
+    //       0   0   0   0
+    //       0   0   0   0
+    //
+    // After the row and column transforms are applied, we have:
+    //       f   h   h   h
+    //       g   i   i   i
+    //       g   i   i   i
+    //       g   i   i   i
+    // where f, g, h, i are computed as follows.
+    int16_t f = (src[0] >> 2) - (src[0] >> 3);
+    const int16_t g = f >> 1;
+    f = f - (f >> 1);
+    const int16_t h = (src[0] >> 3) - (src[0] >> 4);
+    const int16_t i = (src[0] >> 4);
+    s[0] = _mm_set1_epi16(h);
+    s[0] = _mm_insert_epi16(s[0], f, 0);
+    s[1] = _mm_set1_epi16(i);
+    s[1] = _mm_insert_epi16(s[1], g, 0);
+    s[2] = s[3] = s[1];
+  } else {
+    x[0] = LoadLo8(&src[0 * 4]);
+    x[2] = LoadLo8(&src[1 * 4]);
+    x[3] = LoadLo8(&src[2 * 4]);
+    x[1] = LoadLo8(&src[3 * 4]);
+
+    // Row transforms.
+    Transpose4x4_U16(x, x);
+    s[0] = _mm_srai_epi16(x[0], 2);
+    s[2] = _mm_srai_epi16(x[1], 2);
+    s[3] = _mm_srai_epi16(x[2], 2);
+    s[1] = _mm_srai_epi16(x[3], 2);
+    s[0] = _mm_add_epi16(s[0], s[2]);
+    s[3] = _mm_sub_epi16(s[3], s[1]);
+    __m128i e = _mm_sub_epi16(s[0], s[3]);
+    e = _mm_srai_epi16(e, 1);
+    s[1] = _mm_sub_epi16(e, s[1]);
+    s[2] = _mm_sub_epi16(e, s[2]);
+    s[0] = _mm_sub_epi16(s[0], s[1]);
+    s[3] = _mm_add_epi16(s[3], s[2]);
+    Transpose4x4_U16(s, s);
+
+    // Column transforms.
+    s[0] = _mm_add_epi16(s[0], s[2]);
+    s[3] = _mm_sub_epi16(s[3], s[1]);
+    e = _mm_sub_epi16(s[0], s[3]);
+    e = _mm_srai_epi16(e, 1);
+    s[1] = _mm_sub_epi16(e, s[1]);
+    s[2] = _mm_sub_epi16(e, s[2]);
+    s[0] = _mm_sub_epi16(s[0], s[1]);
+    s[3] = _mm_add_epi16(s[3], s[2]);
+  }
+
+  // Store to frame.
+  const int stride = frame.columns();
+  uint8_t* dst = frame[start_y] + start_x;
+  for (int row = 0; row < 4; ++row) {
+    const __m128i frame_data = Load4(dst);
+    const __m128i a = _mm_cvtepu8_epi16(frame_data);
+    // Saturate to prevent overflowing int16_t
+    const __m128i b = _mm_adds_epi16(a, s[row]);
+    Store4(dst, _mm_packus_epi16(b, b));
     dst += stride;
   }
 }
@@ -1661,21 +1578,13 @@
 //------------------------------------------------------------------------------
 // row/column transform loops
 
-constexpr uint8_t kTransformRowShift[kNumTransformSizes] = {
-    0, 0, 1, 0, 1, 1, 2, 1, 1, 2, 1, 2, 2, 1, 2, 1, 2, 1, 2};
-
-constexpr bool kShouldRound[kNumTransformSizes] = {
-    false, true,  false, true, false, true, false, false, true, false,
-    true,  false, false, true, false, true, false, true,  false};
-
 template <bool enable_flip_rows = false>
 LIBGAV1_ALWAYS_INLINE void StoreToFrameWithRound(
     Array2DView<uint8_t> frame, const int start_x, const int start_y,
     const int tx_width, const int tx_height, const int16_t* source,
     TransformType tx_type) {
-  const bool flip_rows = enable_flip_rows
-                             ? ((1U << tx_type) & kTransformFlipRowsMask) != 0
-                             : false;
+  const bool flip_rows =
+      enable_flip_rows ? kTransformFlipRowsMask.Contains(tx_type) : false;
   const __m128i v_eight = _mm_set1_epi16(8);
   const int stride = frame.columns();
   uint8_t* dst = frame[start_y] + start_x;
@@ -1706,11 +1615,11 @@
       dst += stride;
     }
   } else {
-    const __m128i v_eight = _mm_set1_epi16(8);
     for (int i = 0; i < tx_height; ++i) {
       const int y = start_y + i;
       const int row = flip_rows ? (tx_height - i - 1) * tx_width : i * tx_width;
-      for (int j = 0; j < tx_width; j += 16) {
+      int j = 0;
+      do {
         const int x = start_x + j;
         const __m128i residual = LoadUnaligned16(&source[row + j]);
         const __m128i residual_hi = LoadUnaligned16(&source[row + j + 8]);
@@ -1724,7 +1633,8 @@
         const __m128i e = _mm_adds_epi16(d, c);
         const __m128i e_hi = _mm_adds_epi16(d_hi, c_hi);
         StoreUnaligned16(frame[y] + x, _mm_packus_epi16(e, e_hi));
-      }
+        j += 16;
+      } while (j < tx_width);
     }
   }
 }
@@ -1734,7 +1644,8 @@
   const __m128i word_reverse_8 =
       _mm_set_epi32(0x01000302, 0x05040706, 0x09080b0a, 0x0d0c0f0e);
   if (tx_width >= 16) {
-    for (int i = 0; i < tx_width * tx_height; i += 16) {
+    int i = 0;
+    do {
       // read 16 shorts
       const __m128i v3210 = LoadUnaligned16(&source[i]);
       const __m128i v7654 = LoadUnaligned16(&source[i + 8]);
@@ -1742,7 +1653,8 @@
       const __m128i v4567 = _mm_shuffle_epi8(v7654, word_reverse_8);
       StoreUnaligned16(&source[i], v4567);
       StoreUnaligned16(&source[i + 8], v0123);
-    }
+      i += 16;
+    } while (i < tx_width * tx_height);
   } else if (tx_width == 8) {
     for (int i = 0; i < 8 * tx_height; i += 8) {
       const __m128i a = LoadUnaligned16(&source[i]);
@@ -1762,27 +1674,32 @@
 }
 
 template <int tx_width>
-LIBGAV1_ALWAYS_INLINE void ShouldRound(int16_t* source, int num_rows) {
+LIBGAV1_ALWAYS_INLINE void ApplyRounding(int16_t* source, int num_rows) {
   const __m128i v_kTransformRowMultiplier =
       _mm_set1_epi16(kTransformRowMultiplier << 3);
   if (tx_width == 4) {
     // Process two rows per iteration.
-    for (int i = 0; i < tx_width * num_rows; i += 8) {
+    int i = 0;
+    do {
       const __m128i a = LoadUnaligned16(&source[i]);
       const __m128i b = _mm_mulhrs_epi16(a, v_kTransformRowMultiplier);
       StoreUnaligned16(&source[i], b);
-    }
+      i += 8;
+    } while (i < tx_width * num_rows);
   } else {
-    for (int i = 0; i < num_rows; ++i) {
+    int i = 0;
+    do {
       // The last 32 values of every row are always zero if the |tx_width| is
       // 64.
       const int non_zero_width = (tx_width < 64) ? tx_width : 32;
-      for (int j = 0; j < non_zero_width; j += 8) {
+      int j = 0;
+      do {
         const __m128i a = LoadUnaligned16(&source[i * tx_width + j]);
         const __m128i b = _mm_mulhrs_epi16(a, v_kTransformRowMultiplier);
         StoreUnaligned16(&source[i * tx_width + j], b);
-      }
-    }
+        j += 8;
+      } while (j < non_zero_width);
+    } while (++i < num_rows);
   }
 }
 
@@ -1793,7 +1710,8 @@
   const __m128i v_row_shift = _mm_cvtepu32_epi64(v_row_shift_add);
   if (tx_width == 4) {
     // Process two rows per iteration.
-    for (int i = 0; i < tx_width * num_rows; i += 8) {
+    int i = 0;
+    do {
       // Expand to 32 bits to prevent int16_t overflows during the shift add.
       const __m128i residual = LoadUnaligned16(&source[i]);
       const __m128i a = _mm_cvtepi16_epi32(residual);
@@ -1803,9 +1721,11 @@
       const __m128i c = _mm_sra_epi32(b, v_row_shift);
       const __m128i c1 = _mm_sra_epi32(b1, v_row_shift);
       StoreUnaligned16(&source[i], _mm_packs_epi32(c, c1));
-    }
+      i += 8;
+    } while (i < tx_width * num_rows);
   } else {
-    for (int i = 0; i < num_rows; ++i) {
+    int i = 0;
+    do {
       for (int j = 0; j < tx_width; j += 8) {
         // Expand to 32 bits to prevent int16_t overflows during the shift add.
         const __m128i residual = LoadUnaligned16(&source[i * tx_width + j]);
@@ -1817,14 +1737,14 @@
         const __m128i c1 = _mm_sra_epi32(b1, v_row_shift);
         StoreUnaligned16(&source[i * tx_width + j], _mm_packs_epi32(c, c1));
       }
-    }
+    } while (++i < num_rows);
   }
 }
 
 void Dct4TransformLoop_SSE4_1(TransformType tx_type, TransformSize tx_size,
-                              int8_t /*bitdepth*/, void* src_buffer,
-                              int start_x, int start_y, void* dst_frame,
-                              bool is_row, int non_zero_coeff_count) {
+                              void* src_buffer, int start_x, int start_y,
+                              void* dst_frame, bool is_row,
+                              int non_zero_coeff_count) {
   auto& frame = *reinterpret_cast<Array2DView<uint8_t>*>(dst_frame);
   auto* src = static_cast<int16_t*>(src_buffer);
   const int tx_width = kTransformWidth[tx_size];
@@ -1834,7 +1754,7 @@
     const int num_rows = (non_zero_coeff_count == 1) ? 1 : tx_height;
     const bool should_round = (tx_height == 8);
     if (should_round) {
-      ShouldRound<4>(src, num_rows);
+      ApplyRounding<4>(src, num_rows);
     }
 
     if (num_rows <= 4) {
@@ -1843,10 +1763,12 @@
                                               /*transpose=*/true);
     } else {
       // Process 8 1d dct4 rows in parallel per iteration.
-      for (int i = 0; i < num_rows; i += 8) {
+      int i = 0;
+      do {
         Dct4_SSE4_1<ButterflyRotation_8, true>(&src[i * 4], &src[i * 4],
                                                /*step=*/4, /*transpose=*/true);
-      }
+        i += 8;
+      } while (i < num_rows);
     }
     if (tx_height == 16) {
       RowShift<4>(src, num_rows, 1);
@@ -1855,8 +1777,7 @@
   }
 
   assert(!is_row);
-  const bool flip_columns = ((1U << tx_type) & kTransformFlipColumnsMask) != 0;
-  if (flip_columns) {
+  if (kTransformFlipColumnsMask.Contains(tx_type)) {
     FlipColumns<4>(src, tx_width);
   }
 
@@ -1866,18 +1787,20 @@
                                             /*transpose=*/false);
   } else {
     // Process 8 1d dct4 columns in parallel per iteration.
-    for (int i = 0; i < tx_width; i += 8) {
+    int i = 0;
+    do {
       Dct4_SSE4_1<ButterflyRotation_8, true>(&src[i], &src[i], tx_width,
                                              /*transpose=*/false);
-    }
+      i += 8;
+    } while (i < tx_width);
   }
   StoreToFrameWithRound(frame, start_x, start_y, tx_width, 4, src, tx_type);
 }
 
 void Dct8TransformLoop_SSE4_1(TransformType tx_type, TransformSize tx_size,
-                              int8_t /*bitdepth*/, void* src_buffer,
-                              int start_x, int start_y, void* dst_frame,
-                              bool is_row, int non_zero_coeff_count) {
+                              void* src_buffer, int start_x, int start_y,
+                              void* dst_frame, bool is_row,
+                              int non_zero_coeff_count) {
   auto& frame = *reinterpret_cast<Array2DView<uint8_t>*>(dst_frame);
   auto* src = static_cast<int16_t*>(src_buffer);
   const int tx_width = kTransformWidth[tx_size];
@@ -1886,7 +1809,7 @@
   if (is_row) {
     const int num_rows = (non_zero_coeff_count == 1) ? 1 : tx_height;
     if (kShouldRound[tx_size]) {
-      ShouldRound<8>(src, num_rows);
+      ApplyRounding<8>(src, num_rows);
     }
 
     if (num_rows <= 4) {
@@ -1895,10 +1818,12 @@
                                              /*transpose=*/true);
     } else {
       // Process 8 1d dct8 rows in parallel per iteration.
-      for (int i = 0; i < num_rows; i += 8) {
+      int i = 0;
+      do {
         Dct8_SSE4_1<ButterflyRotation_8, false>(&src[i * 8], &src[i * 8],
                                                 /*step=*/8, /*transpose=*/true);
-      }
+        i += 8;
+      } while (i < num_rows);
     }
     const uint8_t row_shift = kTransformRowShift[tx_size];
     if (row_shift > 0) {
@@ -1908,8 +1833,7 @@
   }
 
   assert(!is_row);
-  const bool flip_columns = ((1U << tx_type) & kTransformFlipColumnsMask) != 0;
-  if (flip_columns) {
+  if (kTransformFlipColumnsMask.Contains(tx_type)) {
     FlipColumns<8>(src, tx_width);
   }
 
@@ -1919,18 +1843,20 @@
                                            /*transpose=*/false);
   } else {
     // Process 8 1d dct8 columns in parallel per iteration.
-    for (int i = 0; i < tx_width; i += 8) {
+    int i = 0;
+    do {
       Dct8_SSE4_1<ButterflyRotation_8, false>(&src[i], &src[i], tx_width,
                                               /*transpose=*/false);
-    }
+      i += 8;
+    } while (i < tx_width);
   }
   StoreToFrameWithRound(frame, start_x, start_y, tx_width, 8, src, tx_type);
 }
 
 void Dct16TransformLoop_SSE4_1(TransformType tx_type, TransformSize tx_size,
-                               int8_t /*bitdepth*/, void* src_buffer,
-                               int start_x, int start_y, void* dst_frame,
-                               bool is_row, int non_zero_coeff_count) {
+                               void* src_buffer, int start_x, int start_y,
+                               void* dst_frame, bool is_row,
+                               int non_zero_coeff_count) {
   auto& frame = *reinterpret_cast<Array2DView<uint8_t>*>(dst_frame);
   auto* src = static_cast<int16_t*>(src_buffer);
   const int tx_width = kTransformWidth[tx_size];
@@ -1940,7 +1866,7 @@
     const int num_rows =
         (non_zero_coeff_count == 1) ? 1 : std::min(tx_height, 32);
     if (kShouldRound[tx_size]) {
-      ShouldRound<16>(src, num_rows);
+      ApplyRounding<16>(src, num_rows);
     }
 
     if (num_rows <= 4) {
@@ -1948,11 +1874,13 @@
       Dct16_SSE4_1<ButterflyRotation_4, true>(&src[0], &src[0], 16,
                                               /*transpose=*/true);
     } else {
-      for (int i = 0; i < num_rows; i += 8) {
+      int i = 0;
+      do {
         // Process 8 1d dct16 rows in parallel per iteration.
         Dct16_SSE4_1<ButterflyRotation_8, false>(&src[i * 16], &src[i * 16], 16,
                                                  /*transpose=*/true);
-      }
+        i += 8;
+      } while (i < num_rows);
     }
     const uint8_t row_shift = kTransformRowShift[tx_size];
     // row_shift is always non zero here.
@@ -1962,8 +1890,7 @@
   }
 
   assert(!is_row);
-  const bool flip_columns = ((1U << tx_type) & kTransformFlipColumnsMask) != 0;
-  if (flip_columns) {
+  if (kTransformFlipColumnsMask.Contains(tx_type)) {
     FlipColumns<16>(src, tx_width);
   }
 
@@ -1972,19 +1899,21 @@
     Dct16_SSE4_1<ButterflyRotation_4, true>(&src[0], &src[0], 4,
                                             /*transpose=*/false);
   } else {
-    for (int i = 0; i < tx_width; i += 8) {
+    int i = 0;
+    do {
       // Process 8 1d dct16 columns in parallel per iteration.
       Dct16_SSE4_1<ButterflyRotation_8, false>(&src[i], &src[i], tx_width,
                                                /*transpose=*/false);
-    }
+      i += 8;
+    } while (i < tx_width);
   }
   StoreToFrameWithRound(frame, start_x, start_y, tx_width, 16, src, tx_type);
 }
 
 void Dct32TransformLoop_SSE4_1(TransformType tx_type, TransformSize tx_size,
-                               int8_t /*bitdepth*/, void* src_buffer,
-                               int start_x, int start_y, void* dst_frame,
-                               bool is_row, int non_zero_coeff_count) {
+                               void* src_buffer, int start_x, int start_y,
+                               void* dst_frame, bool is_row,
+                               int non_zero_coeff_count) {
   auto& frame = *reinterpret_cast<Array2DView<uint8_t>*>(dst_frame);
   auto* src = static_cast<int16_t*>(src_buffer);
   const int tx_width = kTransformWidth[tx_size];
@@ -1994,12 +1923,14 @@
     const int num_rows =
         (non_zero_coeff_count == 1) ? 1 : std::min(tx_height, 32);
     if (kShouldRound[tx_size]) {
-      ShouldRound<32>(src, num_rows);
+      ApplyRounding<32>(src, num_rows);
     }
     // Process 8 1d dct32 rows in parallel per iteration.
-    for (int i = 0; i < num_rows; i += 8) {
+    int i = 0;
+    do {
       Dct32_SSE4_1(&src[i * 32], &src[i * 32], 32, /*transpose=*/true);
-    }
+      i += 8;
+    } while (i < num_rows);
     const uint8_t row_shift = kTransformRowShift[tx_size];
     // row_shift is always non zero here.
     RowShift<32>(src, num_rows, row_shift);
@@ -2009,16 +1940,18 @@
 
   assert(!is_row);
   // Process 8 1d dct32 columns in parallel per iteration.
-  for (int i = 0; i < tx_width; i += 8) {
+  int i = 0;
+  do {
     Dct32_SSE4_1(&src[i], &src[i], tx_width, /*transpose=*/false);
-  }
+    i += 8;
+  } while (i < tx_width);
   StoreToFrameWithRound(frame, start_x, start_y, tx_width, 32, src, tx_type);
 }
 
 void Dct64TransformLoop_SSE4_1(TransformType tx_type, TransformSize tx_size,
-                               int8_t /*bitdepth*/, void* src_buffer,
-                               int start_x, int start_y, void* dst_frame,
-                               bool is_row, int non_zero_coeff_count) {
+                               void* src_buffer, int start_x, int start_y,
+                               void* dst_frame, bool is_row,
+                               int non_zero_coeff_count) {
   auto& frame = *reinterpret_cast<Array2DView<uint8_t>*>(dst_frame);
   auto* src = static_cast<int16_t*>(src_buffer);
   const int tx_width = kTransformWidth[tx_size];
@@ -2028,12 +1961,14 @@
     const int num_rows =
         (non_zero_coeff_count == 1) ? 1 : std::min(tx_height, 32);
     if (kShouldRound[tx_size]) {
-      ShouldRound<64>(src, num_rows);
+      ApplyRounding<64>(src, num_rows);
     }
     // Process 8 1d dct64 rows in parallel per iteration.
-    for (int i = 0; i < num_rows; i += 8) {
+    int i = 0;
+    do {
       Dct64_SSE4_1(&src[i * 64], &src[i * 64], 64, /*transpose=*/true);
-    }
+      i += 8;
+    } while (i < num_rows);
     const uint8_t row_shift = kTransformRowShift[tx_size];
     // row_shift is always non zero here.
     RowShift<64>(src, num_rows, row_shift);
@@ -2043,16 +1978,18 @@
 
   assert(!is_row);
   // Process 8 1d dct64 columns in parallel per iteration.
-  for (int i = 0; i < tx_width; i += 8) {
+  int i = 0;
+  do {
     Dct64_SSE4_1(&src[i], &src[i], tx_width, /*transpose=*/false);
-  }
+    i += 8;
+  } while (i < tx_width);
   StoreToFrameWithRound(frame, start_x, start_y, tx_width, 64, src, tx_type);
 }
 
 void Adst4TransformLoop_SSE4_1(TransformType tx_type, TransformSize tx_size,
-                               int8_t /*bitdepth*/, void* src_buffer,
-                               int start_x, int start_y, void* dst_frame,
-                               bool is_row, int non_zero_coeff_count) {
+                               void* src_buffer, int start_x, int start_y,
+                               void* dst_frame, bool is_row,
+                               int non_zero_coeff_count) {
   auto& frame = *reinterpret_cast<Array2DView<uint8_t>*>(dst_frame);
   auto* src = static_cast<int16_t*>(src_buffer);
   const int tx_width = kTransformWidth[tx_size];
@@ -2062,14 +1999,16 @@
     const int num_rows = (non_zero_coeff_count == 1) ? 1 : tx_height;
     const bool should_round = (tx_height == 8);
     if (should_round) {
-      ShouldRound<4>(src, num_rows);
+      ApplyRounding<4>(src, num_rows);
     }
 
     // Process 4 1d adst4 rows in parallel per iteration.
-    for (int i = 0; i < num_rows; i += 4) {
+    int i = 0;
+    do {
       Adst4_SSE4_1<false>(&src[i * 4], &src[i * 4], /*step=*/4,
                           /*transpose=*/true);
-    }
+      i += 4;
+    } while (i < num_rows);
 
     if (tx_height == 16) {
       RowShift<4>(src, num_rows, 1);
@@ -2078,24 +2017,25 @@
   }
 
   assert(!is_row);
-  const bool flip_columns = ((1U << tx_type) & kTransformFlipColumnsMask) != 0;
-  if (flip_columns) {
+  if (kTransformFlipColumnsMask.Contains(tx_type)) {
     FlipColumns<4>(src, tx_width);
   }
 
   // Process 4 1d adst4 columns in parallel per iteration.
-  for (int i = 0; i < tx_width; i += 4) {
+  int i = 0;
+  do {
     Adst4_SSE4_1<false>(&src[i], &src[i], tx_width, /*transpose=*/false);
-  }
+    i += 4;
+  } while (i < tx_width);
 
   StoreToFrameWithRound</*enable_flip_rows=*/true>(frame, start_x, start_y,
                                                    tx_width, 4, src, tx_type);
 }
 
 void Adst8TransformLoop_SSE4_1(TransformType tx_type, TransformSize tx_size,
-                               int8_t /*bitdepth*/, void* src_buffer,
-                               int start_x, int start_y, void* dst_frame,
-                               bool is_row, int non_zero_coeff_count) {
+                               void* src_buffer, int start_x, int start_y,
+                               void* dst_frame, bool is_row,
+                               int non_zero_coeff_count) {
   auto& frame = *reinterpret_cast<Array2DView<uint8_t>*>(dst_frame);
   auto* src = static_cast<int16_t*>(src_buffer);
   const int tx_width = kTransformWidth[tx_size];
@@ -2104,7 +2044,7 @@
   if (is_row) {
     const int num_rows = (non_zero_coeff_count == 1) ? 1 : tx_height;
     if (kShouldRound[tx_size]) {
-      ShouldRound<8>(src, num_rows);
+      ApplyRounding<8>(src, num_rows);
     }
 
     if (num_rows <= 4) {
@@ -2113,11 +2053,13 @@
                                               /*transpose=*/true);
     } else {
       // Process 8 1d adst8 rows in parallel per iteration.
-      for (int i = 0; i < num_rows; i += 8) {
+      int i = 0;
+      do {
         Adst8_SSE4_1<ButterflyRotation_8, false>(&src[i * 8], &src[i * 8],
                                                  /*step=*/8,
                                                  /*transpose=*/true);
-      }
+        i += 8;
+      } while (i < num_rows);
     }
     const uint8_t row_shift = kTransformRowShift[tx_size];
     if (row_shift > 0) {
@@ -2127,8 +2069,7 @@
   }
 
   assert(!is_row);
-  const bool flip_columns = ((1U << tx_type) & kTransformFlipColumnsMask) != 0;
-  if (flip_columns) {
+  if (kTransformFlipColumnsMask.Contains(tx_type)) {
     FlipColumns<8>(src, tx_width);
   }
 
@@ -2138,19 +2079,21 @@
                                             /*transpose=*/false);
   } else {
     // Process 8 1d adst8 columns in parallel per iteration.
-    for (int i = 0; i < tx_width; i += 8) {
+    int i = 0;
+    do {
       Adst8_SSE4_1<ButterflyRotation_8, false>(&src[i], &src[i], tx_width,
                                                /*transpose=*/false);
-    }
+      i += 8;
+    } while (i < tx_width);
   }
   StoreToFrameWithRound</*enable_flip_rows=*/true>(frame, start_x, start_y,
                                                    tx_width, 8, src, tx_type);
 }
 
 void Adst16TransformLoop_SSE4_1(TransformType tx_type, TransformSize tx_size,
-                                int8_t /*bitdepth*/, void* src_buffer,
-                                int start_x, int start_y, void* dst_frame,
-                                bool is_row, int non_zero_coeff_count) {
+                                void* src_buffer, int start_x, int start_y,
+                                void* dst_frame, bool is_row,
+                                int non_zero_coeff_count) {
   auto& frame = *reinterpret_cast<Array2DView<uint8_t>*>(dst_frame);
   auto* src = static_cast<int16_t*>(src_buffer);
   const int tx_width = kTransformWidth[tx_size];
@@ -2160,7 +2103,7 @@
     const int num_rows =
         (non_zero_coeff_count == 1) ? 1 : std::min(tx_height, 32);
     if (kShouldRound[tx_size]) {
-      ShouldRound<16>(src, num_rows);
+      ApplyRounding<16>(src, num_rows);
     }
 
     if (num_rows <= 4) {
@@ -2168,11 +2111,13 @@
       Adst16_SSE4_1<ButterflyRotation_4, true>(&src[0], &src[0], 16,
                                                /*transpose=*/true);
     } else {
-      for (int i = 0; i < num_rows; i += 8) {
+      int i = 0;
+      do {
         // Process 8 1d adst16 rows in parallel per iteration.
         Adst16_SSE4_1<ButterflyRotation_8, false>(&src[i * 16], &src[i * 16],
                                                   16, /*transpose=*/true);
-      }
+        i += 8;
+      } while (i < num_rows);
     }
     const uint8_t row_shift = kTransformRowShift[tx_size];
     // row_shift is always non zero here.
@@ -2182,8 +2127,7 @@
   }
 
   assert(!is_row);
-  const bool flip_columns = ((1U << tx_type) & kTransformFlipColumnsMask) != 0;
-  if (flip_columns) {
+  if (kTransformFlipColumnsMask.Contains(tx_type)) {
     FlipColumns<16>(src, tx_width);
   }
 
@@ -2192,20 +2136,22 @@
     Adst16_SSE4_1<ButterflyRotation_4, true>(&src[0], &src[0], 4,
                                              /*transpose=*/false);
   } else {
-    for (int i = 0; i < tx_width; i += 8) {
+    int i = 0;
+    do {
       // Process 8 1d adst16 columns in parallel per iteration.
       Adst16_SSE4_1<ButterflyRotation_8, false>(&src[i], &src[i], tx_width,
                                                 /*transpose=*/false);
-    }
+      i += 8;
+    } while (i < tx_width);
   }
   StoreToFrameWithRound</*enable_flip_rows=*/true>(frame, start_x, start_y,
                                                    tx_width, 16, src, tx_type);
 }
 
 void Identity4TransformLoop_SSE4_1(TransformType tx_type, TransformSize tx_size,
-                                   int8_t /*bitdepth*/, void* src_buffer,
-                                   int start_x, int start_y, void* dst_frame,
-                                   bool is_row, int non_zero_coeff_count) {
+                                   void* src_buffer, int start_x, int start_y,
+                                   void* dst_frame, bool is_row,
+                                   int non_zero_coeff_count) {
   auto& frame = *reinterpret_cast<Array2DView<uint8_t>*>(dst_frame);
   auto* src = static_cast<int16_t*>(src_buffer);
   const int tx_width = kTransformWidth[tx_size];
@@ -2222,16 +2168,20 @@
     const int num_rows = (non_zero_coeff_count == 1) ? 1 : tx_height;
     const bool should_round = (tx_height == 8);
     if (should_round) {
-      ShouldRound<4>(src, num_rows);
+      ApplyRounding<4>(src, num_rows);
     }
     if (tx_height < 16) {
-      for (int i = 0; i < num_rows; i += 4) {
+      int i = 0;
+      do {
         Identity4_SSE4_1<false>(&src[i * 4], &src[i * 4], /*step=*/4);
-      }
+        i += 4;
+      } while (i < num_rows);
     } else {
-      for (int i = 0; i < num_rows; i += 4) {
+      int i = 0;
+      do {
         Identity4_SSE4_1<true>(&src[i * 4], &src[i * 4], /*step=*/4);
-      }
+        i += 4;
+      } while (i < num_rows);
     }
     return;
   }
@@ -2243,8 +2193,7 @@
     return;
   }
 
-  const bool flip_columns = ((1U << tx_type) & kTransformFlipColumnsMask) != 0;
-  if (flip_columns) {
+  if (kTransformFlipColumnsMask.Contains(tx_type)) {
     FlipColumns<4>(src, tx_width);
   }
 
@@ -2253,9 +2202,9 @@
 }
 
 void Identity8TransformLoop_SSE4_1(TransformType tx_type, TransformSize tx_size,
-                                   int8_t /*bitdepth*/, void* src_buffer,
-                                   int start_x, int start_y, void* dst_frame,
-                                   bool is_row, int non_zero_coeff_count) {
+                                   void* src_buffer, int start_x, int start_y,
+                                   void* dst_frame, bool is_row,
+                                   int non_zero_coeff_count) {
   auto& frame = *reinterpret_cast<Array2DView<uint8_t>*>(dst_frame);
   auto* src = static_cast<int16_t*>(src_buffer);
   const int tx_width = kTransformWidth[tx_size];
@@ -2270,7 +2219,7 @@
     }
     const int num_rows = (non_zero_coeff_count == 1) ? 1 : tx_height;
     if (kShouldRound[tx_size]) {
-      ShouldRound<8>(src, num_rows);
+      ApplyRounding<8>(src, num_rows);
     }
 
     // When combining the identity8 multiplier with the row shift, the
@@ -2280,23 +2229,26 @@
       return;
     }
     if (tx_height == 32) {
-      for (int i = 0; i < num_rows; i += 4) {
+      int i = 0;
+      do {
         Identity8Row32_SSE4_1(&src[i * 8], &src[i * 8], /*step=*/8);
-      }
+        i += 4;
+      } while (i < num_rows);
       return;
     }
 
     // Process kTransformSize8x4
     assert(tx_size == kTransformSize8x4);
-    for (int i = 0; i < num_rows; i += 4) {
+    int i = 0;
+    do {
       Identity8Row4_SSE4_1(&src[i * 8], &src[i * 8], /*step=*/8);
-    }
+      i += 4;
+    } while (i < num_rows);
     return;
   }
 
   assert(!is_row);
-  const bool flip_columns = ((1U << tx_type) & kTransformFlipColumnsMask) != 0;
-  if (flip_columns) {
+  if (kTransformFlipColumnsMask.Contains(tx_type)) {
     FlipColumns<8>(src, tx_width);
   }
 
@@ -2305,10 +2257,9 @@
 }
 
 void Identity16TransformLoop_SSE4_1(TransformType tx_type,
-                                    TransformSize tx_size, int8_t /*bitdepth*/,
-                                    void* src_buffer, int start_x, int start_y,
-                                    void* dst_frame, bool is_row,
-                                    int non_zero_coeff_count) {
+                                    TransformSize tx_size, void* src_buffer,
+                                    int start_x, int start_y, void* dst_frame,
+                                    bool is_row, int non_zero_coeff_count) {
   auto& frame = *reinterpret_cast<Array2DView<uint8_t>*>(dst_frame);
   auto* src = static_cast<int16_t*>(src_buffer);
   const int tx_width = kTransformWidth[tx_size];
@@ -2318,18 +2269,19 @@
     const int num_rows =
         (non_zero_coeff_count == 1) ? 1 : std::min(tx_height, 32);
     if (kShouldRound[tx_size]) {
-      ShouldRound<16>(src, num_rows);
+      ApplyRounding<16>(src, num_rows);
     }
-    for (int i = 0; i < num_rows; i += 4) {
+    int i = 0;
+    do {
       Identity16Row_SSE4_1(&src[i * 16], &src[i * 16], /*step=*/16,
                            kTransformRowShift[tx_size]);
-    }
+      i += 4;
+    } while (i < num_rows);
     return;
   }
 
   assert(!is_row);
-  const bool flip_columns = ((1U << tx_type) & kTransformFlipColumnsMask) != 0;
-  if (flip_columns) {
+  if (kTransformFlipColumnsMask.Contains(tx_type)) {
     FlipColumns<16>(src, tx_width);
   }
   Identity16ColumnStoreToFrame_SSE4_1(frame, start_x, start_y, tx_width,
@@ -2337,10 +2289,9 @@
 }
 
 void Identity32TransformLoop_SSE4_1(TransformType /*tx_type*/,
-                                    TransformSize tx_size, int8_t /*bitdepth*/,
-                                    void* src_buffer, int start_x, int start_y,
-                                    void* dst_frame, bool is_row,
-                                    int non_zero_coeff_count) {
+                                    TransformSize tx_size, void* src_buffer,
+                                    int start_x, int start_y, void* dst_frame,
+                                    bool is_row, int non_zero_coeff_count) {
   auto& frame = *reinterpret_cast<Array2DView<uint8_t>*>(dst_frame);
   auto* src = static_cast<int16_t*>(src_buffer);
   const int tx_width = kTransformWidth[tx_size];
@@ -2358,10 +2309,12 @@
 
     // Process kTransformSize32x16
     assert(tx_size == kTransformSize32x16);
-    ShouldRound<32>(src, num_rows);
-    for (int i = 0; i < num_rows; i += 4) {
+    ApplyRounding<32>(src, num_rows);
+    int i = 0;
+    do {
       Identity32Row16_SSE4_1(&src[i * 32], &src[i * 32], /*step=*/32);
-    }
+      i += 4;
+    } while (i < num_rows);
     return;
   }
 
@@ -2370,6 +2323,26 @@
                                /*tx_height=*/32, src);
 }
 
+void Wht4TransformLoop_SSE4_1(TransformType tx_type, TransformSize tx_size,
+                              void* src_buffer, int start_x, int start_y,
+                              void* dst_frame, bool is_row,
+                              int non_zero_coeff_count) {
+  assert(tx_type == kTransformTypeDctDct);
+  assert(tx_size == kTransformSize4x4);
+  static_cast<void>(tx_type);
+  static_cast<void>(tx_size);
+  if (is_row) {
+    // Do both row and column transforms in the column-transform pass.
+    return;
+  }
+
+  assert(!is_row);
+  // Process 4 1d wht4 rows and columns in parallel.
+  const auto* src = static_cast<int16_t*>(src_buffer);
+  auto& frame = *static_cast<Array2DView<uint8_t>*>(dst_frame);
+  Wht4_SSE4_1(frame, start_x, start_y, src, non_zero_coeff_count);
+}
+
 //------------------------------------------------------------------------------
 
 template <typename Residual, typename Pixel>
@@ -2403,6 +2376,10 @@
       Identity16TransformLoop_SSE4_1;
   dsp->inverse_transforms[k1DTransformSize32][k1DTransformIdentity] =
       Identity32TransformLoop_SSE4_1;
+
+  // Maximum transform size for Wht is 4.
+  dsp->inverse_transforms[k1DTransformSize4][k1DTransformWht] =
+      Wht4TransformLoop_SSE4_1;
 }
 
 void Init8bpp() {
@@ -2459,6 +2436,10 @@
   dsp->inverse_transforms[k1DTransformSize32][k1DTransformIdentity] =
       Identity32TransformLoop_SSE4_1;
 #endif
+#if DSP_ENABLED_8BPP_SSE4_1(1DTransformSize4_1DTransformWht)
+  dsp->inverse_transforms[k1DTransformSize4][k1DTransformWht] =
+      Wht4TransformLoop_SSE4_1;
+#endif
 #endif
 }
 
diff --git a/libgav1/src/dsp/x86/inverse_transform_sse4.h b/libgav1/src/dsp/x86/inverse_transform_sse4.h
index dd30533..72fda5a 100644
--- a/libgav1/src/dsp/x86/inverse_transform_sse4.h
+++ b/libgav1/src/dsp/x86/inverse_transform_sse4.h
@@ -3,12 +3,12 @@
 
 #include "src/dsp/cpu.h"
 #include "src/dsp/dsp.h"
-#include "src/dsp/inverse_transform.h"
 
 namespace libgav1 {
 namespace dsp {
 
-// Initializes Dsp::inverse_transforms. This function is not thread-safe.
+// Initializes Dsp::inverse_transforms, see the defines below for specifics.
+// This function is not thread-safe.
 void InverseTransformInit_SSE4_1();
 
 }  // namespace dsp
@@ -65,5 +65,9 @@
 #ifndef LIBGAV1_Dsp8bpp_1DTransformSize32_1DTransformIdentity
 #define LIBGAV1_Dsp8bpp_1DTransformSize32_1DTransformIdentity LIBGAV1_DSP_SSE4_1
 #endif
+
+#ifndef LIBGAV1_Dsp8bpp_1DTransformSize4_1DTransformWht
+#define LIBGAV1_Dsp8bpp_1DTransformSize4_1DTransformWht LIBGAV1_DSP_SSE4_1
+#endif
 #endif  // LIBGAV1_ENABLE_SSE4_1
 #endif  // LIBGAV1_SRC_DSP_X86_INVERSE_TRANSFORM_SSE4_H_
diff --git a/libgav1/src/dsp/x86/loop_filter_sse4.cc b/libgav1/src/dsp/x86/loop_filter_sse4.cc
index d68e76d..4c5ab23 100644
--- a/libgav1/src/dsp/x86/loop_filter_sse4.cc
+++ b/libgav1/src/dsp/x86/loop_filter_sse4.cc
@@ -1,4 +1,5 @@
-#include "src/dsp/x86/loop_filter_sse4.h"
+#include "src/dsp/dsp.h"
+#include "src/dsp/loop_filter.h"
 
 #if LIBGAV1_ENABLE_SSE4_1
 
diff --git a/libgav1/src/dsp/x86/loop_filter_sse4.h b/libgav1/src/dsp/x86/loop_filter_sse4.h
index 76f2846..841eeec 100644
--- a/libgav1/src/dsp/x86/loop_filter_sse4.h
+++ b/libgav1/src/dsp/x86/loop_filter_sse4.h
@@ -3,13 +3,12 @@
 
 #include "src/dsp/cpu.h"
 #include "src/dsp/dsp.h"
-#include "src/dsp/loop_filter.h"
 
 namespace libgav1 {
 namespace dsp {
 
-// Initializes Dsp::loop_filters with sse4 implementations. This function
-// is not thread-safe.
+// Initializes Dsp::loop_filters, see the defines below for specifics. This
+// function is not thread-safe.
 void LoopFilterInit_SSE4_1();
 
 }  // namespace dsp
diff --git a/libgav1/src/dsp/x86/loop_restoration_sse4.cc b/libgav1/src/dsp/x86/loop_restoration_sse4.cc
index fa48bcc..3a08fc4 100644
--- a/libgav1/src/dsp/x86/loop_restoration_sse4.cc
+++ b/libgav1/src/dsp/x86/loop_restoration_sse4.cc
@@ -1,4 +1,5 @@
-#include "src/dsp/x86/loop_restoration_sse4.h"
+#include "src/dsp/dsp.h"
+#include "src/dsp/loop_restoration.h"
 
 #if LIBGAV1_ENABLE_SSE4_1
 #include <smmintrin.h>
@@ -9,6 +10,7 @@
 #include <cstring>
 
 #include "src/dsp/common.h"
+#include "src/dsp/constants.h"
 #include "src/dsp/x86/common_sse4.h"
 #include "src/utils/common.h"
 #include "src/utils/compiler_attributes.h"
@@ -63,10 +65,9 @@
                          ptrdiff_t source_stride, ptrdiff_t dest_stride,
                          int width, int height,
                          RestorationBuffer* const buffer) {
-  const int* const inter_round_bits = buffer->inter_round_bits;
   int8_t filter[kSubPixelTaps];
   const int limit =
-      (1 << (8 + 1 + kWienerFilterBits - inter_round_bits[0])) - 1;
+      (1 << (8 + 1 + kWienerFilterBits - kInterRoundBitsHorizontal)) - 1;
   const auto* src = static_cast<const uint8_t*>(source);
   auto* dst = static_cast<uint8_t*>(dest);
   const ptrdiff_t buffer_stride = buffer->wiener_buffer_stride;
@@ -77,7 +78,7 @@
   src -= center_tap * source_stride + center_tap;
 
   const int horizontal_rounding =
-      1 << (8 + kWienerFilterBits - inter_round_bits[0] - 1);
+      1 << (8 + kWienerFilterBits - kInterRoundBitsHorizontal - 1);
   const __m128i v_horizontal_rounding =
       _mm_shufflelo_epi16(_mm_cvtsi32_si128(horizontal_rounding), 0);
   const __m128i v_limit = _mm_shufflelo_epi16(_mm_cvtsi32_si128(limit), 0);
@@ -86,13 +87,16 @@
   __m128i v_k3k2 = _mm_shufflelo_epi16(v_horizontal_filter, 0x55);
   __m128i v_k5k4 = _mm_shufflelo_epi16(v_horizontal_filter, 0xaa);
   __m128i v_k7k6 = _mm_shufflelo_epi16(v_horizontal_filter, 0xff);
-  const __m128i v_round_0 =
-      _mm_shufflelo_epi16(_mm_cvtsi32_si128(1 << (inter_round_bits[0] - 1)), 0);
-  const __m128i v_round_0_shift = _mm_cvtsi32_si128(inter_round_bits[0]);
-  const __m128i v_offset_shift = _mm_cvtsi32_si128(7 - inter_round_bits[0]);
+  const __m128i v_round_0 = _mm_shufflelo_epi16(
+      _mm_cvtsi32_si128(1 << (kInterRoundBitsHorizontal - 1)), 0);
+  const __m128i v_round_0_shift = _mm_cvtsi32_si128(kInterRoundBitsHorizontal);
+  const __m128i v_offset_shift =
+      _mm_cvtsi32_si128(7 - kInterRoundBitsHorizontal);
 
-  for (int y = 0; y < height + kSubPixelTaps - 2; ++y) {
-    for (int x = 0; x < width; x += 4) {
+  int y = 0;
+  do {
+    int x = 0;
+    do {
       // Run the Wiener filter on four sets of source samples at a time:
       //   src[x + 0] ... src[x + 6]
       //   src[x + 1] ... src[x + 7]
@@ -126,29 +130,30 @@
           _mm_add_epi16(v_rounded_sum0, v_horizontal_rounding);
       // Zero out the even bytes, calculate scaled down offset correction, and
       // add to sum here to prevent signed 16 bit outranging.
-      // (src[3] * 128) >> inter_round_bits[0]
+      // (src[3] * 128) >> kInterRoundBitsHorizontal
       const __m128i v_src_3x128 =
           _mm_sll_epi16(_mm_srli_epi16(v_src_32, 8), v_offset_shift);
       const __m128i v_rounded_sum = _mm_add_epi16(v_rounded_sum1, v_src_3x128);
       const __m128i v_a = _mm_max_epi16(v_rounded_sum, _mm_setzero_si128());
       const __m128i v_b = _mm_min_epi16(v_a, v_limit);
       StoreLo8(&wiener_buffer[x], v_b);
-    }
+      x += 4;
+    } while (x < width);
     src += source_stride;
     wiener_buffer += buffer_stride;
-  }
+  } while (++y < height + kSubPixelTaps - 2);
 
   wiener_buffer = buffer->wiener_buffer;
   // vertical filtering.
   PopulateWienerCoefficients(restoration_info, WienerInfo::kVertical, filter);
 
-  const int vertical_rounding = -(1 << (8 + inter_round_bits[1] - 1));
+  const int vertical_rounding = -(1 << (8 + kInterRoundBitsVertical - 1));
   const __m128i v_vertical_rounding =
       _mm_shuffle_epi32(_mm_cvtsi32_si128(vertical_rounding), 0);
   const __m128i v_offset_correction = _mm_set_epi16(0, 0, 0, 0, 128, 0, 0, 0);
-  const __m128i v_round_1 =
-      _mm_shuffle_epi32(_mm_cvtsi32_si128(1 << (inter_round_bits[1] - 1)), 0);
-  const __m128i v_round_1_shift = _mm_cvtsi32_si128(inter_round_bits[1]);
+  const __m128i v_round_1 = _mm_shuffle_epi32(
+      _mm_cvtsi32_si128(1 << (kInterRoundBitsVertical - 1)), 0);
+  const __m128i v_round_1_shift = _mm_cvtsi32_si128(kInterRoundBitsVertical);
   const __m128i v_vertical_filter0 = _mm_cvtepi8_epi16(LoadLo8(filter));
   const __m128i v_vertical_filter =
       _mm_add_epi16(v_vertical_filter0, v_offset_correction);
@@ -156,8 +161,10 @@
   v_k3k2 = _mm_shuffle_epi32(v_vertical_filter, 0x55);
   v_k5k4 = _mm_shuffle_epi32(v_vertical_filter, 0xaa);
   v_k7k6 = _mm_shuffle_epi32(v_vertical_filter, 0xff);
-  for (int y = 0; y < height; ++y) {
-    for (int x = 0; x < width; x += 4) {
+  y = 0;
+  do {
+    int x = 0;
+    do {
       const __m128i v_wb_0 = LoadLo8(&wiener_buffer[0 * buffer_stride + x]);
       const __m128i v_wb_1 = LoadLo8(&wiener_buffer[1 * buffer_stride + x]);
       const __m128i v_wb_2 = LoadLo8(&wiener_buffer[2 * buffer_stride + x]);
@@ -182,10 +189,11 @@
       const __m128i v_a = _mm_packs_epi32(v_rounded_sum, v_rounded_sum);
       const __m128i v_b = _mm_packus_epi16(v_a, v_a);
       Store4(&dst[x], v_b);
-    }
+      x += 4;
+    } while (x < width);
     dst += dest_stride;
     wiener_buffer += buffer_stride;
-  }
+  } while (++y < height);
 }
 
 // Section 7.17.3.
@@ -196,7 +204,7 @@
 //   a2 = 1;
 // else
 //   a2 = ((z << kSgrProjSgrBits) + (z >> 1)) / (z + 1);
-constexpr int x_by_xplus1[256] = {
+constexpr int kXByXPlus1[256] = {
     1,   128, 171, 192, 205, 213, 219, 224, 228, 230, 233, 235, 236, 238, 239,
     240, 241, 242, 243, 243, 244, 244, 245, 245, 246, 246, 247, 247, 247, 247,
     248, 248, 248, 248, 249, 249, 249, 249, 249, 250, 250, 250, 250, 250, 250,
@@ -262,10 +270,12 @@
 
   // Calculate intermediate results, including one-pixel border, for example,
   // if unit size is 64x64, we calculate 66x66 pixels.
-  for (int y = -1; y <= height; ++y) {
+  int y = -1;
+  do {
     const uint8_t* top_left = &src[(y - 1) * stride - 2];
     // Calculate the box vertical sums for each x position.
-    for (int vsx = -2; vsx <= width + 1; vsx += 4, top_left += 4) {
+    int vsx = -2;
+    do {
       const __m128i v_box0 = _mm_cvtepu8_epi32(Load4(top_left));
       const __m128i v_box1 = _mm_cvtepu8_epi32(Load4(top_left + stride));
       const __m128i v_box2 = _mm_cvtepu8_epi32(Load4(top_left + stride * 2));
@@ -278,9 +288,12 @@
       const __m128i v_b012 = _mm_add_epi32(v_b01, v_box2);
       StoreUnaligned16(&vertical_sum_of_squares[vsx], v_a012);
       StoreUnaligned16(&vertical_sums[vsx], v_b012);
-    }
+      top_left += 4;
+      vsx += 4;
+    } while (vsx <= width + 1);
 
-    for (int x = -1; x <= width; x += 4) {
+    int x = -1;
+    do {
       const __m128i v_a =
           HorizontalAddVerticalSumsRadius1(&vertical_sum_of_squares[x - 1]);
       const __m128i v_b =
@@ -297,11 +310,10 @@
       const __m128i v_z = _mm_min_epi32(
           v_255, RightShiftWithRounding_U32(_mm_mullo_epi32(v_p, v_s),
                                             kSgrProjScaleBits));
-      const __m128i v_a2 =
-          _mm_set_epi32(x_by_xplus1[_mm_extract_epi32(v_z, 3)],
-                        x_by_xplus1[_mm_extract_epi32(v_z, 2)],
-                        x_by_xplus1[_mm_extract_epi32(v_z, 1)],
-                        x_by_xplus1[_mm_extract_epi32(v_z, 0)]);
+      const __m128i v_a2 = _mm_set_epi32(kXByXPlus1[_mm_extract_epi32(v_z, 3)],
+                                         kXByXPlus1[_mm_extract_epi32(v_z, 2)],
+                                         kXByXPlus1[_mm_extract_epi32(v_z, 1)],
+                                         kXByXPlus1[_mm_extract_epi32(v_z, 0)]);
       // -----------------------
       // calc b2 and store
       // -----------------------
@@ -312,10 +324,11 @@
       StoreUnaligned16(
           &intermediate_result[1][x],
           RightShiftWithRounding_U32(v_b2, kSgrProjReciprocalBits));
-    }
+      x += 4;
+    } while (x <= width);
     intermediate_result[0] += array_stride;
     intermediate_result[1] += array_stride;
-  }
+  } while (++y <= height);
 }
 
 void BoxFilterPreProcessRadius2_SSE4_1(
@@ -332,10 +345,12 @@
 
   // Calculate intermediate results, including one-pixel border, for example,
   // if unit size is 64x64, we calculate 66x66 pixels.
-  for (int y = -1; y <= height; y += 2) {
+  int y = -1;
+  do {
     // Calculate the box vertical sums for each x position.
     const uint8_t* top_left = &src[(y - 2) * stride - 3];
-    for (int vsx = -3; vsx <= width + 2; vsx += 4, top_left += 4) {
+    int vsx = -3;
+    do {
       const __m128i v_box0 = _mm_cvtepu8_epi32(Load4(top_left));
       const __m128i v_box1 = _mm_cvtepu8_epi32(Load4(top_left + stride));
       const __m128i v_box2 = _mm_cvtepu8_epi32(Load4(top_left + stride * 2));
@@ -356,9 +371,12 @@
       const __m128i v_b01234 = _mm_add_epi32(v_b0123, v_box4);
       StoreUnaligned16(&vertical_sum_of_squares[vsx], v_a01234);
       StoreUnaligned16(&vertical_sums[vsx], v_b01234);
-    }
+      top_left += 4;
+      vsx += 4;
+    } while (vsx <= width + 2);
 
-    for (int x = -1; x <= width; x += 4) {
+    int x = -1;
+    do {
       const __m128i v_a =
           HorizontalAddVerticalSumsRadius2(&vertical_sum_of_squares[x - 2]);
       const __m128i v_b =
@@ -375,11 +393,10 @@
       const __m128i v_z = _mm_min_epi32(
           v_255, RightShiftWithRounding_U32(_mm_mullo_epi32(v_p, v_s),
                                             kSgrProjScaleBits));
-      const __m128i v_a2 =
-          _mm_set_epi32(x_by_xplus1[_mm_extract_epi32(v_z, 3)],
-                        x_by_xplus1[_mm_extract_epi32(v_z, 2)],
-                        x_by_xplus1[_mm_extract_epi32(v_z, 1)],
-                        x_by_xplus1[_mm_extract_epi32(v_z, 0)]);
+      const __m128i v_a2 = _mm_set_epi32(kXByXPlus1[_mm_extract_epi32(v_z, 3)],
+                                         kXByXPlus1[_mm_extract_epi32(v_z, 2)],
+                                         kXByXPlus1[_mm_extract_epi32(v_z, 1)],
+                                         kXByXPlus1[_mm_extract_epi32(v_z, 0)]);
       // -----------------------
       // calc b2 and store
       // -----------------------
@@ -390,10 +407,12 @@
       StoreUnaligned16(
           &intermediate_result[1][x],
           RightShiftWithRounding_U32(v_b2, kSgrProjReciprocalBits));
-    }
+      x += 4;
+    } while (x <= width);
     intermediate_result[0] += 2 * array_stride;
     intermediate_result[1] += 2 * array_stride;
-  }
+    y += 2;
+  } while (y <= height);
 }
 
 void BoxFilterPreProcess_SSE4_1(const RestorationUnitInfo& restoration_info,
@@ -543,7 +562,8 @@
         kRestorationBorder * intermediate_stride + kRestorationBorder;
 
     if (pass == 0) {
-      for (int y = 0; y < height; ++y) {
+      int y = 0;
+      do {
         const int shift = ((y & 1) != 0) ? 4 : 5;
         uint32_t* const array_start[2] = {
             buffer->box_filter_process_intermediate[0] +
@@ -554,7 +574,8 @@
             array_start[0] - intermediate_stride,
             array_start[1] - intermediate_stride};
         if ((y & 1) == 0) {  // even row
-          for (int x = 0; x < width; x += 4) {
+          int x = 0;
+          do {
             // 5 6 5
             // 0 0 0
             // 5 6 5
@@ -569,9 +590,11 @@
                 v_v, kSgrProjSgrBits + shift - kSgrProjRestoreBits);
 
             StoreUnaligned16(&filtered_output[x], v_filtered);
-          }
+            x += 4;
+          } while (x < width);
         } else {
-          for (int x = 0; x < width; x += 4) {
+          int x = 0;
+          do {
             // 0 0 0
             // 5 6 5
             // 0 0 0
@@ -586,13 +609,15 @@
                 v_v, kSgrProjSgrBits + shift - kSgrProjRestoreBits);
 
             StoreUnaligned16(&filtered_output[x], v_filtered);
-          }
+            x += 4;
+          } while (x < width);
         }
         src_ptr += stride;
         filtered_output += filtered_output_stride;
-      }
+      } while (++y < height);
     } else {
-      for (int y = 0; y < height; ++y) {
+      int y = 0;
+      do {
         const int shift = 5;
         uint32_t* const array_start[2] = {
             buffer->box_filter_process_intermediate[0] +
@@ -602,7 +627,8 @@
         uint32_t* intermediate_result2[2] = {
             array_start[0] - intermediate_stride,
             array_start[1] - intermediate_stride};
-        for (int x = 0; x < width; x += 4) {
+        int x = 0;
+        do {
           const __m128i v_A = Process3x3Block_343(&intermediate_result2[0][x],
                                                   intermediate_stride);
           const __m128i v_B = Process3x3Block_343(&intermediate_result2[1][x],
@@ -614,10 +640,11 @@
               v_v, kSgrProjSgrBits + shift - kSgrProjRestoreBits);
 
           StoreUnaligned16(&filtered_output[x], v_filtered);
-        }
+          x += 4;
+        } while (x < width);
         src_ptr += stride;
         filtered_output += filtered_output_stride;
-      }
+      } while (++y < height);
     }
   }
 }
@@ -652,8 +679,10 @@
   const __m128i v_r0_mask = _mm_cmpeq_epi32(v_r0, zero);
   const __m128i v_r1_mask = _mm_cmpeq_epi32(v_r1, zero);
 
-  for (int y = 0; y < height; ++y) {
-    for (int x = 0; x < width; x += 4) {
+  int y = 0;
+  do {
+    int x = 0;
+    do {
       const __m128i v_src = _mm_cvtepu8_epi32(Load4(src + x));
       const __m128i v_u = _mm_slli_epi32(v_src, kSgrProjRestoreBits);
       const __m128i v_v_a = _mm_mullo_epi32(v_w1, v_u);
@@ -674,13 +703,14 @@
       v_s = _mm_packs_epi32(v_s, v_s);
       v_s = _mm_packus_epi16(v_s, v_s);
       Store4(&dst[x], v_s);
-    }
+      x += 4;
+    } while (x < width);
 
     src += source_stride;
     dst += dest_stride;
     box_filter_process_output[0] += array_stride;
     box_filter_process_output[1] += array_stride;
-  }
+  } while (++y < height);
 }
 
 void Init8bpp() {
diff --git a/libgav1/src/dsp/x86/loop_restoration_sse4.h b/libgav1/src/dsp/x86/loop_restoration_sse4.h
index 522950f..2a91938 100644
--- a/libgav1/src/dsp/x86/loop_restoration_sse4.h
+++ b/libgav1/src/dsp/x86/loop_restoration_sse4.h
@@ -3,13 +3,12 @@
 
 #include "src/dsp/cpu.h"
 #include "src/dsp/dsp.h"
-#include "src/dsp/loop_restoration.h"
 
 namespace libgav1 {
 namespace dsp {
 
-// Initializes Dsp::loop_restorations with sse4 implementations. This function
-// is not thread-safe.
+// Initializes Dsp::loop_restorations, see the defines below for specifics.
+// This function is not thread-safe.
 void LoopRestorationInit_SSE4_1();
 
 }  // namespace dsp
diff --git a/libgav1/src/dsp/x86/obmc_sse4.cc b/libgav1/src/dsp/x86/obmc_sse4.cc
new file mode 100644
index 0000000..33aa8e4
--- /dev/null
+++ b/libgav1/src/dsp/x86/obmc_sse4.cc
@@ -0,0 +1,313 @@
+#include "src/dsp/dsp.h"
+#include "src/dsp/obmc.h"
+
+#if LIBGAV1_ENABLE_SSE4_1
+
+#include <xmmintrin.h>
+
+#include <cassert>
+#include <cstddef>
+#include <cstdint>
+
+#include "src/dsp/x86/common_sse4.h"
+#include "src/utils/common.h"
+#include "src/utils/constants.h"
+
+namespace libgav1 {
+namespace dsp {
+namespace {
+
+#include "src/dsp/obmc.inc"
+
+inline void OverlapBlendFromLeft2xH_SSE4_1(
+    uint8_t* const prediction, const ptrdiff_t prediction_stride,
+    const int height, const uint8_t* const obmc_prediction,
+    const ptrdiff_t obmc_prediction_stride) {
+  uint8_t* pred = prediction;
+  const uint8_t* obmc_pred = obmc_prediction;
+  const __m128i mask_inverter = _mm_cvtsi32_si128(0x40404040);
+  const __m128i mask_val = _mm_shufflelo_epi16(Load4(kObmcMask), 0);
+  // 64 - mask
+  const __m128i obmc_mask_val = _mm_sub_epi8(mask_inverter, mask_val);
+  const __m128i masks = _mm_unpacklo_epi8(mask_val, obmc_mask_val);
+  int y = height;
+  do {
+    const __m128i pred_val = Load2x2(pred, pred + prediction_stride);
+    const __m128i obmc_pred_val =
+        Load2x2(obmc_pred, obmc_pred + obmc_prediction_stride);
+
+    const __m128i terms = _mm_unpacklo_epi8(pred_val, obmc_pred_val);
+    const __m128i result =
+        RightShiftWithRounding_U16(_mm_maddubs_epi16(terms, masks), 6);
+    const __m128i packed_result = _mm_packus_epi16(result, result);
+    Store2(pred, packed_result);
+    pred += prediction_stride;
+    const int16_t second_row_result = _mm_extract_epi16(packed_result, 1);
+    memcpy(pred, &second_row_result, sizeof(second_row_result));
+    pred += prediction_stride;
+    obmc_pred += obmc_prediction_stride << 1;
+    y -= 2;
+  } while (y != 0);
+}
+
+inline void OverlapBlendFromLeft4xH_SSE4_1(
+    uint8_t* const prediction, const ptrdiff_t prediction_stride,
+    const int height, const uint8_t* const obmc_prediction,
+    const ptrdiff_t obmc_prediction_stride) {
+  uint8_t* pred = prediction;
+  const uint8_t* obmc_pred = obmc_prediction;
+  const __m128i mask_inverter = _mm_cvtsi32_si128(0x40404040);
+  const __m128i mask_val = Load4(kObmcMask + 2);
+  // 64 - mask
+  const __m128i obmc_mask_val = _mm_sub_epi8(mask_inverter, mask_val);
+  // Duplicate first half of vector.
+  const __m128i masks =
+      _mm_shuffle_epi32(_mm_unpacklo_epi8(mask_val, obmc_mask_val), 0x44);
+  int y = height;
+  do {
+    const __m128i pred_val0 = Load4(pred);
+    const __m128i obmc_pred_val0 = Load4(obmc_pred);
+    pred += prediction_stride;
+    obmc_pred += obmc_prediction_stride;
+
+    // Place the second row of each source in the second four bytes.
+    const __m128i pred_val =
+        _mm_alignr_epi8(Load4(pred), _mm_slli_si128(pred_val0, 12), 12);
+    const __m128i obmc_pred_val = _mm_alignr_epi8(
+        Load4(obmc_pred), _mm_slli_si128(obmc_pred_val0, 12), 12);
+    const __m128i terms = _mm_unpacklo_epi8(pred_val, obmc_pred_val);
+    const __m128i result =
+        RightShiftWithRounding_U16(_mm_maddubs_epi16(terms, masks), 6);
+    const __m128i packed_result = _mm_packus_epi16(result, result);
+    Store4(pred - prediction_stride, packed_result);
+    const int second_row_result = _mm_extract_epi32(packed_result, 1);
+    memcpy(pred, &second_row_result, sizeof(second_row_result));
+    pred += prediction_stride;
+    obmc_pred += obmc_prediction_stride;
+    y -= 2;
+  } while (y != 0);
+}
+
+inline void OverlapBlendFromLeft8xH_SSE4_1(
+    uint8_t* const prediction, const ptrdiff_t prediction_stride,
+    const int height, const uint8_t* const obmc_prediction,
+    const ptrdiff_t obmc_prediction_stride) {
+  uint8_t* pred = prediction;
+  const uint8_t* obmc_pred = obmc_prediction;
+  const __m128i mask_inverter = _mm_set1_epi8(64);
+  const __m128i mask_val = LoadLo8(kObmcMask + 6);
+  // 64 - mask
+  const __m128i obmc_mask_val = _mm_sub_epi8(mask_inverter, mask_val);
+  const __m128i masks = _mm_unpacklo_epi8(mask_val, obmc_mask_val);
+  int y = height;
+  do {
+    const __m128i pred_val = LoadLo8(pred);
+    const __m128i obmc_pred_val = LoadLo8(obmc_pred);
+    const __m128i terms = _mm_unpacklo_epi8(pred_val, obmc_pred_val);
+    const __m128i result =
+        RightShiftWithRounding_U16(_mm_maddubs_epi16(terms, masks), 6);
+
+    StoreLo8(pred, _mm_packus_epi16(result, result));
+    pred += prediction_stride;
+    obmc_pred += obmc_prediction_stride;
+  } while (--y != 0);
+}
+
+void OverlapBlendFromLeft_SSE4_1(void* const prediction,
+                                 const ptrdiff_t prediction_stride,
+                                 const int width, const int height,
+                                 const void* const obmc_prediction,
+                                 const ptrdiff_t obmc_prediction_stride) {
+  auto* pred = static_cast<uint8_t*>(prediction);
+  const auto* obmc_pred = static_cast<const uint8_t*>(obmc_prediction);
+
+  if (width == 2) {
+    OverlapBlendFromLeft2xH_SSE4_1(pred, prediction_stride, height, obmc_pred,
+                                   obmc_prediction_stride);
+    return;
+  }
+  if (width == 4) {
+    OverlapBlendFromLeft4xH_SSE4_1(pred, prediction_stride, height, obmc_pred,
+                                   obmc_prediction_stride);
+    return;
+  }
+  if (width == 8) {
+    OverlapBlendFromLeft8xH_SSE4_1(pred, prediction_stride, height, obmc_pred,
+                                   obmc_prediction_stride);
+    return;
+  }
+  const __m128i mask_inverter = _mm_set1_epi8(64);
+  const uint8_t* mask = kObmcMask + width - 2;
+  int x = 0;
+  do {
+    pred = static_cast<uint8_t*>(prediction) + x;
+    obmc_pred = static_cast<const uint8_t*>(obmc_prediction) + x;
+    const __m128i mask_val = LoadUnaligned16(mask + x);
+    // 64 - mask
+    const __m128i obmc_mask_val = _mm_sub_epi8(mask_inverter, mask_val);
+    const __m128i masks_lo = _mm_unpacklo_epi8(mask_val, obmc_mask_val);
+    const __m128i masks_hi = _mm_unpackhi_epi8(mask_val, obmc_mask_val);
+
+    int y = 0;
+    do {
+      const __m128i pred_val = LoadUnaligned16(pred);
+      const __m128i obmc_pred_val = LoadUnaligned16(obmc_pred);
+      const __m128i terms_lo = _mm_unpacklo_epi8(pred_val, obmc_pred_val);
+      const __m128i result_lo =
+          RightShiftWithRounding_U16(_mm_maddubs_epi16(terms_lo, masks_lo), 6);
+      const __m128i terms_hi = _mm_unpackhi_epi8(pred_val, obmc_pred_val);
+      const __m128i result_hi =
+          RightShiftWithRounding_U16(_mm_maddubs_epi16(terms_hi, masks_hi), 6);
+      StoreUnaligned16(pred, _mm_packus_epi16(result_lo, result_hi));
+
+      pred += prediction_stride;
+      obmc_pred += obmc_prediction_stride;
+    } while (++y < height);
+    x += 16;
+  } while (x < width);
+}
+
+inline void OverlapBlendFromTop4xH_SSE4_1(
+    uint8_t* const prediction, const ptrdiff_t prediction_stride,
+    const int height, const uint8_t* const obmc_prediction,
+    const ptrdiff_t obmc_prediction_stride) {
+  uint8_t* pred = prediction;
+  const uint8_t* obmc_pred = obmc_prediction;
+  const __m128i mask_inverter = _mm_set1_epi16(64);
+  const __m128i mask_shuffler = _mm_set_epi32(0x01010101, 0x01010101, 0, 0);
+  const __m128i mask_preinverter = _mm_set1_epi16(-256 | 1);
+
+  const uint8_t* mask = kObmcMask + height - 2;
+  const int compute_height = height - (height >> 2);
+  int y = 0;
+  do {
+    // First mask in the first half, second mask in the second half.
+    const __m128i mask_val = _mm_shuffle_epi8(
+        _mm_cvtsi32_si128(*reinterpret_cast<const uint16_t*>(mask + y)),
+        mask_shuffler);
+    const __m128i masks =
+        _mm_sub_epi8(mask_inverter, _mm_sign_epi8(mask_val, mask_preinverter));
+    const __m128i pred_val0 = Load4(pred);
+
+    const __m128i obmc_pred_val0 = Load4(obmc_pred);
+    pred += prediction_stride;
+    obmc_pred += obmc_prediction_stride;
+    const __m128i pred_val =
+        _mm_alignr_epi8(Load4(pred), _mm_slli_si128(pred_val0, 12), 12);
+    const __m128i obmc_pred_val = _mm_alignr_epi8(
+        Load4(obmc_pred), _mm_slli_si128(obmc_pred_val0, 12), 12);
+    const __m128i terms = _mm_unpacklo_epi8(obmc_pred_val, pred_val);
+    const __m128i result =
+        RightShiftWithRounding_U16(_mm_maddubs_epi16(terms, masks), 6);
+
+    const __m128i packed_result = _mm_packus_epi16(result, result);
+    Store4(pred - prediction_stride, packed_result);
+    Store4(pred, _mm_srli_si128(packed_result, 4));
+    pred += prediction_stride;
+    obmc_pred += obmc_prediction_stride;
+    y += 2;
+  } while (y < compute_height);
+}
+
+inline void OverlapBlendFromTop8xH_SSE4_1(
+    uint8_t* const prediction, const ptrdiff_t prediction_stride,
+    const int height, const uint8_t* const obmc_prediction,
+    const ptrdiff_t obmc_prediction_stride) {
+  uint8_t* pred = prediction;
+  const uint8_t* obmc_pred = obmc_prediction;
+  const uint8_t* mask = kObmcMask + height - 2;
+  const __m128i mask_inverter = _mm_set1_epi8(64);
+  const int compute_height = height - (height >> 2);
+  int y = compute_height;
+  do {
+    const __m128i mask_val = _mm_set1_epi8(mask[compute_height - y]);
+    // 64 - mask
+    const __m128i obmc_mask_val = _mm_sub_epi8(mask_inverter, mask_val);
+    const __m128i masks = _mm_unpacklo_epi8(mask_val, obmc_mask_val);
+    const __m128i pred_val = LoadLo8(pred);
+    const __m128i obmc_pred_val = LoadLo8(obmc_pred);
+    const __m128i terms = _mm_unpacklo_epi8(pred_val, obmc_pred_val);
+    const __m128i result =
+        RightShiftWithRounding_U16(_mm_maddubs_epi16(terms, masks), 6);
+
+    StoreLo8(pred, _mm_packus_epi16(result, result));
+    pred += prediction_stride;
+    obmc_pred += obmc_prediction_stride;
+  } while (--y != 0);
+}
+
+void OverlapBlendFromTop_SSE4_1(void* const prediction,
+                                const ptrdiff_t prediction_stride,
+                                const int width, const int height,
+                                const void* const obmc_prediction,
+                                const ptrdiff_t obmc_prediction_stride) {
+  auto* pred = static_cast<uint8_t*>(prediction);
+  const auto* obmc_pred = static_cast<const uint8_t*>(obmc_prediction);
+
+  if (width <= 4) {
+    OverlapBlendFromTop4xH_SSE4_1(pred, prediction_stride, height, obmc_pred,
+                                  obmc_prediction_stride);
+    return;
+  }
+  if (width == 8) {
+    OverlapBlendFromTop8xH_SSE4_1(pred, prediction_stride, height, obmc_pred,
+                                  obmc_prediction_stride);
+    return;
+  }
+
+  // Stop when mask value becomes 64.
+  const int compute_height = height - (height >> 2);
+  const __m128i mask_inverter = _mm_set1_epi8(64);
+  int y = 0;
+  const uint8_t* mask = kObmcMask + height - 2;
+  do {
+    const __m128i mask_val = _mm_set1_epi8(mask[y]);
+    // 64 - mask
+    const __m128i obmc_mask_val = _mm_sub_epi8(mask_inverter, mask_val);
+    const __m128i masks = _mm_unpacklo_epi8(mask_val, obmc_mask_val);
+    int x = 0;
+    do {
+      const __m128i pred_val = LoadUnaligned16(pred + x);
+      const __m128i obmc_pred_val = LoadUnaligned16(obmc_pred + x);
+      const __m128i terms_lo = _mm_unpacklo_epi8(pred_val, obmc_pred_val);
+      const __m128i result_lo =
+          RightShiftWithRounding_U16(_mm_maddubs_epi16(terms_lo, masks), 6);
+      const __m128i terms_hi = _mm_unpackhi_epi8(pred_val, obmc_pred_val);
+      const __m128i result_hi =
+          RightShiftWithRounding_U16(_mm_maddubs_epi16(terms_hi, masks), 6);
+      StoreUnaligned16(pred + x, _mm_packus_epi16(result_lo, result_hi));
+      x += 16;
+    } while (x < width);
+    pred += prediction_stride;
+    obmc_pred += obmc_prediction_stride;
+  } while (++y < compute_height);
+}
+
+void Init8bpp() {
+  Dsp* const dsp = dsp_internal::GetWritableDspTable(8);
+  assert(dsp != nullptr);
+#if DSP_ENABLED_8BPP_SSE4_1(ObmcVertical)
+  dsp->obmc_blend[kObmcDirectionVertical] = OverlapBlendFromTop_SSE4_1;
+#endif
+#if DSP_ENABLED_8BPP_SSE4_1(ObmcHorizontal)
+  dsp->obmc_blend[kObmcDirectionHorizontal] = OverlapBlendFromLeft_SSE4_1;
+#endif
+}
+
+}  // namespace
+
+void ObmcInit_SSE4_1() { Init8bpp(); }
+
+}  // namespace dsp
+}  // namespace libgav1
+
+#else   // !LIBGAV1_ENABLE_SSE4_1
+
+namespace libgav1 {
+namespace dsp {
+
+void ObmcInit_SSE4_1() {}
+
+}  // namespace dsp
+}  // namespace libgav1
+#endif  // LIBGAV1_ENABLE_SSE4_1
diff --git a/libgav1/src/dsp/x86/obmc_sse4.h b/libgav1/src/dsp/x86/obmc_sse4.h
new file mode 100644
index 0000000..aa00124
--- /dev/null
+++ b/libgav1/src/dsp/x86/obmc_sse4.h
@@ -0,0 +1,27 @@
+#ifndef LIBGAV1_SRC_DSP_X86_OBMC_SSE4_H_
+#define LIBGAV1_SRC_DSP_X86_OBMC_SSE4_H_
+
+#include "src/dsp/cpu.h"
+#include "src/dsp/dsp.h"
+
+namespace libgav1 {
+namespace dsp {
+
+// Initializes Dsp::obmc_blend[]. This function is not thread-safe.
+void ObmcInit_SSE4_1();
+
+}  // namespace dsp
+}  // namespace libgav1
+
+// If sse4 is enabled and the baseline isn't set due to a higher level of
+// optimization being enabled, signal the sse4 implementation should be used.
+#if LIBGAV1_ENABLE_SSE4_1
+#ifndef LIBGAV1_Dsp8bpp_ObmcVertical
+#define LIBGAV1_Dsp8bpp_ObmcVertical LIBGAV1_DSP_SSE4_1
+#endif
+#ifndef LIBGAV1_Dsp8bpp_ObmcHorizontal
+#define LIBGAV1_Dsp8bpp_ObmcHorizontal LIBGAV1_DSP_SSE4_1
+#endif
+#endif  // LIBGAV1_ENABLE_SSE4_1
+
+#endif  // LIBGAV1_SRC_DSP_X86_OBMC_SSE4_H_
diff --git a/libgav1/src/dsp/x86/transpose_sse4.h b/libgav1/src/dsp/x86/transpose_sse4.h
index 0b750cd..b222266 100644
--- a/libgav1/src/dsp/x86/transpose_sse4.h
+++ b/libgav1/src/dsp/x86/transpose_sse4.h
@@ -2,6 +2,7 @@
 #define LIBGAV1_SRC_DSP_X86_TRANSPOSE_SSE4_H_
 
 #include "src/dsp/dsp.h"
+#include "src/utils/compiler_attributes.h"
 
 #if LIBGAV1_ENABLE_SSE4_1
 #include <emmintrin.h>
@@ -9,7 +10,7 @@
 namespace libgav1 {
 namespace dsp {
 
-inline __m128i Transpose4x4_U8(const __m128i* const in) {
+LIBGAV1_ALWAYS_INLINE __m128i Transpose4x4_U8(const __m128i* const in) {
   // Unpack 16 bit elements. Goes from:
   // in[0]: 00 01 02 03
   // in[1]: 10 11 12 13
@@ -26,7 +27,125 @@
   return _mm_unpacklo_epi16(a0, a1);
 }
 
-inline void Transpose8x8_U16(const __m128i* const in, __m128i* const out) {
+LIBGAV1_ALWAYS_INLINE void Transpose4x4_U16(const __m128i* in, __m128i* out) {
+  // Unpack 16 bit elements. Goes from:
+  // in[0]: 00 01 02 03  XX XX XX XX
+  // in[1]: 10 11 12 13  XX XX XX XX
+  // in[2]: 20 21 22 23  XX XX XX XX
+  // in[3]: 30 31 32 33  XX XX XX XX
+  // to:
+  // ba:    00 10 01 11  02 12 03 13
+  // dc:    20 30 21 31  22 32 23 33
+  const __m128i ba = _mm_unpacklo_epi16(in[0], in[1]);
+  const __m128i dc = _mm_unpacklo_epi16(in[2], in[3]);
+  // Unpack 32 bit elements resulting in:
+  // dcba_lo: 00 10 20 30  01 11 21 31
+  // dcba_hi: 02 12 22 32  03 13 23 33
+  const __m128i dcba_lo = _mm_unpacklo_epi32(ba, dc);
+  const __m128i dcba_hi = _mm_unpackhi_epi32(ba, dc);
+  // Assign or shift right by 8 bytes resulting in:
+  // out[0]: 00 10 20 30  01 11 21 31
+  // out[1]: 01 11 21 31  XX XX XX XX
+  // out[2]: 02 12 22 32  03 13 23 33
+  // out[3]: 03 13 23 33  XX XX XX XX
+  out[0] = dcba_lo;
+  out[1] = _mm_srli_si128(dcba_lo, 8);
+  out[2] = dcba_hi;
+  out[3] = _mm_srli_si128(dcba_hi, 8);
+}
+
+LIBGAV1_ALWAYS_INLINE void Transpose4x8To8x4_U16(const __m128i* in,
+                                                 __m128i* out) {
+  // Unpack 16 bit elements. Goes from:
+  // in[0]: 00 01 02 03  XX XX XX XX
+  // in[1]: 10 11 12 13  XX XX XX XX
+  // in[2]: 20 21 22 23  XX XX XX XX
+  // in[3]: 30 31 32 33  XX XX XX XX
+  // in[4]: 40 41 42 43  XX XX XX XX
+  // in[5]: 50 51 52 53  XX XX XX XX
+  // in[6]: 60 61 62 63  XX XX XX XX
+  // in[7]: 70 71 72 73  XX XX XX XX
+  // to:
+  // a0:    00 10 01 11  02 12 03 13
+  // a1:    20 30 21 31  22 32 23 33
+  // a2:    40 50 41 51  42 52 43 53
+  // a3:    60 70 61 71  62 72 63 73
+  const __m128i a0 = _mm_unpacklo_epi16(in[0], in[1]);
+  const __m128i a1 = _mm_unpacklo_epi16(in[2], in[3]);
+  const __m128i a2 = _mm_unpacklo_epi16(in[4], in[5]);
+  const __m128i a3 = _mm_unpacklo_epi16(in[6], in[7]);
+
+  // Unpack 32 bit elements resulting in:
+  // b0: 00 10 20 30  01 11 21 31
+  // b1: 40 50 60 70  41 51 61 71
+  // b2: 02 12 22 32  03 13 23 33
+  // b3: 42 52 62 72  43 53 63 73
+  const __m128i b0 = _mm_unpacklo_epi32(a0, a1);
+  const __m128i b1 = _mm_unpacklo_epi32(a2, a3);
+  const __m128i b2 = _mm_unpackhi_epi32(a0, a1);
+  const __m128i b3 = _mm_unpackhi_epi32(a2, a3);
+
+  // Unpack 64 bit elements resulting in:
+  // out[0]: 00 10 20 30  40 50 60 70
+  // out[1]: 01 11 21 31  41 51 61 71
+  // out[2]: 02 12 22 32  42 52 62 72
+  // out[3]: 03 13 23 33  43 53 63 73
+  out[0] = _mm_unpacklo_epi64(b0, b1);
+  out[1] = _mm_unpackhi_epi64(b0, b1);
+  out[2] = _mm_unpacklo_epi64(b2, b3);
+  out[3] = _mm_unpackhi_epi64(b2, b3);
+}
+
+LIBGAV1_ALWAYS_INLINE void Transpose8x4To4x8_U16(const __m128i* in,
+                                                 __m128i* out) {
+  // Unpack 16 bit elements. Goes from:
+  // in[0]: 00 01 02 03  04 05 06 07
+  // in[1]: 10 11 12 13  14 15 16 17
+  // in[2]: 20 21 22 23  24 25 26 27
+  // in[3]: 30 31 32 33  34 35 36 37
+
+  // to:
+  // a0:    00 10 01 11  02 12 03 13
+  // a1:    20 30 21 31  22 32 23 33
+  // a4:    04 14 05 15  06 16 07 17
+  // a5:    24 34 25 35  26 36 27 37
+  const __m128i a0 = _mm_unpacklo_epi16(in[0], in[1]);
+  const __m128i a1 = _mm_unpacklo_epi16(in[2], in[3]);
+  const __m128i a4 = _mm_unpackhi_epi16(in[0], in[1]);
+  const __m128i a5 = _mm_unpackhi_epi16(in[2], in[3]);
+
+  // Unpack 32 bit elements resulting in:
+  // b0: 00 10 20 30  01 11 21 31
+  // b2: 04 14 24 34  05 15 25 35
+  // b4: 02 12 22 32  03 13 23 33
+  // b6: 06 16 26 36  07 17 27 37
+  const __m128i b0 = _mm_unpacklo_epi32(a0, a1);
+  const __m128i b2 = _mm_unpacklo_epi32(a4, a5);
+  const __m128i b4 = _mm_unpackhi_epi32(a0, a1);
+  const __m128i b6 = _mm_unpackhi_epi32(a4, a5);
+
+  // Unpack 64 bit elements resulting in:
+  // out[0]: 00 10 20 30  XX XX XX XX
+  // out[1]: 01 11 21 31  XX XX XX XX
+  // out[2]: 02 12 22 32  XX XX XX XX
+  // out[3]: 03 13 23 33  XX XX XX XX
+  // out[4]: 04 14 24 34  XX XX XX XX
+  // out[5]: 05 15 25 35  XX XX XX XX
+  // out[6]: 06 16 26 36  XX XX XX XX
+  // out[7]: 07 17 27 37  XX XX XX XX
+  const __m128i zeros = _mm_setzero_si128();
+  out[0] = _mm_unpacklo_epi64(b0, zeros);
+  out[1] = _mm_unpackhi_epi64(b0, zeros);
+  out[2] = _mm_unpacklo_epi64(b4, zeros);
+  out[3] = _mm_unpackhi_epi64(b4, zeros);
+  out[4] = _mm_unpacklo_epi64(b2, zeros);
+  out[5] = _mm_unpackhi_epi64(b2, zeros);
+  out[6] = _mm_unpacklo_epi64(b6, zeros);
+  out[7] = _mm_unpackhi_epi64(b6, zeros);
+}
+
+LIBGAV1_ALWAYS_INLINE void Transpose8x8_U16(const __m128i* const in,
+                                            __m128i* const out) {
   // Unpack 16 bit elements. Goes from:
   // in[0]: 00 01 02 03  04 05 06 07
   // in[1]: 10 11 12 13  14 15 16 17
diff --git a/libgav1/src/loop_filter_mask.cc b/libgav1/src/loop_filter_mask.cc
index 9304288..aaae2c7 100644
--- a/libgav1/src/loop_filter_mask.cc
+++ b/libgav1/src/loop_filter_mask.cc
@@ -10,6 +10,9 @@
 
 namespace libgav1 {
 
+// static.
+constexpr BitMaskSet LoopFilterMask::kPredictionModeDeltasMask;
+
 bool LoopFilterMask::Reset(int width, int height) {
   num_64x64_blocks_per_row_ = DivideBy64(width + 63);
   num_64x64_blocks_per_column_ = DivideBy64(height + 63);
@@ -31,10 +34,10 @@
   return true;
 }
 
-bool LoopFilterMask::Build(
+void LoopFilterMask::Build(
     const ObuSequenceHeader& sequence_header,
     const ObuFrameHeader& frame_header, int tile_group_start,
-    int tile_group_end, BlockParametersHolder* const block_parameters_holder,
+    int tile_group_end, const BlockParametersHolder& block_parameters_holder,
     const Array2D<TransformSize>& inter_transform_sizes) {
   for (int tile_number = tile_group_start; tile_number <= tile_group_end;
        ++tile_number) {
@@ -59,128 +62,124 @@
           (plane == kPlaneY) ? 0 : sequence_header.color_config.subsampling_x;
       const int8_t subsampling_y =
           (plane == kPlaneY) ? 0 : sequence_header.color_config.subsampling_y;
-      const int plane_width =
-          RightShiftWithRounding(frame_header.width, subsampling_x);
-      const int plane_height =
-          RightShiftWithRounding(frame_header.height, subsampling_y);
       const int vertical_step = 1 << subsampling_y;
       const int horizontal_step = 1 << subsampling_x;
 
-      // Build bit masks for vertical edges.
-      for (int row4x4 = GetDeblockPosition(row4x4_start, subsampling_y);
-           row4x4 < row4x4_end &&
-           MultiplyBy4(row4x4 >> subsampling_y) < plane_height;
-           row4x4 += vertical_step) {
-        if (column4x4_start == 0) break;  // Do not filter frame boundary.
-        const int column4x4 =
-            GetDeblockPosition(column4x4_start, subsampling_x);
-        const BlockParameters& bp =
-            *block_parameters_holder->Find(row4x4, column4x4);
-        const uint8_t vertical_level =
-            bp.deblock_filter_level[plane][kLoopFilterTypeVertical];
-        const BlockParameters& bp_left =
-            *block_parameters_holder->Find(row4x4, column4x4 - horizontal_step);
-        const uint8_t left_level =
-            bp_left.deblock_filter_level[plane][kLoopFilterTypeVertical];
-        const int unit_id = DivideBy16(row4x4) * num_64x64_blocks_per_row_ +
-                            DivideBy16(column4x4);
-        const int row = row4x4 % kNum4x4InLoopFilterMaskUnit;
-        const int column = column4x4 % kNum4x4InLoopFilterMaskUnit;
-        const int shift = LoopFilterMask::GetShift(row, column);
-        const int index = LoopFilterMask::GetIndex(row);
-        const auto mask = static_cast<uint64_t>(1) << shift;
-        // Tile boundary must be coding block boundary. So we don't have to
-        // check (!left_skip || !skip || is_vertical_border).
-        if (vertical_level != 0 || left_level != 0) {
-          const TransformSize tx_size = GetTransformSize(
-              frame_header.segmentation.lossless[bp.segment_id], bp.size,
-              static_cast<Plane>(plane),
-              inter_transform_sizes[row4x4][column4x4], subsampling_x,
-              subsampling_y);
-          const TransformSize left_tx_size = GetTransformSize(
-              frame_header.segmentation.lossless[bp_left.segment_id],
-              bp_left.size, static_cast<Plane>(plane),
-              inter_transform_sizes[row4x4][column4x4 - horizontal_step],
-              subsampling_x, subsampling_y);
-          // 0: 4x4, 1: 8x8, 2: 16x16.
-          const int transform_size_id =
-              std::min({kTransformWidthLog2[tx_size] - 2,
-                        kTransformWidthLog2[left_tx_size] - 2, 2});
-          SetLeft(mask, unit_id, plane, transform_size_id, index);
-          const uint8_t current_level =
-              (vertical_level == 0) ? left_level : vertical_level;
-          SetLevel(current_level, unit_id, plane, kLoopFilterTypeVertical,
-                   LoopFilterMask::GetLevelOffset(row, column));
+      // Build bit masks for vertical edges (except the frame boundary).
+      if (column4x4_start != 0) {
+        const int plane_height =
+            RightShiftWithRounding(frame_header.height, subsampling_y);
+        const int row4x4_limit =
+            std::min(row4x4_end, DivideBy4(plane_height + 3) << subsampling_y);
+        const int vertical_level_index =
+            kDeblockFilterLevelIndex[plane][kLoopFilterTypeVertical];
+        for (int row4x4 = GetDeblockPosition(row4x4_start, subsampling_y);
+             row4x4 < row4x4_limit; row4x4 += vertical_step) {
+          const int column4x4 =
+              GetDeblockPosition(column4x4_start, subsampling_x);
+          const BlockParameters& bp =
+              *block_parameters_holder.Find(row4x4, column4x4);
+          const uint8_t vertical_level =
+              bp.deblock_filter_level[vertical_level_index];
+          const BlockParameters& bp_left = *block_parameters_holder.Find(
+              row4x4, column4x4 - horizontal_step);
+          const uint8_t left_level =
+              bp_left.deblock_filter_level[vertical_level_index];
+          const int unit_id = DivideBy16(row4x4) * num_64x64_blocks_per_row_ +
+                              DivideBy16(column4x4);
+          const int row = row4x4 % kNum4x4InLoopFilterMaskUnit;
+          const int column = column4x4 % kNum4x4InLoopFilterMaskUnit;
+          const int shift = LoopFilterMask::GetShift(row, column);
+          const int index = LoopFilterMask::GetIndex(row);
+          const auto mask = static_cast<uint64_t>(1) << shift;
+          // Tile boundary must be coding block boundary. So we don't have to
+          // check (!left_skip || !skip || is_vertical_border).
+          if (vertical_level != 0 || left_level != 0) {
+            assert(inter_transform_sizes[row4x4] != nullptr);
+            const TransformSize tx_size =
+                (plane == kPlaneY) ? inter_transform_sizes[row4x4][column4x4]
+                                   : bp.uv_transform_size;
+            const TransformSize left_tx_size =
+                (plane == kPlaneY)
+                    ? inter_transform_sizes[row4x4][column4x4 - horizontal_step]
+                    : bp_left.uv_transform_size;
+            const LoopFilterTransformSizeId transform_size_id =
+                GetTransformSizeIdWidth(tx_size, left_tx_size);
+            SetLeft(mask, unit_id, plane, transform_size_id, index);
+            const uint8_t current_level =
+                (vertical_level == 0) ? left_level : vertical_level;
+            SetLevel(current_level, unit_id, plane, kLoopFilterTypeVertical,
+                     LoopFilterMask::GetLevelOffset(row, column));
+          }
         }
       }
 
-      // Build bit masks for horizontal edges.
-      for (int column4x4 = GetDeblockPosition(column4x4_start, subsampling_x);
-           column4x4 < column4x4_end &&
-           MultiplyBy4(column4x4 >> subsampling_x) < plane_width;
-           column4x4 += horizontal_step) {
-        if (row4x4_start == 0) break;  // Do not filter frame boundary.
-        const int row4x4 = GetDeblockPosition(row4x4_start, subsampling_y);
-        const BlockParameters& bp =
-            *block_parameters_holder->Find(row4x4, column4x4);
-        const uint8_t horizontal_level =
-            bp.deblock_filter_level[plane][kLoopFilterTypeHorizontal];
-        const BlockParameters& bp_top =
-            *block_parameters_holder->Find(row4x4 - vertical_step, column4x4);
-        const uint8_t top_level =
-            bp_top.deblock_filter_level[plane][kLoopFilterTypeHorizontal];
-        const int unit_id = DivideBy16(row4x4) * num_64x64_blocks_per_row_ +
-                            DivideBy16(column4x4);
-        const int row = row4x4 % kNum4x4InLoopFilterMaskUnit;
-        const int column = column4x4 % kNum4x4InLoopFilterMaskUnit;
-        const int shift = LoopFilterMask::GetShift(row, column);
-        const int index = LoopFilterMask::GetIndex(row);
-        const auto mask = static_cast<uint64_t>(1) << shift;
-        // Tile boundary must be coding block boundary. So we don't have to
-        // check (!top_skip || !skip || is_horizontal_border).
-        if (horizontal_level != 0 || top_level != 0) {
-          const TransformSize tx_size = GetTransformSize(
-              frame_header.segmentation.lossless[bp.segment_id], bp.size,
-              static_cast<Plane>(plane),
-              inter_transform_sizes[row4x4][column4x4], subsampling_x,
-              subsampling_y);
-          const TransformSize top_tx_size = GetTransformSize(
-              frame_header.segmentation.lossless[bp_top.segment_id],
-              bp_top.size, static_cast<Plane>(plane),
-              inter_transform_sizes[row4x4 - vertical_step][column4x4],
-              subsampling_x, subsampling_y);
-          // 0: 4x4, 1: 8x8, 2: 16x16.
-          const int transform_size_id =
-              std::min({kTransformHeightLog2[tx_size] - 2,
-                        kTransformHeightLog2[top_tx_size] - 2, 2});
-          SetTop(mask, unit_id, plane, transform_size_id, index);
-          const uint8_t current_level =
-              (horizontal_level == 0) ? top_level : horizontal_level;
-          SetLevel(current_level, unit_id, plane, kLoopFilterTypeHorizontal,
-                   LoopFilterMask::GetLevelOffset(row, column));
+      // Build bit masks for horizontal edges (except the frame boundary).
+      if (row4x4_start != 0) {
+        const int plane_width =
+            RightShiftWithRounding(frame_header.width, subsampling_x);
+        const int column4x4_limit = std::min(
+            column4x4_end, DivideBy4(plane_width + 3) << subsampling_y);
+        const int horizontal_level_index =
+            kDeblockFilterLevelIndex[plane][kLoopFilterTypeHorizontal];
+        for (int column4x4 = GetDeblockPosition(column4x4_start, subsampling_x);
+             column4x4 < column4x4_limit; column4x4 += horizontal_step) {
+          const int row4x4 = GetDeblockPosition(row4x4_start, subsampling_y);
+          const BlockParameters& bp =
+              *block_parameters_holder.Find(row4x4, column4x4);
+          const uint8_t horizontal_level =
+              bp.deblock_filter_level[horizontal_level_index];
+          const BlockParameters& bp_top =
+              *block_parameters_holder.Find(row4x4 - vertical_step, column4x4);
+          const uint8_t top_level =
+              bp_top.deblock_filter_level[horizontal_level_index];
+          const int unit_id = DivideBy16(row4x4) * num_64x64_blocks_per_row_ +
+                              DivideBy16(column4x4);
+          const int row = row4x4 % kNum4x4InLoopFilterMaskUnit;
+          const int column = column4x4 % kNum4x4InLoopFilterMaskUnit;
+          const int shift = LoopFilterMask::GetShift(row, column);
+          const int index = LoopFilterMask::GetIndex(row);
+          const auto mask = static_cast<uint64_t>(1) << shift;
+          // Tile boundary must be coding block boundary. So we don't have to
+          // check (!top_skip || !skip || is_horizontal_border).
+          if (horizontal_level != 0 || top_level != 0) {
+            assert(inter_transform_sizes[row4x4] != nullptr);
+            const TransformSize tx_size =
+                (plane == kPlaneY) ? inter_transform_sizes[row4x4][column4x4]
+                                   : bp.uv_transform_size;
+            const TransformSize top_tx_size =
+                (plane == kPlaneY)
+                    ? inter_transform_sizes[row4x4 - vertical_step][column4x4]
+                    : bp_top.uv_transform_size;
+            const LoopFilterTransformSizeId transform_size_id =
+                static_cast<LoopFilterTransformSizeId>(
+                    std::min({kTransformHeightLog2[tx_size] - 2,
+                              kTransformHeightLog2[top_tx_size] - 2, 2}));
+            SetTop(mask, unit_id, plane, transform_size_id, index);
+            const uint8_t current_level =
+                (horizontal_level == 0) ? top_level : horizontal_level;
+            SetLevel(current_level, unit_id, plane, kLoopFilterTypeHorizontal,
+                     LoopFilterMask::GetLevelOffset(row, column));
+          }
         }
       }
     }
   }
-  // Check bit masks.
-  for (int i = 0; i < num_64x64_blocks_; ++i) {
-    if (!IsValid(i)) return false;
-  }
-
-  return true;
+  assert(IsValid());
 }
 
-// Loop filter masks at different transform sizes should be mutually exclusive.
-bool LoopFilterMask::IsValid(int mask_id) const {
-  for (int plane = 0; plane < kMaxPlanes; ++plane) {
-    for (int i = 0; i < kNumTransformSizesLoopFilter; ++i) {
-      for (int j = i + 1; j < kNumTransformSizesLoopFilter; ++j) {
-        for (int k = 0; k < kNumLoopFilterMasks; ++k) {
-          if ((loop_filter_masks_[mask_id].left[plane][i][k] &
-               loop_filter_masks_[mask_id].left[plane][j][k]) != 0 ||
-              (loop_filter_masks_[mask_id].top[plane][i][k] &
-               loop_filter_masks_[mask_id].top[plane][j][k]) != 0) {
-            return false;
+bool LoopFilterMask::IsValid() const {
+  for (int mask_id = 0; mask_id < num_64x64_blocks_; ++mask_id) {
+    for (int plane = 0; plane < kMaxPlanes; ++plane) {
+      for (int i = 0; i < kNumLoopFilterTransformSizeIds; ++i) {
+        for (int j = i + 1; j < kNumLoopFilterTransformSizeIds; ++j) {
+          for (int k = 0; k < kNumLoopFilterMasks; ++k) {
+            if ((loop_filter_masks_[mask_id].left[plane][i][k] &
+                 loop_filter_masks_[mask_id].left[plane][j][k]) != 0 ||
+                (loop_filter_masks_[mask_id].top[plane][i][k] &
+                 loop_filter_masks_[mask_id].top[plane][j][k]) != 0) {
+              return false;
+            }
           }
         }
       }
diff --git a/libgav1/src/loop_filter_mask.h b/libgav1/src/loop_filter_mask.h
index eca4938..472c95f 100644
--- a/libgav1/src/loop_filter_mask.h
+++ b/libgav1/src/loop_filter_mask.h
@@ -10,6 +10,7 @@
 #include "src/dsp/dsp.h"
 #include "src/obu_parser.h"
 #include "src/utils/array_2d.h"
+#include "src/utils/bit_mask_set.h"
 #include "src/utils/block_parameters_holder.h"
 #include "src/utils/common.h"
 #include "src/utils/constants.h"
@@ -25,9 +26,10 @@
   // 4x4 blocks. It requires kNumLoopFilterMasks = 4 uint64_t to represent them.
   struct Data : public Allocable {
     uint8_t level[kMaxPlanes][kNumLoopFilterTypes][kNum4x4In64x64];
-    uint64_t left[kMaxPlanes][kNumTransformSizesLoopFilter]
+    uint64_t left[kMaxPlanes][kNumLoopFilterTransformSizeIds]
                  [kNumLoopFilterMasks];
-    uint64_t top[kMaxPlanes][kNumTransformSizesLoopFilter][kNumLoopFilterMasks];
+    uint64_t top[kMaxPlanes][kNumLoopFilterTransformSizeIds]
+                [kNumLoopFilterMasks];
   };
 
   LoopFilterMask() = default;
@@ -50,9 +52,10 @@
   // Before this function call, bit masks of transform edges other than those
   // on tile boundaries are built together with tile decoding, in
   // Tile::BuildBitMask().
-  bool Build(const ObuSequenceHeader& sequence_header,
+  void Build(const ObuSequenceHeader& sequence_header,
              const ObuFrameHeader& frame_header, int tile_group_start,
-             int tile_group_end, BlockParametersHolder* block_parameters_holder,
+             int tile_group_end,
+             const BlockParametersHolder& block_parameters_holder,
              const Array2D<TransformSize>& inter_transform_sizes);
 
   uint8_t GetLevel(int mask_id, int plane, LoopFilterType type,
@@ -60,24 +63,26 @@
     return loop_filter_masks_[mask_id].level[plane][type][offset];
   }
 
-  uint64_t GetLeft(int mask_id, int plane, int tx_size_id, int index) const {
+  uint64_t GetLeft(int mask_id, int plane, LoopFilterTransformSizeId tx_size_id,
+                   int index) const {
     return loop_filter_masks_[mask_id].left[plane][tx_size_id][index];
   }
 
-  uint64_t GetTop(int mask_id, int plane, int tx_size_id, int index) const {
+  uint64_t GetTop(int mask_id, int plane, LoopFilterTransformSizeId tx_size_id,
+                  int index) const {
     return loop_filter_masks_[mask_id].top[plane][tx_size_id][index];
   }
 
   int num_64x64_blocks_per_row() const { return num_64x64_blocks_per_row_; }
 
-  void SetLeft(uint64_t new_mask, int mask_id, int plane, int transform_size_id,
-               int index) {
+  void SetLeft(uint64_t new_mask, int mask_id, int plane,
+               LoopFilterTransformSizeId transform_size_id, int index) {
     loop_filter_masks_[mask_id].left[plane][transform_size_id][index] |=
         new_mask;
   }
 
-  void SetTop(uint64_t new_mask, int mask_id, int plane, int transform_size_id,
-              int index) {
+  void SetTop(uint64_t new_mask, int mask_id, int plane,
+              LoopFilterTransformSizeId transform_size_id, int index) {
     loop_filter_masks_[mask_id].top[plane][transform_size_id][index] |=
         new_mask;
   }
@@ -99,50 +104,68 @@
     return (row4x4 << 4) | column4x4;
   }
 
-  // 7.14.5.
-  static uint8_t GetDeblockFilterLevel(const ObuFrameHeader& frame_header,
-                                       const BlockParameters& bp, Plane plane,
-                                       int pass,
-                                       const int8_t delta_lf[kFrameLfCount]) {
-    const int filter_level_delta = (plane == kPlaneY) ? pass : plane + 1;
-    const int delta = frame_header.delta_lf.multi ? delta_lf[filter_level_delta]
-                                                  : delta_lf[0];
-    // TODO(chengchen): Could we reduce number of clips?
-    int level =
-        Clip3(frame_header.loop_filter.level[filter_level_delta] + delta, 0,
-              kMaxLoopFilterValue);
-    const auto feature = static_cast<SegmentFeature>(
-        kSegmentFeatureLoopFilterYVertical + filter_level_delta);
-    if (frame_header.segmentation.FeatureActive(bp.segment_id, feature)) {
-      level = Clip3(
-          level +
-              frame_header.segmentation.feature_data[bp.segment_id][feature],
-          0, kMaxLoopFilterValue);
-    }
-    if (frame_header.loop_filter.delta_enabled) {
-      const int shift = level >> 5;
-      if (bp.reference_frame[0] == kReferenceFrameIntra) {
-        level += LeftShift(
-            frame_header.loop_filter.ref_deltas[kReferenceFrameIntra], shift);
-      } else {
-        const int mode_id = kPredictionModeDeltasLookup[bp.y_mode];
-        level += LeftShift(
-            frame_header.loop_filter.ref_deltas[bp.reference_frame[0]] +
-                frame_header.loop_filter.mode_deltas[mode_id],
-            shift);
-      }
-      level = Clip3(level, 0, kMaxLoopFilterValue);
-    }
-    return level;
+  static constexpr int GetModeId(PredictionMode mode) {
+    return static_cast<int>(kPredictionModeDeltasMask.Contains(mode));
   }
 
-  bool IsValid(int mask_id) const;
+  // 7.14.5.
+  static void ComputeDeblockFilterLevels(
+      const ObuFrameHeader& frame_header, int segment_id, int level_index,
+      const int8_t delta_lf[kFrameLfCount],
+      uint8_t deblock_filter_levels[kNumReferenceFrameTypes][2]) {
+    const int delta = delta_lf[frame_header.delta_lf.multi ? level_index : 0];
+    uint8_t level = Clip3(frame_header.loop_filter.level[level_index] + delta,
+                          0, kMaxLoopFilterValue);
+    const auto feature = static_cast<SegmentFeature>(
+        kSegmentFeatureLoopFilterYVertical + level_index);
+    level = Clip3(
+        level + frame_header.segmentation.feature_data[segment_id][feature], 0,
+        kMaxLoopFilterValue);
+    if (!frame_header.loop_filter.delta_enabled) {
+      static_assert(sizeof(deblock_filter_levels[0][0]) == 1, "");
+      memset(deblock_filter_levels, level, kNumReferenceFrameTypes * 2);
+      return;
+    }
+    assert(frame_header.loop_filter.delta_enabled);
+    const int shift = level >> 5;
+    deblock_filter_levels[kReferenceFrameIntra][0] = Clip3(
+        level +
+            LeftShift(frame_header.loop_filter.ref_deltas[kReferenceFrameIntra],
+                      shift),
+        0, kMaxLoopFilterValue);
+    // deblock_filter_levels[kReferenceFrameIntra][1] is never used. So it does
+    // not have to be populated.
+    for (int reference_frame = kReferenceFrameIntra + 1;
+         reference_frame < kNumReferenceFrameTypes; ++reference_frame) {
+      for (int mode_id = 0; mode_id < 2; ++mode_id) {
+        deblock_filter_levels[reference_frame][mode_id] = Clip3(
+            level +
+                LeftShift(frame_header.loop_filter.ref_deltas[reference_frame] +
+                              frame_header.loop_filter.mode_deltas[mode_id],
+                          shift),
+            0, kMaxLoopFilterValue);
+      }
+    }
+  }
 
  private:
   std::unique_ptr<Data[]> loop_filter_masks_;
   int num_64x64_blocks_ = -1;
   int num_64x64_blocks_per_row_;
   int num_64x64_blocks_per_column_;
+
+  // Mask used to determine the index for mode_deltas lookup.
+  static constexpr BitMaskSet kPredictionModeDeltasMask{
+      BitMaskSet(kPredictionModeNearestMv, kPredictionModeNearMv,
+                 kPredictionModeNewMv, kPredictionModeNearestNearestMv,
+                 kPredictionModeNearNearMv, kPredictionModeNearestNewMv,
+                 kPredictionModeNewNearestMv, kPredictionModeNearNewMv,
+                 kPredictionModeNewNearMv, kPredictionModeNewNewMv)};
+
+  // Validates that the loop filter masks at different transform sizes are
+  // mutually exclusive. Only used in an assert. This function will not be used
+  // in optimized builds.
+  bool IsValid() const;
 };
 
 }  // namespace libgav1
diff --git a/libgav1/src/loop_restoration_info.cc b/libgav1/src/loop_restoration_info.cc
index e2b570e..cfeebed 100644
--- a/libgav1/src/loop_restoration_info.cc
+++ b/libgav1/src/loop_restoration_info.cc
@@ -6,6 +6,7 @@
 #include <cstddef>
 #include <cstdint>
 #include <memory>
+#include <new>
 
 #include "src/utils/common.h"
 #include "src/utils/logging.h"
@@ -24,37 +25,42 @@
 
 bool LoopRestorationInfo::Allocate() {
   const int num_planes = is_monochrome_ ? kMaxPlanesMonochrome : kMaxPlanes;
-  loop_restoration_info_.reserve(num_planes);
-  if (loop_restoration_info_.capacity() < static_cast<size_t>(num_planes)) {
-    return false;
-  }
-  loop_restoration_info_.resize(num_planes);
+  int total_num_units = 0;
   for (int plane = kPlaneY; plane < num_planes; ++plane) {
     if (loop_restoration_.type[plane] == kLoopRestorationTypeNone) {
       plane_needs_filtering_[plane] = false;
       continue;
     }
     plane_needs_filtering_[plane] = true;
-    const int tile_width = plane == kPlaneY
-                               ? width_
-                               : RightShiftWithRounding(width_, subsampling_x_);
-    const int tile_height =
-        plane == kPlaneY ? height_
-                         : RightShiftWithRounding(height_, subsampling_y_);
-    num_horizontal_units_[plane] = std::max(
-        1, (tile_width + DivideBy2(loop_restoration_.unit_size[plane])) /
-               loop_restoration_.unit_size[plane]);
-    num_vertical_units_[plane] = std::max(
-        1, (tile_height + DivideBy2(loop_restoration_.unit_size[plane])) /
-               loop_restoration_.unit_size[plane]);
+    const int width = (plane == kPlaneY)
+                          ? width_
+                          : RightShiftWithRounding(width_, subsampling_x_);
+    const int height = (plane == kPlaneY)
+                           ? height_
+                           : RightShiftWithRounding(height_, subsampling_y_);
+    num_horizontal_units_[plane] =
+        std::max(1, (width + DivideBy2(loop_restoration_.unit_size[plane])) /
+                        loop_restoration_.unit_size[plane]);
+    num_vertical_units_[plane] =
+        std::max(1, (height + DivideBy2(loop_restoration_.unit_size[plane])) /
+                        loop_restoration_.unit_size[plane]);
     num_units_[plane] =
         num_horizontal_units_[plane] * num_vertical_units_[plane];
-    loop_restoration_info_[plane].reserve(num_units_[plane]);
-    if (loop_restoration_info_[plane].capacity() <
-        static_cast<size_t>(num_units_[plane])) {
-      return false;
+    total_num_units += num_units_[plane];
+  }
+  // Allocate the RestorationUnitInfo arrays for all planes in a single heap
+  // allocation and divide up the buffer into arrays of the right sizes.
+  loop_restoration_info_buffer_.reset(new (std::nothrow)
+                                          RestorationUnitInfo[total_num_units]);
+  if (loop_restoration_info_buffer_ == nullptr) return false;
+  RestorationUnitInfo* loop_restoration_info =
+      loop_restoration_info_buffer_.get();
+  for (int plane = kPlaneY; plane < num_planes; ++plane) {
+    if (loop_restoration_.type[plane] == kLoopRestorationTypeNone) {
+      continue;
     }
-    loop_restoration_info_[plane].resize(num_units_[plane]);
+    loop_restoration_info_[plane] = loop_restoration_info;
+    loop_restoration_info += num_units_[plane];
   }
   return true;
 }
@@ -101,9 +107,9 @@
     std::array<RestorationUnitInfo, kMaxPlanes>* const reference_unit_info) {
   LoopRestorationType unit_restoration_type = kLoopRestorationTypeNone;
   if (loop_restoration_.type[plane] == kLoopRestorationTypeSwitchable) {
-    unit_restoration_type = kBitstreamRestorationTypeMap[reader->ReadSymbol(
-        symbol_decoder_context->restoration_type_cdf,
-        kRestorationTypeSymbolCount)];
+    unit_restoration_type = kBitstreamRestorationTypeMap
+        [reader->ReadSymbol<kRestorationTypeSymbolCount>(
+            symbol_decoder_context->restoration_type_cdf)];
   } else if (loop_restoration_.type[plane] == kLoopRestorationTypeWiener) {
     const bool use_wiener =
         reader->ReadSymbol(symbol_decoder_context->use_wiener_cdf);
diff --git a/libgav1/src/loop_restoration_info.h b/libgav1/src/loop_restoration_info.h
index 5792e69..61a1cfd 100644
--- a/libgav1/src/loop_restoration_info.h
+++ b/libgav1/src/loop_restoration_info.h
@@ -72,7 +72,11 @@
   int num_units(Plane plane) const { return num_units_[plane]; }
 
  private:
-  std::vector<std::vector<RestorationUnitInfo>> loop_restoration_info_;
+  // If plane_needs_filtering_[plane] is true, loop_restoration_info_[plane]
+  // points to an array of num_units_[plane] elements.
+  RestorationUnitInfo* loop_restoration_info_[kMaxPlanes];
+  // Owns the memory that loop_restoration_info_[plane] points to.
+  std::unique_ptr<RestorationUnitInfo[]> loop_restoration_info_buffer_;
   bool plane_needs_filtering_[kMaxPlanes];
   const LoopRestoration& loop_restoration_;
   uint32_t width_;
diff --git a/libgav1/src/motion_vector.cc b/libgav1/src/motion_vector.cc
index 2992896..f807da8 100644
--- a/libgav1/src/motion_vector.cc
+++ b/libgav1/src/motion_vector.cc
@@ -6,17 +6,18 @@
 #include <cstdlib>
 #include <memory>
 
+#include "src/utils/bit_mask_set.h"
 #include "src/utils/common.h"
 #include "src/utils/logging.h"
 
 namespace libgav1 {
 namespace {
 
-const int kMvBorder = 128;
-const int kProjectionMvClamp = 16383;
-const int kProjectionMvMaxVerticalOffset = 0;
-const int kProjectionMvMaxHorizontalOffset = 8;
-const int kInvalidMvValue = -32768;
+constexpr int kMvBorder = 128;
+constexpr int kProjectionMvClamp = 16383;
+constexpr int kProjectionMvMaxVerticalOffset = 0;
+constexpr int kProjectionMvMaxHorizontalOffset = 8;
+constexpr int kInvalidMvValue = -32768;
 
 // Applies the sign of |sign_value| to |value| (and does so without a branch).
 int ApplySign(int value, int sign_value) {
@@ -121,12 +122,12 @@
   }
 }
 
-inline bool HasNewMv(PredictionMode mode) {
-  return mode == kPredictionModeNewMv || mode == kPredictionModeNewNewMv ||
-         mode == kPredictionModeNearNewMv || mode == kPredictionModeNewNearMv ||
-         mode == kPredictionModeNearestNewMv ||
-         mode == kPredictionModeNewNearestMv;
-}
+constexpr BitMaskSet kPredictionModeNewMvMask(kPredictionModeNewMv,
+                                              kPredictionModeNewNewMv,
+                                              kPredictionModeNearNewMv,
+                                              kPredictionModeNewNearMv,
+                                              kPredictionModeNearestNewMv,
+                                              kPredictionModeNewNearestMv);
 
 // 7.10.2.8.
 void SearchStackSingle(const Tile::Block& block, int row, int column, int index,
@@ -148,7 +149,7 @@
     candidate_mv = bp.mv[index];
   }
   LowerMvPrecision(block, candidate_mv.mv);
-  *found_new_mv |= HasNewMv(candidate_mode);
+  *found_new_mv |= kPredictionModeNewMvMask.Contains(candidate_mode);
   *found_match = true;
   for (int i = 0; i < *num_mv_found; ++i) {
     if (ref_mv_stack[i].mv[0] == candidate_mv) {
@@ -183,7 +184,7 @@
     }
     LowerMvPrecision(block, candidate_mv[i].mv);
   }
-  *found_new_mv |= HasNewMv(candidate_mode);
+  *found_new_mv |= kPredictionModeNewMvMask.Contains(candidate_mode);
   *found_match = true;
   for (int i = 0; i < *num_mv_found; ++i) {
     if (ref_mv_stack[i].mv[0] == candidate_mv[0] &&
@@ -717,6 +718,21 @@
           sequence_header.enable_order_hint, sequence_header.order_hint_bits) *
       dst_sign;
   if (std::abs(reference_to_current_with_sign) > kMaxFrameDistance) return true;
+  // Index 0 of these two arrays are never used.
+  int reference_offsets[kNumReferenceFrameTypes];
+  bool skip_reference[kNumReferenceFrameTypes];
+  for (int source_reference_type = kReferenceFrameLast;
+       source_reference_type <= kNumInterReferenceFrameTypes;
+       ++source_reference_type) {
+    const int reference_offset = GetRelativeDistance(
+        current_frame.order_hint(source),
+        source_frame->order_hint(
+            static_cast<ReferenceFrameType>(source_reference_type)),
+        sequence_header.enable_order_hint, sequence_header.order_hint_bits);
+    skip_reference[source_reference_type] =
+        std::abs(reference_offset) > kMaxFrameDistance || reference_offset <= 0;
+    reference_offsets[source_reference_type] = reference_offset;
+  }
   // The column range has to be offset by kProjectionMvMaxHorizontalOffset since
   // coordinates in that range could end up being position_x8 because of
   // projection.
@@ -729,15 +745,11 @@
     for (int x8 = adjusted_x8_start; x8 < adjusted_x8_end; ++x8) {
       const ReferenceFrameType source_reference =
           *source_frame->motion_field_reference_frame(y8, x8);
-      if (source_reference <= kReferenceFrameIntra) continue;
-      const int reference_offset = GetRelativeDistance(
-          current_frame.order_hint(source),
-          source_frame->order_hint(source_reference),
-          sequence_header.enable_order_hint, sequence_header.order_hint_bits);
-      if (std::abs(reference_offset) > kMaxFrameDistance ||
-          reference_offset <= 0) {
+      if (source_reference <= kReferenceFrameIntra ||
+          skip_reference[source_reference]) {
         continue;
       }
+      const int reference_offset = reference_offsets[source_reference];
       const MotionVector& mv = *source_frame->motion_field_mv(y8, x8);
       MotionVector projection_mv;
       GetMvProjectionNoClamp(mv, reference_to_current_with_sign,
diff --git a/libgav1/src/obu_parser.cc b/libgav1/src/obu_parser.cc
index d7873c2..644f1a5 100644
--- a/libgav1/src/obu_parser.cc
+++ b/libgav1/src/obu_parser.cc
@@ -10,6 +10,7 @@
 #include "src/buffer_pool.h"
 #include "src/decoder_impl.h"
 #include "src/motion_vector.h"
+#include "src/utils/common.h"
 #include "src/utils/logging.h"
 
 namespace libgav1 {
@@ -22,15 +23,20 @@
 // 5.9.16.
 // Find the smallest value of k such that block_size << k is greater than or
 // equal to target.
-inline int TileLog2(int block_size, int target) {
+//
+// NOTE: TileLog2(block_size, target) is equal to
+//   CeilLog2(ceil((double)target / block_size))
+// where the division is a floating-point number division. (This equality holds
+// even when |target| is equal to 0.) In the special case of block_size == 1,
+// TileLog2(1, target) is equal to CeilLog2(target).
+int TileLog2(int block_size, int target) {
   int k = 0;
   for (; (block_size << k) < target; ++k) {
   }
   return k;
 }
 
-inline void ParseBitStreamLevel(BitStreamLevel* const level,
-                                uint8_t level_bits) {
+void ParseBitStreamLevel(BitStreamLevel* const level, uint8_t level_bits) {
   level->major = kMinimumMajorBitstreamLevel + (level_bits >> 2);
   level->minor = level_bits & 3;
 }
@@ -396,6 +402,7 @@
   sequence_header.film_grain_params_present = static_cast<bool>(scratch);
   // TODO(wtc): Compare new sequence header with old sequence header.
   sequence_header_ = sequence_header;
+  has_sequence_header_ = true;
   // Section 6.4.1: It is a requirement of bitstream conformance that if
   // OperatingPointIdc is equal to 0, then obu_extension_flag is equal to 0 for
   // all OBUs that follow this sequence header until the next sequence header.
@@ -489,6 +496,15 @@
   } else {
     frame_header_.superres_scale_denominator = kSuperResScaleNumerator;
   }
+  assert(frame_header_.width != 0);
+  assert(frame_header_.height != 0);
+  // Check if multiplying upscaled_width by height would overflow.
+  assert(frame_header_.upscaled_width >= frame_header_.width);
+  if (frame_header_.upscaled_width > INT32_MAX / frame_header_.height) {
+    LIBGAV1_DLOG(ERROR, "Frame dimensions too big: width=%d height=%d.",
+                 frame_header_.width, frame_header_.height);
+    return false;
+  }
   frame_header_.columns4x4 = ((frame_header_.width + 7) >> 3) << 1;
   frame_header_.rows4x4 = ((frame_header_.height + 7) >> 3) << 1;
   return true;
@@ -1510,9 +1526,9 @@
   const int sb_max_tile_area = kMaxTileArea >> MultiplyBy2(sb_size);
   const int minlog2_tile_columns = TileLog2(sb_max_tile_width, sb_columns);
   const int maxlog2_tile_columns =
-      TileLog2(1, std::min(sb_columns, static_cast<int>(kMaxTileColumns)));
+      CeilLog2(std::min(sb_columns, static_cast<int>(kMaxTileColumns)));
   const int maxlog2_tile_rows =
-      TileLog2(1, std::min(sb_rows, static_cast<int>(kMaxTileRows)));
+      CeilLog2(std::min(sb_rows, static_cast<int>(kMaxTileRows)));
   const int min_log2_tiles = std::max(
       minlog2_tile_columns, TileLog2(sb_max_tile_area, sb_rows * sb_columns));
   int64_t scratch;
@@ -1534,7 +1550,11 @@
     if (sb_tile_width <= 0) return false;
     int i = 0;
     for (int sb_start = 0; sb_start < sb_columns; sb_start += sb_tile_width) {
-      if (i > kMaxTileColumns) return false;
+      if (i >= kMaxTileColumns) {
+        LIBGAV1_DLOG(ERROR,
+                     "tile_columns would be greater than kMaxTileColumns.");
+        return false;
+      }
       tile_info->tile_column_start[i++] = sb_start << sb_shift;
     }
     tile_info->tile_column_start[i] = frame_header_.columns4x4;
@@ -1557,7 +1577,10 @@
     if (sb_tile_height <= 0) return false;
     i = 0;
     for (int sb_start = 0; sb_start < sb_rows; sb_start += sb_tile_height) {
-      if (i > kMaxTileRows) return false;
+      if (i >= kMaxTileRows) {
+        LIBGAV1_DLOG(ERROR, "tile_rows would be greater than kMaxTileRows.");
+        return false;
+      }
       tile_info->tile_row_start[i++] = sb_start << sb_shift;
     }
     tile_info->tile_row_start[i] = frame_header_.rows4x4;
@@ -1565,13 +1588,19 @@
   } else {
     int widest_tile_sb = 1;
     int i = 0;
-    for (int sb_start = 0; sb_start < sb_columns && i < kMaxTileColumns; ++i) {
+    for (int sb_start = 0; sb_start < sb_columns; ++i) {
+      if (i >= kMaxTileColumns) {
+        LIBGAV1_DLOG(ERROR,
+                     "tile_columns would be greater than kMaxTileColumns.");
+        return false;
+      }
       tile_info->tile_column_start[i] = sb_start << sb_shift;
       const int max_width =
           std::min(sb_columns - sb_start, static_cast<int>(sb_max_tile_width));
       int sb_size;
       if (!bit_reader_->DecodeUniform(max_width, &sb_size)) {
         LIBGAV1_DLOG(ERROR, "Not enough bits.");
+        return false;
       }
       ++sb_size;
       widest_tile_sb = std::max(sb_size, widest_tile_sb);
@@ -1579,7 +1608,7 @@
     }
     tile_info->tile_column_start[i] = frame_header_.columns4x4;
     tile_info->tile_columns = i;
-    tile_info->tile_columns_log2 = TileLog2(1, tile_info->tile_columns);
+    tile_info->tile_columns_log2 = CeilLog2(tile_info->tile_columns);
 
     int max_tile_area_sb = sb_rows * sb_columns;
     if (min_log2_tiles > 0) max_tile_area_sb >>= min_log2_tiles + 1;
@@ -1587,26 +1616,32 @@
         std::max(max_tile_area_sb / widest_tile_sb, 1);
 
     i = 0;
-    for (int sb_start = 0; sb_start < sb_rows && i < kMaxTileRows; ++i) {
+    for (int sb_start = 0; sb_start < sb_rows; ++i) {
+      if (i >= kMaxTileRows) {
+        LIBGAV1_DLOG(ERROR, "tile_rows would be greater than kMaxTileRows.");
+        return false;
+      }
       tile_info->tile_row_start[i] = sb_start << sb_shift;
       const int max_height = std::min(sb_rows - sb_start, max_tile_height_sb);
       int sb_size;
       if (!bit_reader_->DecodeUniform(max_height, &sb_size)) {
         LIBGAV1_DLOG(ERROR, "Not enough bits.");
+        return false;
       }
       ++sb_size;
       sb_start += sb_size;
     }
     tile_info->tile_row_start[i] = frame_header_.rows4x4;
     tile_info->tile_rows = i;
-    tile_info->tile_rows_log2 = TileLog2(1, tile_info->tile_rows);
+    tile_info->tile_rows_log2 = CeilLog2(tile_info->tile_rows);
   }
   tile_info->tile_count = tile_info->tile_rows * tile_info->tile_columns;
   tile_info->context_update_id = 0;
-  if ((tile_info->tile_columns | tile_info->tile_rows) > 1) {
-    OBU_READ_LITERAL_OR_FAIL(tile_info->tile_columns_log2 +
-                             tile_info->tile_rows_log2);
-    tile_info->context_update_id = scratch;
+  const int tile_bits =
+      tile_info->tile_columns_log2 + tile_info->tile_rows_log2;
+  if (tile_bits != 0) {
+    OBU_READ_LITERAL_OR_FAIL(tile_bits);
+    tile_info->context_update_id = static_cast<int16_t>(scratch);
     if (tile_info->context_update_id >= tile_info->tile_count) {
       LIBGAV1_DLOG(ERROR, "Invalid context_update_tile_id (%d) >= %d.",
                    tile_info->context_update_id, tile_info->tile_count);
@@ -1814,8 +1849,9 @@
           continue;
         }
         const int index = sequence_header_.operating_point_idc[i];
-        if (index == 0 || (InTemporalLayer(index, temporal_ids_.back()) &&
-                           InSpatialLayer(index, spatial_ids_.back()))) {
+        if (index == 0 ||
+            (InTemporalLayer(index, obu_headers_.back().temporal_id) &&
+             InSpatialLayer(index, obu_headers_.back().spatial_id))) {
           OBU_READ_LITERAL_OR_FAIL(
               sequence_header_.decoder_model_info.buffer_removal_time_length);
           frame_header_.buffer_removal_time[i] = static_cast<uint32_t>(scratch);
@@ -1991,6 +2027,9 @@
 }
 
 bool ObuParser::ParseFrameHeader() {
+  // Section 6.8.1: It is a requirement of bitstream conformance that a
+  // sequence header OBU has been received before a frame header OBU.
+  if (!has_sequence_header_) return false;
   if (!ParseFrameParameters()) return false;
   if (frame_header_.show_existing_frame) return true;
   bool status = ParseTileInfoSyntax() && ParseQuantizerParameters() &&
@@ -2000,6 +2039,7 @@
       frame_header_.segmentation);
   status =
       ParseQuantizerIndexDeltaParameters() && ParseLoopFilterDeltaParameters();
+  if (!status) return false;
   ComputeSegmentLosslessAndQIndex();
   status = ParseLoopFilterParameters();
   if (!status) return false;
@@ -2029,12 +2069,6 @@
   size -= 2;
   const auto type = static_cast<MetadataType>(scratch);
   switch (type) {
-    case kMetadataTypePrivateData:
-      for (size_t i = 0; i < size; ++i) {
-        OBU_READ_LITERAL_OR_FAIL(8);
-        metadata_.private_data.push_back(scratch);
-      }
-      break;
     case kMetadataTypeHdrContentLightLevel:
       OBU_READ_LITERAL_OR_FAIL(16);
       metadata_.max_cll = scratch;
@@ -2069,11 +2103,11 @@
   if (tile_group.start != next_tile_group_start_ ||
       tile_group.start > tile_group.end ||
       tile_group.end >= frame_header_.tile_info.tile_count) {
-    LIBGAV1_DLOG(
-        ERROR,
-        "Invalid tile group start %d (expected %d), end %d, tile_count %d.",
-        tile_group.start, next_tile_group_start_, tile_group.end,
-        frame_header_.tile_info.tile_count);
+    LIBGAV1_DLOG(ERROR,
+                 "Invalid tile group start %d or end %d: expected tile group "
+                 "start %d, tile_count %d.",
+                 tile_group.start, tile_group.end, next_tile_group_start_,
+                 frame_header_.tile_info.tile_count);
     return false;
   }
   next_tile_group_start_ = tile_group.end + 1;
@@ -2093,7 +2127,10 @@
   const size_t start_offset = bit_reader_->byte_offset();
   const int tile_bits =
       tile_info->tile_columns_log2 + tile_info->tile_rows_log2;
-  tile_groups_.emplace_back();
+  if (!tile_groups_.emplace_back()) {
+    LIBGAV1_DLOG(ERROR, "Could not add an element to tile_groups_.");
+    return false;
+  }
   auto& tile_group = tile_groups_.back();
   if (tile_bits == 0) {
     tile_group.start = 0;
@@ -2116,7 +2153,7 @@
     SetTileDataOffset(size, 1, bytes_consumed_so_far);
     return true;
   }
-  if (types_.back() == kObuFrame) {
+  if (obu_headers_.back().type == kObuFrame) {
     // 6.10.1: If obu_type is equal to OBU_FRAME, it is a requirement of
     // bitstream conformance that the value of tile_start_and_end_present_flag
     // is equal to 0.
@@ -2139,13 +2176,14 @@
 }
 
 bool ObuParser::ParseHeader() {
+  ObuHeader obu_header;
   int64_t scratch = bit_reader_->ReadBit();
   if (scratch != 0) {
     LIBGAV1_DLOG(ERROR, "forbidden_bit is not zero.");
     return false;
   }
   OBU_READ_LITERAL_OR_FAIL(4);
-  types_.push_back(static_cast<libgav1::ObuType>(scratch));
+  obu_header.type = static_cast<libgav1::ObuType>(scratch);
   OBU_READ_BIT_OR_FAIL;
   const auto extension_flag = static_cast<bool>(scratch);
   OBU_READ_BIT_OR_FAIL;
@@ -2161,7 +2199,7 @@
     LIBGAV1_DLOG(ERROR, "obu_reserved_1bit is not zero.");
     return false;
   }
-  has_extension_.push_back(extension_flag);
+  obu_header.has_extension = extension_flag;
   if (extension_flag) {
     if (extension_disallowed_) {
       LIBGAV1_DLOG(ERROR,
@@ -2169,19 +2207,19 @@
       return false;
     }
     OBU_READ_LITERAL_OR_FAIL(3);
-    temporal_ids_.push_back(scratch);
+    obu_header.temporal_id = scratch;
     OBU_READ_LITERAL_OR_FAIL(2);
-    spatial_ids_.push_back(scratch);
+    obu_header.spatial_id = scratch;
     OBU_READ_LITERAL_OR_FAIL(3);  // reserved.
     if (scratch != 0) {
       LIBGAV1_DLOG(ERROR, "extension_header_reserved_3bits is not zero.");
       return false;
     }
   } else {
-    temporal_ids_.push_back(0);
-    spatial_ids_.push_back(0);
+    obu_header.temporal_id = 0;
+    obu_header.spatial_id = 0;
   }
-  return true;
+  return obu_headers_.push_back(obu_header);
 }
 
 #undef OBU_READ_UVLC_OR_FAIL
@@ -2203,10 +2241,7 @@
   size_t size = size_;
 
   // Clear everything except the sequence header.
-  types_.clear();
-  has_extension_.clear();
-  temporal_ids_.clear();
-  spatial_ids_.clear();
+  obu_headers_.clear();
   frame_header_ = {};
   tile_groups_.clear();
   next_tile_group_start_ = 0;
@@ -2238,18 +2273,17 @@
       return false;
     }
 
-    if (types_.back() != kObuSequenceHeader &&
-        types_.back() != kObuTemporalDelimiter &&
+    const ObuHeader& obu_header = obu_headers_.back();
+    const ObuType obu_type = obu_header.type;
+    if (obu_type != kObuSequenceHeader && obu_type != kObuTemporalDelimiter &&
+        has_sequence_header_ &&
         sequence_header_.operating_point_idc[kOperatingPoint] != 0 &&
-        has_extension_.back() &&
+        obu_header.has_extension &&
         (!InTemporalLayer(sequence_header_.operating_point_idc[kOperatingPoint],
-                          temporal_ids_.back()) ||
+                          obu_header.temporal_id) ||
          !InSpatialLayer(sequence_header_.operating_point_idc[kOperatingPoint],
-                         spatial_ids_.back()))) {
-      types_.pop_back();
-      spatial_ids_.pop_back();
-      temporal_ids_.pop_back();
-      has_extension_.pop_back();
+                         obu_header.spatial_id))) {
+      obu_headers_.pop_back();
       bit_reader_->SkipBytes(obu_size);
       data += bit_reader_->byte_offset();
       size -= bit_reader_->byte_offset();
@@ -2258,7 +2292,7 @@
 
     const size_t obu_start_position = bit_reader_->bit_offset();
     bool obu_skipped = false;
-    switch (types_.back()) {
+    switch (obu_type) {
       case kObuTemporalDelimiter:
         break;
       case kObuSequenceHeader:
@@ -2348,6 +2382,9 @@
         parsed_one_full_frame =
             (tile_groups_.back().end == frame_header_.tile_info.tile_count - 1);
         break;
+      case kObuTileList:
+        LIBGAV1_DLOG(ERROR, "Decoding of tile list OBUs is not supported.");
+        return false;
       case kObuPadding:
       // TODO(b/120903866): Fix ParseMetadata() and then invoke that for the
       // kObuMetadata case.
@@ -2356,11 +2393,14 @@
         obu_skipped = true;
         break;
       default:
-        LIBGAV1_DLOG(ERROR, "Unknown OBU type: %d.", types_.back());
-        return false;
+        // Skip reserved OBUs. Section 6.2.2: Reserved units are for future use
+        // and shall be ignored by AV1 decoder.
+        bit_reader_->SkipBytes(obu_size);
+        obu_skipped = true;
+        break;
     }
-    if (obu_size > 0 && !obu_skipped && types_.back() != kObuFrame &&
-        types_.back() != kObuTileGroup) {
+    if (obu_size > 0 && !obu_skipped && obu_type != kObuFrame &&
+        obu_type != kObuTileGroup) {
       const size_t parsed_obu_size_in_bits =
           bit_reader_->bit_offset() - obu_start_position;
       if (obu_size * 8 < parsed_obu_size_in_bits) {
@@ -2368,14 +2408,14 @@
             ERROR,
             "Parsed OBU size (%zu bits) is greater than expected OBU size "
             "(%zu bytes) obu_type: %d.",
-            parsed_obu_size_in_bits, obu_size, types_.back());
+            parsed_obu_size_in_bits, obu_size, obu_type);
         return false;
       }
       if (!bit_reader_->VerifyAndSkipTrailingBits(obu_size * 8 -
                                                   parsed_obu_size_in_bits)) {
         LIBGAV1_DLOG(ERROR,
                      "Error when verifying trailing bits for obu type: %d",
-                     types_.back());
+                     obu_type);
         return false;
       }
     }
@@ -2386,12 +2426,16 @@
       LIBGAV1_DLOG(ERROR,
                    "OBU size (%zu) and consumed size (%zu) does not match for "
                    "obu_type: %d.",
-                   obu_size, consumed_obu_size, types_.back());
+                   obu_size, consumed_obu_size, obu_type);
       return false;
     }
     data += bytes_consumed;
     size -= bytes_consumed;
   }
+  if (!parsed_one_full_frame && seen_frame_header) {
+    LIBGAV1_DLOG(ERROR, "The last tile group in the frame was not received.");
+    return false;
+  }
   data_ = data;
   size_ = size;
   return true;
diff --git a/libgav1/src/obu_parser.h b/libgav1/src/obu_parser.h
index 0e1c0fa..9d972b1 100644
--- a/libgav1/src/obu_parser.h
+++ b/libgav1/src/obu_parser.h
@@ -5,7 +5,6 @@
 #include <cstddef>
 #include <cstdint>
 #include <memory>
-#include <vector>
 
 #include "src/decoder_buffer.h"
 #include "src/dsp/common.h"
@@ -15,6 +14,7 @@
 #include "src/utils/constants.h"
 #include "src/utils/raw_bit_reader.h"
 #include "src/utils/segmentation.h"
+#include "src/utils/vector.h"
 
 namespace libgav1 {
 
@@ -43,6 +43,13 @@
   kPrimaryReferenceNone = 7
 };  // anonymous enum
 
+struct ObuHeader {
+  ObuType type;
+  bool has_extension;
+  int8_t temporal_id;
+  int8_t spatial_id;
+};
+
 enum BitstreamProfile : uint8_t {
   kProfile0,
   kProfile1,
@@ -50,9 +57,17 @@
   kMaxProfiles
 };
 
+// In the bitstream the level is encoded in five bits: the first three bits
+// encode |major| - 2 and the last two bits encode |minor|.
+//
+// If the mapped level (major.minor) is in the tables in Annex A.3, there are
+// bitstream conformance requirements on the maximum or minimum values of
+// several variables. The encoded value of 31 (which corresponds to the mapped
+// level 9.3) is the "maximum parameters" level and imposes no level-based
+// constraints on the bitstream.
 struct BitStreamLevel {
-  uint8_t major;
-  uint8_t minor;
+  uint8_t major;  // Range: 2-9.
+  uint8_t minor;  // Range: 0-3.
 };
 
 enum ColorPrimaries : uint8_t {
@@ -262,6 +277,21 @@
   int32_t params[6];
 
   // Represent two shearing operations. Computed from |params| by SetupShear().
+  //
+  // The least significant six (= kWarpParamRoundingBits) bits are all zeros.
+  // (This means alpha, beta, gamma, and delta could be represented by a 10-bit
+  // signed integer.) The minimum value is INT16_MIN (= -32768) and the maximum
+  // value is 32704 = 0x7fc0, the largest int16_t value whose least significant
+  // six bits are all zeros.
+  //
+  // Valid warp parameters (as validated by SetupShear()) have smaller ranges.
+  // Their absolute values are less than 2^14 (= 16384). (This follows from
+  // the warpValid check at the end of Section 7.11.3.6.)
+  //
+  // NOTE: Section 7.11.3.6 of the spec allows a maximum value of 32768, which
+  // is outside the range of int16_t. When cast to int16_t, 32768 becomes
+  // -32768. This potential int16_t overflow does not matter because either
+  // 32768 or -32768 causes SetupShear() to return false,
   int16_t alpha;
   int16_t beta;
   int16_t gamma;
@@ -279,7 +309,7 @@
   int tile_rows_log2;
   int tile_rows;
   int tile_row_start[kMaxTileRows + 1];
-  uint8_t context_update_id;
+  int16_t context_update_id;
   uint8_t tile_size_bytes;
 };
 
@@ -368,13 +398,12 @@
 };
 
 enum MetadataType : uint8_t {
-  kMetadataTypePrivateData,
-  kMetadataTypeHdrContentLightLevel,
+  // 0 is reserved for AOM use.
+  kMetadataTypeHdrContentLightLevel = 1,
   kMetadataTypeHdrMasteringDisplayColorVolume
 };
 
 struct ObuMetadata {
-  std::vector<uint8_t> private_data;
   // Maximum content light level.
   uint16_t max_cll;
   // Maximum frame-average light level.
@@ -424,17 +453,16 @@
   bool ParseOneFrame();
 
   // Getters. Only valid if ParseOneFrame() completes successfully.
-  const std::vector<ObuType>& types() const { return types_; }
-  const std::vector<int8_t>& temporal_ids() const { return temporal_ids_; }
-  const std::vector<int8_t>& spatial_ids() const { return spatial_ids_; }
+  const Vector<ObuHeader>& obu_headers() const { return obu_headers_; }
   const ObuSequenceHeader& sequence_header() const { return sequence_header_; }
   const ObuFrameHeader& frame_header() const { return frame_header_; }
   const ObuMetadata& metadata() const { return metadata_; }
-  const std::vector<ObuTileGroup>& tile_groups() const { return tile_groups_; }
+  const Vector<ObuTileGroup>& tile_groups() const { return tile_groups_; }
 
   // Setters.
   void set_sequence_header(const ObuSequenceHeader& sequence_header) {
     sequence_header_ = sequence_header;
+    has_sequence_header_ = true;
   }
 
  private:
@@ -512,16 +540,15 @@
   size_t size_;
 
   // OBU elements. Only valid if ParseOneFrame() completes successfully.
-  std::vector<ObuType> types_;
-  std::vector<bool> has_extension_;
-  std::vector<int8_t> temporal_ids_;
-  std::vector<int8_t> spatial_ids_;
+  Vector<ObuHeader> obu_headers_;
   ObuSequenceHeader sequence_header_ = {};
   ObuFrameHeader frame_header_ = {};
   ObuMetadata metadata_ = {};
-  std::vector<ObuTileGroup> tile_groups_;
+  Vector<ObuTileGroup> tile_groups_;
   // The expected |start| value of the next ObuTileGroup.
   int next_tile_group_start_ = 0;
+  // If true, the sequence_header_ field is valid.
+  bool has_sequence_header_ = false;
   // If true, the obu_extension_flag syntax element in the OBU header must be
   // 0. Set to true when parsing a sequence header if OperatingPointIdc is 0.
   bool extension_disallowed_ = false;
diff --git a/libgav1/src/post_filter.cc b/libgav1/src/post_filter.cc
index bb29520..107a04c 100644
--- a/libgav1/src/post_filter.cc
+++ b/libgav1/src/post_filter.cc
@@ -1,15 +1,16 @@
 #include "src/post_filter.h"
 
 #include <algorithm>
+#include <atomic>
 #include <cassert>
-#include <condition_variable>  // NOLINT (unapproved c++11 header)
 #include <cstddef>
 #include <cstdint>
 #include <cstring>
-#include <mutex>  // NOLINT (unapproved c++11 header)
+#include <memory>
 
 #include "src/dsp/constants.h"
 #include "src/utils/array_2d.h"
+#include "src/utils/blocking_counter.h"
 #include "src/utils/logging.h"
 #include "src/utils/memory.h"
 #include "src/utils/types.h"
@@ -22,6 +23,37 @@
     {{7, 0, 2, 4, 5, 6, 6, 6}, {0, 1, 2, 3, 4, 5, 6, 7}}};
 
 template <typename Pixel>
+void ExtendFrame(uint8_t* const frame_start, const int width, const int height,
+                 ptrdiff_t stride, const int left, const int right,
+                 const int top, const int bottom) {
+  auto* const start = reinterpret_cast<Pixel*>(frame_start);
+  const Pixel* src = start;
+  Pixel* dst = start - left;
+  stride /= sizeof(Pixel);
+  // Copy to left and right borders.
+  for (int y = 0; y < height; ++y) {
+    Memset(dst, src[0], left);
+    Memset(dst + (left + width), src[width - 1], right);
+    src += stride;
+    dst += stride;
+  }
+  // Copy to top borders.
+  src = start - left;
+  dst = start - left - top * stride;
+  for (int y = 0; y < top; ++y) {
+    memcpy(dst, src, sizeof(Pixel) * stride);
+    dst += stride;
+  }
+  // Copy to bottom borders.
+  dst = start - left + height * stride;
+  src = dst - stride;
+  for (int y = 0; y < bottom; ++y) {
+    memcpy(dst, src, sizeof(Pixel) * stride);
+    dst += stride;
+  }
+}
+
+template <typename Pixel>
 void CopyPlane(const uint8_t* source, int source_stride, const int width,
                const int height, uint8_t* dest, int dest_stride) {
   auto* dst = reinterpret_cast<Pixel*>(dest);
@@ -35,29 +67,6 @@
   }
 }
 
-void CopyYuvBufferToSource(const YuvBuffer* const filtered_frame,
-                           YuvBuffer* const source_frame) {
-  const int num_planes =
-      filtered_frame->is_monochrome() ? kMaxPlanesMonochrome : kMaxPlanes;
-  for (int plane = kPlaneY; plane < num_planes; ++plane) {
-    if (filtered_frame->bitdepth() == 8) {
-      CopyPlane<uint8_t>(
-          filtered_frame->data(plane), filtered_frame->stride(plane),
-          filtered_frame->displayed_width(plane),
-          filtered_frame->displayed_height(plane), source_frame->data(plane),
-          source_frame->stride(plane));
-#if LIBGAV1_MAX_BITDEPTH >= 10
-    } else {
-      CopyPlane<uint16_t>(
-          filtered_frame->data(plane), filtered_frame->stride(plane),
-          filtered_frame->displayed_width(plane),
-          filtered_frame->displayed_height(plane), source_frame->data(plane),
-          source_frame->stride(plane));
-#endif
-    }
-  }
-}
-
 template <int bitdepth, typename Pixel>
 void ComputeSuperRes(const uint8_t* source, uint32_t source_stride,
                      const int upscaled_width, const int height,
@@ -89,49 +98,14 @@
 
 }  // namespace
 
-template <typename Pixel>
-void ExtendFrame(uint8_t* const frame_start, const int width, const int height,
-                 ptrdiff_t stride, const int left, const int right,
-                 const int top, const int bottom) {
-  auto* const start = reinterpret_cast<Pixel*>(frame_start);
-  const Pixel* src = start;
-  Pixel* dst = start - left;
-  stride /= sizeof(Pixel);
-  // Copy to left and right borders.
-  for (int y = 0; y < height; ++y) {
-    Memset(dst, src[0], left);
-    Memset(dst + (left + width), src[width - 1], right);
-    src += stride;
-    dst += stride;
-  }
-  // Copy to top borders.
-  src = start - left;
-  dst = start - left - top * stride;
-  for (int y = 0; y < top; ++y) {
-    memcpy(dst, src, sizeof(Pixel) * stride);
-    dst += stride;
-  }
-  // Copy to bottom borders.
-  dst = start - left + height * stride;
-  src = dst - stride;
-  for (int y = 0; y < bottom; ++y) {
-    memcpy(dst, src, sizeof(Pixel) * stride);
-    dst += stride;
-  }
-}
-
-// Matching definition of static data members.
-constexpr int PostFilter::kRestorationWindowWidth;
+// Static data member definitions.
+constexpr int PostFilter::kCdefLargeValue;
 
 bool PostFilter::ApplyFiltering() {
   if (DoDeblock() && !ApplyDeblockFilter()) return false;
   if (DoCdef() && !ApplyCdef()) return false;
   if (DoSuperRes() && !ApplySuperRes()) return false;
   if (DoRestoration() && !ApplyLoopRestoration()) return false;
-  if (DoCdef() && !DoRestoration()) {
-    // Copy (upscaled) cdef filtered frame to output source buffer.
-    CopyYuvBufferToSource(cdef_buffer_, source_buffer_);
-  }
   // Extend frame boundary for inter frame convolution, referencing.
   for (int plane = kPlaneY; plane < planes_; ++plane) {
     const int8_t subsampling_x = (plane == kPlaneY) ? 0 : subsampling_x_;
@@ -177,18 +151,39 @@
   }
 }
 
+void PostFilter::DeblockFilterWorker(const DeblockFilterJob* jobs, int num_jobs,
+                                     std::atomic<int>* job_counter,
+                                     DeblockFilter deblock_filter) {
+  int job_index;
+  while ((job_index = job_counter->fetch_add(1, std::memory_order_relaxed)) <
+         num_jobs) {
+    const DeblockFilterJob& job = jobs[job_index];
+    for (int column4x4 = 0, column_unit = 0;
+         column4x4 < frame_header_.columns4x4;
+         column4x4 += kNum4x4InLoopFilterMaskUnit, ++column_unit) {
+      const int unit_id = GetDeblockUnitId(job.row_unit, column_unit);
+      (this->*deblock_filter)(static_cast<Plane>(job.plane), job.row4x4,
+                              column4x4, unit_id);
+    }
+  }
+}
+
 bool PostFilter::ApplyDeblockFilterThreaded() {
-  std::mutex mutex;
-  int pending_jobs = 0;  // Guarded by |mutex|.
-  std::condition_variable pending_jobs_zero_condvar;
   const int jobs_per_plane = DivideBy16(frame_header_.rows4x4 + 15);
   const int num_workers = thread_pool_->num_threads();
-  const int jobs_for_threadpool_per_plane =
-      jobs_per_plane * num_workers / (num_workers + 1);
-  assert(jobs_for_threadpool_per_plane < jobs_per_plane);
-  std::vector<std::function<void()>> current_thread_jobs;
-  current_thread_jobs.reserve(planes_ *
-                              (jobs_per_plane - jobs_for_threadpool_per_plane));
+  int planes[kMaxPlanes];
+  planes[0] = kPlaneY;
+  int num_planes = 1;
+  for (int plane = kPlaneU; plane < planes_; ++plane) {
+    if (frame_header_.loop_filter.level[plane + 1] != 0) {
+      planes[num_planes++] = plane;
+    }
+  }
+  const int num_jobs = num_planes * jobs_per_plane;
+  std::unique_ptr<DeblockFilterJob[]> jobs_unique_ptr(
+      new (std::nothrow) DeblockFilterJob[num_jobs]);
+  if (jobs_unique_ptr == nullptr) return false;
+  DeblockFilterJob* jobs = jobs_unique_ptr.get();
   // The vertical filters are not dependent on each other. So simply schedule
   // them for all possible rows.
   //
@@ -200,62 +195,34 @@
   //
   // The only synchronization involved is to know when the each directional
   // filter is complete for the entire frame.
-  for (auto deblock_filter : {&PostFilter::VerticalDeblockFilter,
-                              &PostFilter::HorizontalDeblockFilter}) {
-    for (int plane = kPlaneY; plane < planes_; ++plane) {
-      if (plane != kPlaneY && frame_header_.loop_filter.level[plane + 1] == 0) {
-        continue;
-      }
-      {
-        std::lock_guard<std::mutex> lock(mutex);
-        pending_jobs += jobs_for_threadpool_per_plane;
-      }
+  for (DeblockFilter deblock_filter : {&PostFilter::VerticalDeblockFilter,
+                                       &PostFilter::HorizontalDeblockFilter}) {
+    int job_index = 0;
+    for (int i = 0; i < num_planes; ++i) {
+      const int plane = planes[i];
       for (int row4x4 = 0, row_unit = 0; row4x4 < frame_header_.rows4x4;
            row4x4 += kNum4x4InLoopFilterMaskUnit, ++row_unit) {
-        if (row_unit < jobs_for_threadpool_per_plane) {
-          thread_pool_->Schedule([this, plane, row4x4, row_unit, deblock_filter,
-                                  &mutex, &pending_jobs,
-                                  &pending_jobs_zero_condvar]() {
-            for (int column4x4 = 0, column_unit = 0;
-                 column4x4 < frame_header_.columns4x4;
-                 column4x4 += kNum4x4InLoopFilterMaskUnit, ++column_unit) {
-              const int unit_id = GetDeblockUnitId(row_unit, column_unit);
-              (this->*deblock_filter)(static_cast<Plane>(plane), row4x4,
-                                      column4x4, unit_id);
-            }
-            std::lock_guard<std::mutex> lock(mutex);
-            if (--pending_jobs == 0) {
-              // TODO(jzern): the mutex doesn't need to be locked to signal the
-              // condition.
-              pending_jobs_zero_condvar.notify_one();
-            }
-          });
-        } else {
-          current_thread_jobs.push_back(
-              [this, plane, row4x4, row_unit, deblock_filter]() {
-                for (int column4x4 = 0, column_unit = 0;
-                     column4x4 < frame_header_.columns4x4;
-                     column4x4 += kNum4x4InLoopFilterMaskUnit, ++column_unit) {
-                  const int unit_id = GetDeblockUnitId(row_unit, column_unit);
-                  (this->*deblock_filter)(static_cast<Plane>(plane), row4x4,
-                                          column4x4, unit_id);
-                }
-              });
-        }
+        assert(job_index < num_jobs);
+        DeblockFilterJob& job = jobs[job_index++];
+        job.plane = plane;
+        job.row4x4 = row4x4;
+        job.row_unit = row_unit;
       }
     }
-    // Run the jobs for current thread.
-    for (const auto& job : current_thread_jobs) {
-      job();
+    assert(job_index == num_jobs);
+    std::atomic<int> job_counter(0);
+    BlockingCounter pending_workers(num_workers);
+    for (int i = 0; i < num_workers; ++i) {
+      thread_pool_->Schedule([this, jobs, num_jobs, &job_counter,
+                              deblock_filter, &pending_workers]() {
+        DeblockFilterWorker(jobs, num_jobs, &job_counter, deblock_filter);
+        pending_workers.Decrement();
+      });
     }
-    current_thread_jobs.clear();
+    // Run the jobs on the current thread.
+    DeblockFilterWorker(jobs, num_jobs, &job_counter, deblock_filter);
     // Wait for the threadpool jobs to finish.
-    {
-      std::unique_lock<std::mutex> lock(mutex);
-      while (pending_jobs != 0) {
-        pending_jobs_zero_condvar.wait(lock);
-      }
-    }
+    pending_workers.Wait();
   }
   return true;
 }
@@ -300,132 +267,352 @@
   return true;
 }
 
-bool PostFilter::ApplyCdef() {
-  if (!cdef_filtered_buffer_.Realloc(
-          bitdepth_, planes_ == kMaxPlanesMonochrome, upscaled_width_, height_,
-          subsampling_x_, subsampling_y_, kBorderPixels,
-          /*byte_alignment=*/0, nullptr, nullptr, nullptr)) {
-    return false;
+void PostFilter::ComputeDeblockFilterLevels(
+    const int8_t delta_lf[kFrameLfCount],
+    uint8_t deblock_filter_levels[kMaxSegments][kFrameLfCount]
+                                 [kNumReferenceFrameTypes][2]) const {
+  if (!DoDeblock()) return;
+  for (int segment_id = 0;
+       segment_id < (frame_header_.segmentation.enabled ? kMaxSegments : 1);
+       ++segment_id) {
+    int level_index = 0;
+    for (; level_index < 2; ++level_index) {
+      LoopFilterMask::ComputeDeblockFilterLevels(
+          frame_header_, segment_id, level_index, delta_lf,
+          deblock_filter_levels[segment_id][level_index]);
+    }
+    for (; level_index < kFrameLfCount; ++level_index) {
+      if (frame_header_.loop_filter.level[level_index] != 0) {
+        LoopFilterMask::ComputeDeblockFilterLevels(
+            frame_header_, segment_id, level_index, delta_lf,
+            deblock_filter_levels[segment_id][level_index]);
+      }
+    }
   }
-  cdef_buffer_ = &cdef_filtered_buffer_;
+}
 
-  const size_t pixel_size =
-      (bitdepth_ == 8) ? sizeof(uint8_t) : sizeof(uint16_t);
+uint8_t* PostFilter::GetCdefBufferAndStride(
+    const int start_x, const int start_y, const int plane,
+    const int subsampling_x, const int subsampling_y,
+    const int window_buffer_plane_size, const int vertical_shift,
+    const int horizontal_shift, int* cdef_stride) {
+  if (!DoRestoration() && thread_pool_ != nullptr) {
+    // write output to threaded_window_buffer.
+    *cdef_stride = window_buffer_width_ * pixel_size_;
+    const int column_window = start_x % (window_buffer_width_ >> subsampling_x);
+    const int row_window = start_y % (window_buffer_height_ >> subsampling_y);
+    return threaded_window_buffer_ + plane * window_buffer_plane_size +
+           row_window * (*cdef_stride) + column_window * pixel_size_;
+  }
+  // write output to cdef_buffer_.
+  *cdef_stride = cdef_buffer_->stride(plane);
+  // In-place cdef is applied by writing the output to the top-left
+  // corner, if restoration is not present. In this case,
+  // cdef_buffer_ == source_buffer_.
+  const ptrdiff_t buffer_offset =
+      DoRestoration()
+          ? 0
+          : vertical_shift * (*cdef_stride) + horizontal_shift * pixel_size_;
+  return cdef_buffer_->data(plane) + start_y * (*cdef_stride) +
+         start_x * pixel_size_ + buffer_offset;
+}
+
+template <typename Pixel>
+void PostFilter::ApplyCdefForOneUnit(uint16_t* cdef_block, const int index,
+                                     const int block_width4x4,
+                                     const int block_height4x4,
+                                     const int row4x4_start,
+                                     const int column4x4_start) {
   const int coeff_shift = bitdepth_ - 8;
-  const int step_64x64 = kNum4x4BlocksWide[kBlock64x64];
   const int step = kNum4x4BlocksWide[kBlock8x8];
-  // Apply cdef on each 8x8 Y block and
-  // (8 >> subsampling_x)x(8 >> subsampling_y) UV block.
-  for (int row = 0; row < frame_header_.rows4x4; row += step_64x64) {
-    for (int column = 0; column < frame_header_.columns4x4;
-         column += step_64x64) {
-      const int index = cdef_index_[DivideBy16(row)][DivideBy16(column)];
-      const int block_width4x4 =
-          std::min(step_64x64, frame_header_.columns4x4 - column);
-      const int block_height4x4 =
-          std::min(step_64x64, frame_header_.rows4x4 - row);
-      if (index == -1) {
-        for (int plane = kPlaneY; plane < planes_; ++plane) {
-          const int subsampling_x = (plane == kPlaneY) ? 0 : subsampling_x_;
-          const int subsampling_y = (plane == kPlaneY) ? 0 : subsampling_y_;
-          const int start_x = MultiplyBy4(column) >> subsampling_x;
-          const int start_y = MultiplyBy4(row) >> subsampling_y;
-          const ptrdiff_t cdef_stride = cdef_buffer_->stride(plane);
-          uint8_t* const cdef_buffer = cdef_buffer_->data(plane) +
-                                       start_y * cdef_stride +
-                                       start_x * pixel_size;
-          const int src_stride = source_buffer_->stride(plane);
-          uint8_t* const src_buffer = source_buffer_->data(plane) +
-                                      start_y * src_stride +
-                                      start_x * pixel_size;
-          const int block_width = MultiplyBy4(block_width4x4) >> subsampling_x;
-          const int block_height =
-              MultiplyBy4(block_height4x4) >> subsampling_y;
+  const int horizontal_shift = -source_buffer_->alignment() / pixel_size_;
+  const int vertical_shift = -kCdefBorder;
+  const int window_buffer_plane_size =
+      window_buffer_width_ * window_buffer_height_ * pixel_size_;
+
+  if (index == -1) {
+    for (int plane = kPlaneY; plane < planes_; ++plane) {
+      const int subsampling_x = (plane == kPlaneY) ? 0 : subsampling_x_;
+      const int subsampling_y = (plane == kPlaneY) ? 0 : subsampling_y_;
+      const int start_x = MultiplyBy4(column4x4_start) >> subsampling_x;
+      const int start_y = MultiplyBy4(row4x4_start) >> subsampling_y;
+      int cdef_stride;
+      uint8_t* const cdef_buffer = GetCdefBufferAndStride(
+          start_x, start_y, plane, subsampling_x, subsampling_y,
+          window_buffer_plane_size, vertical_shift, horizontal_shift,
+          &cdef_stride);
+      const int src_stride = source_buffer_->stride(plane);
+      uint8_t* const src_buffer = source_buffer_->data(plane) +
+                                  start_y * src_stride + start_x * pixel_size_;
+      const int block_width = MultiplyBy4(block_width4x4) >> subsampling_x;
+      const int block_height = MultiplyBy4(block_height4x4) >> subsampling_y;
+      for (int y = 0; y < block_height; ++y) {
+        memcpy(cdef_buffer + y * cdef_stride, src_buffer + y * src_stride,
+               block_width * pixel_size_);
+      }
+    }
+    return;
+  }
+
+  PrepareCdefBlock<Pixel>(source_buffer_, planes_, subsampling_x_,
+                          subsampling_y_, frame_header_.width,
+                          frame_header_.height, block_width4x4, block_height4x4,
+                          row4x4_start, column4x4_start, cdef_block,
+                          kRestorationProcessingUnitSizeWithBorders);
+
+  for (int row4x4 = row4x4_start; row4x4 < row4x4_start + block_height4x4;
+       row4x4 += step) {
+    for (int column4x4 = column4x4_start;
+         column4x4 < column4x4_start + block_width4x4; column4x4 += step) {
+      const bool skip =
+          block_parameters_.Find(row4x4, column4x4) != nullptr &&
+          block_parameters_.Find(row4x4 + 1, column4x4) != nullptr &&
+          block_parameters_.Find(row4x4, column4x4 + 1) != nullptr &&
+          block_parameters_.Find(row4x4 + 1, column4x4 + 1) != nullptr &&
+          block_parameters_.Find(row4x4, column4x4)->skip &&
+          block_parameters_.Find(row4x4 + 1, column4x4)->skip &&
+          block_parameters_.Find(row4x4, column4x4 + 1)->skip &&
+          block_parameters_.Find(row4x4 + 1, column4x4 + 1)->skip;
+      int damping = frame_header_.cdef.damping + coeff_shift;
+      int direction_y;
+      int direction;
+      int variance;
+      uint8_t primary_strength;
+      uint8_t secondary_strength;
+
+      for (int plane = kPlaneY; plane < planes_; ++plane) {
+        const int subsampling_x = (plane == kPlaneY) ? 0 : subsampling_x_;
+        const int subsampling_y = (plane == kPlaneY) ? 0 : subsampling_y_;
+        const int start_x = MultiplyBy4(column4x4) >> subsampling_x;
+        const int start_y = MultiplyBy4(row4x4) >> subsampling_y;
+        const int block_width = 8 >> subsampling_x;
+        const int block_height = 8 >> subsampling_y;
+        int cdef_stride;
+        uint8_t* const cdef_buffer = GetCdefBufferAndStride(
+            start_x, start_y, plane, subsampling_x, subsampling_y,
+            window_buffer_plane_size, vertical_shift, horizontal_shift,
+            &cdef_stride);
+        const int src_stride = source_buffer_->stride(plane);
+        uint8_t* const src_buffer = source_buffer_->data(plane) +
+                                    start_y * src_stride +
+                                    start_x * pixel_size_;
+
+        if (skip) {  // No cdef filtering.
           for (int y = 0; y < block_height; ++y) {
             memcpy(cdef_buffer + y * cdef_stride, src_buffer + y * src_stride,
-                   block_width * pixel_size);
+                   block_width * pixel_size_);
           }
+          continue;
         }
+
+        if (plane == kPlaneY) {
+          dsp_.cdef_direction(src_buffer, src_stride, &direction_y, &variance);
+          primary_strength = frame_header_.cdef.y_primary_strength[index]
+                             << coeff_shift;
+          secondary_strength = frame_header_.cdef.y_secondary_strength[index]
+                               << coeff_shift;
+          direction = (primary_strength == 0) ? 0 : direction_y;
+          const int variance_strength =
+              ((variance >> 6) != 0) ? std::min(FloorLog2(variance >> 6), 12)
+                                     : 0;
+          primary_strength =
+              (variance != 0)
+                  ? (primary_strength * (4 + variance_strength) + 8) >> 4
+                  : 0;
+        } else {
+          primary_strength = frame_header_.cdef.uv_primary_strength[index]
+                             << coeff_shift;
+          secondary_strength = frame_header_.cdef.uv_secondary_strength[index]
+                               << coeff_shift;
+          direction = (primary_strength == 0)
+                          ? 0
+                          : kCdefUvDirection[subsampling_x_][subsampling_y_]
+                                            [direction_y];
+          damping = frame_header_.cdef.damping + coeff_shift - 1;
+        }
+
+        if ((primary_strength | secondary_strength) == 0) {
+          for (int y = 0; y < block_height; ++y) {
+            memcpy(cdef_buffer + y * cdef_stride, src_buffer + y * src_stride,
+                   block_width * pixel_size_);
+          }
+          continue;
+        }
+        uint16_t* cdef_src =
+            cdef_block + plane * kRestorationProcessingUnitSizeWithBorders *
+                             kRestorationProcessingUnitSizeWithBorders;
+        cdef_src += kCdefBorder * kRestorationProcessingUnitSizeWithBorders +
+                    kCdefBorder;
+        cdef_src += (MultiplyBy4(row4x4 - row4x4_start) >> subsampling_y) *
+                        kRestorationProcessingUnitSizeWithBorders +
+                    (MultiplyBy4(column4x4 - column4x4_start) >> subsampling_x);
+        dsp_.cdef_filter(cdef_src, kRestorationProcessingUnitSizeWithBorders,
+                         frame_header_.rows4x4, frame_header_.columns4x4,
+                         start_x, start_y, subsampling_x, subsampling_y,
+                         primary_strength, secondary_strength, damping,
+                         direction, cdef_buffer, cdef_stride);
+      }
+    }
+  }
+}
+
+template <typename Pixel>
+void PostFilter::ApplyCdefForOneRowInWindow(const int row4x4,
+                                            const int column4x4_start) {
+  const int step_64x64 = 16;  // = 64/4.
+  uint16_t cdef_block[kRestorationProcessingUnitSizeWithBorders *
+                      kRestorationProcessingUnitSizeWithBorders * 3];
+
+  for (int column4x4_64x64 = 0;
+       column4x4_64x64 < std::min(DivideBy4(window_buffer_width_),
+                                  frame_header_.columns4x4 - column4x4_start);
+       column4x4_64x64 += step_64x64) {
+    const int column4x4 = column4x4_start + column4x4_64x64;
+    const int index = cdef_index_[DivideBy16(row4x4)][DivideBy16(column4x4)];
+    const int block_width4x4 =
+        std::min(step_64x64, frame_header_.columns4x4 - column4x4);
+    const int block_height4x4 =
+        std::min(step_64x64, frame_header_.rows4x4 - row4x4);
+
+    ApplyCdefForOneUnit<Pixel>(cdef_block, index, block_width4x4,
+                               block_height4x4, row4x4, column4x4);
+  }
+}
+
+// Each thread processes one row inside the window.
+// Y, U, V planes are processed together inside one thread.
+template <typename Pixel>
+bool PostFilter::ApplyCdefThreaded() {
+  assert((window_buffer_height_ & 63) == 0);
+  const int num_workers = thread_pool_->num_threads();
+  const int horizontal_shift = -source_buffer_->alignment() / pixel_size_;
+  const int vertical_shift = -kCdefBorder;
+  const int window_buffer_plane_size =
+      window_buffer_width_ * window_buffer_height_ * pixel_size_;
+  const int window_buffer_height4x4 = DivideBy4(window_buffer_height_);
+  const int step_64x64 = 16;  // = 64/4.
+  for (int row4x4 = 0; row4x4 < frame_header_.rows4x4;
+       row4x4 += window_buffer_height4x4) {
+    const int actual_window_height4x4 =
+        std::min(window_buffer_height4x4, frame_header_.rows4x4 - row4x4);
+    const int vertical_units_per_window =
+        DivideBy16(actual_window_height4x4 + 15);
+    for (int column4x4 = 0; column4x4 < frame_header_.columns4x4;
+         column4x4 += DivideBy4(window_buffer_width_)) {
+      const int jobs_for_threadpool =
+          vertical_units_per_window * num_workers / (num_workers + 1);
+      BlockingCounter pending_jobs(jobs_for_threadpool);
+      int job_count = 0;
+      for (int row64x64 = 0; row64x64 < actual_window_height4x4;
+           row64x64 += step_64x64) {
+        if (job_count < jobs_for_threadpool) {
+          thread_pool_->Schedule(
+              [this, row4x4, column4x4, row64x64, &pending_jobs]() {
+                ApplyCdefForOneRowInWindow<Pixel>(row4x4 + row64x64, column4x4);
+                pending_jobs.Decrement();
+              });
+        } else {
+          ApplyCdefForOneRowInWindow<Pixel>(row4x4 + row64x64, column4x4);
+        }
+        ++job_count;
+      }
+      pending_jobs.Wait();
+      if (DoRestoration()) continue;
+
+      // Copy |threaded_window_buffer_| to cdef_buffer_ (== source_buffer_).
+      assert(cdef_buffer_ == source_buffer_);
+      for (int plane = kPlaneY; plane < planes_; ++plane) {
+        const int cdef_stride = cdef_buffer_->stride(plane);
+        const ptrdiff_t buffer_offset =
+            vertical_shift * cdef_stride + horizontal_shift * pixel_size_;
+        const int8_t subsampling_x = (plane == kPlaneY) ? 0 : subsampling_x_;
+        const int8_t subsampling_y = (plane == kPlaneY) ? 0 : subsampling_y_;
+        const int plane_row = MultiplyBy4(row4x4) >> subsampling_y;
+        const int plane_column = MultiplyBy4(column4x4) >> subsampling_x;
+        int copy_width = std::min(frame_header_.columns4x4 - column4x4,
+                                  DivideBy4(window_buffer_width_));
+        copy_width = MultiplyBy4(copy_width) >> subsampling_x;
+        int copy_height =
+            std::min(frame_header_.rows4x4 - row4x4, window_buffer_height4x4);
+        copy_height = MultiplyBy4(copy_height) >> subsampling_y;
+        CopyPlane<Pixel>(
+            threaded_window_buffer_ + plane * window_buffer_plane_size,
+            window_buffer_width_ * pixel_size_, copy_width, copy_height,
+            cdef_buffer_->data(plane) + plane_row * cdef_stride +
+                plane_column * pixel_size_ + buffer_offset,
+            cdef_stride);
+      }
+    }
+  }
+  if (!DoRestoration()) {
+    for (int plane = kPlaneY; plane < planes_; ++plane) {
+      if (!cdef_buffer_->ShiftBuffer(plane, horizontal_shift, vertical_shift)) {
+        LIBGAV1_DLOG(ERROR,
+                     "Error shifting frame buffer head pointer at plane: %d",
+                     plane);
+        return false;
+      }
+    }
+  }
+
+  return true;
+}
+
+bool PostFilter::ApplyCdef() {
+  if (!DoRestoration()) {
+    cdef_buffer_ = source_buffer_;
+  } else {
+    if (!cdef_filtered_buffer_.Realloc(
+            bitdepth_, planes_ == kMaxPlanesMonochrome, upscaled_width_,
+            height_, subsampling_x_, subsampling_y_, kBorderPixels,
+            /*byte_alignment=*/0, nullptr, nullptr, nullptr)) {
+      return false;
+    }
+    cdef_buffer_ = &cdef_filtered_buffer_;
+  }
+
+  if (thread_pool_ != nullptr) {
+#if LIBGAV1_MAX_BITDEPTH >= 10
+    if (bitdepth_ >= 10) {
+      return ApplyCdefThreaded<uint16_t>();
+    }
+#endif
+    return ApplyCdefThreaded<uint8_t>();
+  }
+
+  const int step_64x64 = 16;  // = 64/4.
+  // Apply cdef on each 8x8 Y block and
+  // (8 >> subsampling_x)x(8 >> subsampling_y) UV block.
+  for (int row4x4 = 0; row4x4 < frame_header_.rows4x4; row4x4 += step_64x64) {
+    for (int column4x4 = 0; column4x4 < frame_header_.columns4x4;
+         column4x4 += step_64x64) {
+      const int index = cdef_index_[DivideBy16(row4x4)][DivideBy16(column4x4)];
+      const int block_width4x4 =
+          std::min(step_64x64, frame_header_.columns4x4 - column4x4);
+      const int block_height4x4 =
+          std::min(step_64x64, frame_header_.rows4x4 - row4x4);
+
+#if LIBGAV1_MAX_BITDEPTH >= 10
+      if (bitdepth_ >= 10) {
+        ApplyCdefForOneUnit<uint16_t>(cdef_block_, index, block_width4x4,
+                                      block_height4x4, row4x4, column4x4);
         continue;
       }
-
-      for (int row4x4 = row; row4x4 < row + block_height4x4; row4x4 += step) {
-        for (int column4x4 = column; column4x4 < column + block_width4x4;
-             column4x4 += step) {
-          const bool skip =
-              block_parameters_->Find(row4x4, column4x4) != nullptr &&
-              block_parameters_->Find(row4x4 + 1, column4x4) != nullptr &&
-              block_parameters_->Find(row4x4, column4x4 + 1) != nullptr &&
-              block_parameters_->Find(row4x4 + 1, column4x4 + 1) != nullptr &&
-              block_parameters_->Find(row4x4, column4x4)->skip &&
-              block_parameters_->Find(row4x4 + 1, column4x4)->skip &&
-              block_parameters_->Find(row4x4, column4x4 + 1)->skip &&
-              block_parameters_->Find(row4x4 + 1, column4x4 + 1)->skip;
-          int damping = frame_header_.cdef.damping + coeff_shift;
-          int direction_y;
-          int direction;
-          int variance;
-          uint8_t primary_strength;
-          uint8_t secondary_strength;
-
-          for (int plane = kPlaneY; plane < planes_; ++plane) {
-            const int subsampling_x = (plane == kPlaneY) ? 0 : subsampling_x_;
-            const int subsampling_y = (plane == kPlaneY) ? 0 : subsampling_y_;
-            const int start_x = MultiplyBy4(column4x4) >> subsampling_x;
-            const int start_y = MultiplyBy4(row4x4) >> subsampling_y;
-            const int block_width = 8 >> subsampling_x;
-            const int block_height = 8 >> subsampling_y;
-            const ptrdiff_t cdef_stride = cdef_buffer_->stride(plane);
-            uint8_t* const cdef_buffer = cdef_buffer_->data(plane) +
-                                         start_y * cdef_stride +
-                                         start_x * pixel_size;
-            const int src_stride = source_buffer_->stride(plane);
-            uint8_t* const src_buffer = source_buffer_->data(plane) +
-                                        start_y * src_stride +
-                                        start_x * pixel_size;
-            for (int y = 0; y < block_height; ++y) {
-              memcpy(cdef_buffer + y * cdef_stride, src_buffer + y * src_stride,
-                     block_width * pixel_size);
-            }
-            if (skip) continue;  // No cdef filtering.
-
-            if (plane == kPlaneY) {
-              dsp_.cdef_direction(cdef_buffer, cdef_stride, &direction_y,
-                                  &variance);
-              primary_strength = frame_header_.cdef.y_primary_strength[index]
-                                 << coeff_shift;
-              secondary_strength =
-                  frame_header_.cdef.y_secondary_strength[index] << coeff_shift;
-              direction = (primary_strength == 0) ? 0 : direction_y;
-              const int variance_strength =
-                  ((variance >> 6) != 0)
-                      ? std::min(FloorLog2(variance >> 6), 12)
-                      : 0;
-              primary_strength =
-                  (variance != 0)
-                      ? (primary_strength * (4 + variance_strength) + 8) >> 4
-                      : 0;
-            } else {
-              primary_strength = frame_header_.cdef.uv_primary_strength[index]
-                                 << coeff_shift;
-              secondary_strength =
-                  frame_header_.cdef.uv_secondary_strength[index]
-                  << coeff_shift;
-              direction = (primary_strength == 0)
-                              ? 0
-                              : kCdefUvDirection[subsampling_x_][subsampling_y_]
-                                                [direction_y];
-              damping = frame_header_.cdef.damping + coeff_shift - 1;
-            }
-            // TODO(chengchen): Possible early termination if all parameters are
-            // 0.
-            dsp_.cdef_filter(src_buffer, src_stride, frame_header_.rows4x4,
-                             frame_header_.columns4x4, start_x, start_y,
-                             subsampling_x, subsampling_y, primary_strength,
-                             secondary_strength, damping, direction,
-                             cdef_buffer, cdef_stride);
-          }
-        }
+#endif  // LIBGAV1_MAX_BITDEPTH >= 10
+      ApplyCdefForOneUnit<uint8_t>(cdef_block_, index, block_width4x4,
+                                   block_height4x4, row4x4, column4x4);
+    }
+  }
+  if (!DoRestoration()) {
+    const int horizontal_shift = -source_buffer_->alignment() / pixel_size_;
+    const int vertical_shift = -kCdefBorder;
+    for (int plane = kPlaneY; plane < planes_; ++plane) {
+      if (!source_buffer_->ShiftBuffer(plane, horizontal_shift,
+                                       vertical_shift)) {
+        LIBGAV1_DLOG(ERROR,
+                     "Error shifting frame buffer head pointer at plane: %d",
+                     plane);
+        return false;
       }
     }
   }
@@ -492,16 +679,14 @@
   // Extend original frame, copy to borders.
   for (int plane = kPlaneY; plane < planes_; ++plane) {
     const int8_t subsampling_x = (plane == kPlaneY) ? 0 : subsampling_x_;
-    const int8_t subsampling_y = (plane == kPlaneY) ? 0 : subsampling_y_;
     uint8_t* const frame_start = input_buffer->data(plane);
     const int plane_width =
         RightShiftWithRounding(upscaled_width_, subsampling_x);
-    const int border_height = kBorderPixels >> subsampling_y;
-    const int border_width = kBorderPixels >> subsampling_x;
-    ExtendFrameBoundary(frame_start, plane_width,
-                        input_buffer->displayed_height(plane),
-                        input_buffer->stride(plane), border_width, border_width,
-                        border_height, border_height);
+    ExtendFrameBoundary(
+        frame_start, plane_width, input_buffer->displayed_height(plane),
+        input_buffer->stride(plane), input_buffer->left_border(plane),
+        input_buffer->right_border(plane), input_buffer->top_border(plane),
+        input_buffer->bottom_border(plane));
   }
 }
 
@@ -536,9 +721,7 @@
     const int x, const int y, const int row, const int unit_row,
     const int current_process_unit_height, const int process_unit_width,
     const int window_width, const int plane_unit_size,
-    const int num_horizontal_units, std::mutex* const mutex,
-    int* const pending_jobs,
-    std::condition_variable* const pending_jobs_zero_condvar) {
+    const int num_horizontal_units) {
   for (int column = 0; column < window_width; column += process_unit_width) {
     const int unit_x = x + column;
     const int unit_column =
@@ -555,17 +738,7 @@
         cdef_buffer, cdef_buffer_stride, deblock_buffer, deblock_buffer_stride,
         plane, plane_height, unit_id, type, x, y, row, column,
         current_process_unit_width, current_process_unit_height,
-        process_unit_width, kRestorationWindowWidth);
-  }
-  if (mutex != nullptr) {
-    assert(pending_jobs != nullptr);
-    assert(pending_jobs_zero_condvar != nullptr);
-    std::lock_guard<std::mutex> lock(*mutex);
-    if (--*pending_jobs == 0) {
-      // TODO(jzern): the mutex doesn't need to be locked to signal the
-      // condition.
-      pending_jobs_zero_condvar->notify_one();
-    }
+        process_unit_width, window_buffer_width_);
   }
 }
 
@@ -581,15 +754,14 @@
   const int unit_x = x + column;
   const int unit_y = y + row;
   uint8_t* cdef_unit_buffer =
-      cdef_buffer + unit_y * cdef_buffer_stride + unit_x * sizeof(Pixel);
+      cdef_buffer + unit_y * cdef_buffer_stride + unit_x * pixel_size_;
   Array2DView<Pixel> loop_restored_window(
-      restoration_window_height_, kRestorationWindowWidth,
-      reinterpret_cast<Pixel*>(threaded_loop_restoration_buffer_));
+      window_buffer_height_, window_buffer_width_,
+      reinterpret_cast<Pixel*>(threaded_window_buffer_));
   if (type == kLoopRestorationTypeNone) {
     Pixel* dest = &loop_restored_window[row][column];
     for (int k = 0; k < current_process_unit_height; ++k) {
-      memcpy(dest, cdef_unit_buffer,
-             current_process_unit_width * sizeof(Pixel));
+      memcpy(dest, cdef_unit_buffer, current_process_unit_width * pixel_size_);
       dest += window_width;
       cdef_unit_buffer += cdef_buffer_stride;
     }
@@ -605,7 +777,7 @@
                                sizeof(Pixel) +
                            ((sizeof(Pixel) == 1) ? 6 : 0)];
   const ptrdiff_t block_buffer_stride =
-      kRestorationProcessingUnitSizeWithBorders * sizeof(Pixel);
+      kRestorationProcessingUnitSizeWithBorders * pixel_size_;
   IntermediateBuffers intermediate_buffers;
 
   RestorationBuffer restoration_buffer = {
@@ -616,10 +788,9 @@
        intermediate_buffers.box_filter.intermediate_b},
       kRestorationProcessingUnitSizeWithBorders + kRestorationPadding,
       intermediate_buffers.wiener,
-      kMaxSuperBlockSizeInPixels,
-      {(bitdepth_ == 12) ? 5 : 3, (bitdepth_ == 12) ? 9 : 11}};
+      kMaxSuperBlockSizeInPixels};
   uint8_t* deblock_unit_buffer =
-      deblock_buffer + unit_y * deblock_buffer_stride + unit_x * sizeof(Pixel);
+      deblock_buffer + unit_y * deblock_buffer_stride + unit_x * pixel_size_;
   assert(type == kLoopRestorationTypeSgrProj ||
          type == kLoopRestorationTypeWiener);
   const dsp::LoopRestorationFunc restoration_func =
@@ -631,23 +802,23 @@
       unit_y + current_process_unit_height >= plane_height);
   restoration_func(reinterpret_cast<const uint8_t*>(
                        block_buffer + kRestorationBorder * block_buffer_stride +
-                       kRestorationBorder * sizeof(Pixel)),
+                       kRestorationBorder * pixel_size_),
                    &loop_restored_window[row][column],
                    restoration_info_->loop_restoration_info(
                        static_cast<Plane>(plane), unit_id),
-                   block_buffer_stride, window_width * sizeof(Pixel),
+                   block_buffer_stride, window_width * pixel_size_,
                    current_process_unit_width, current_process_unit_height,
                    &restoration_buffer);
 }
 
 // Multi-thread version of loop restoration, based on a moving window of size
-// |kRestorationWindowWidth|x|restoration_window_height_|. Inside the moving
-// window, we create a filtering job for each row and each filtering job is
-// submitted to the thread pool. Each free thread takes one job from the thread
-// pool and completes filtering until all jobs are finished. This approach
-// requires an extra buffer (|threaded_loop_restoration_buffer_|) to hold the
-// filtering output, whose size is the size of the window. It also needs block
-// buffers (i.e., |block_buffer| and |intermediate_buffers| in
+// |window_buffer_width_|x|window_buffer_height_|. Inside the moving window, we
+// create a filtering job for each row and each filtering job is submitted to
+// the thread pool. Each free thread takes one job from the thread pool and
+// completes filtering until all jobs are finished. This approach requires an
+// extra buffer (|threaded_window_buffer_|) to hold the filtering output, whose
+// size is the size of the window. It also needs block buffers (i.e.,
+// |block_buffer| and |intermediate_buffers| in
 // ApplyLoopRestorationForOneUnit()) to store intermediate results in loop
 // restoration for each thread. After all units inside the window are filtered,
 // the output is written to the frame buffer.
@@ -663,9 +834,7 @@
       kRestorationProcessingUnitSize >> subsampling_y_,
       kRestorationProcessingUnitSize >> subsampling_y_};
 
-  std::mutex mutex;
-  std::condition_variable pending_jobs_zero_condvar;
-  const int horizontal_shift = -source_buffer_->alignment() / sizeof(Pixel);
+  const int horizontal_shift = -source_buffer_->alignment() / pixel_size_;
   const int vertical_shift = -kRestorationBorder;
   for (int plane = kPlaneY; plane < planes_; ++plane) {
     if (loop_restoration_.type[plane] == kLoopRestorationTypeNone) {
@@ -696,7 +865,7 @@
         RightShiftWithRounding(upscaled_width_, subsampling_x);
     const int plane_height = RightShiftWithRounding(height_, subsampling_y);
     const ptrdiff_t src_unit_buffer_offset =
-        vertical_shift * src_stride + horizontal_shift * sizeof(Pixel);
+        vertical_shift * src_stride + horizontal_shift * pixel_size_;
     ExtendFrameBoundary(cdef_buffer, plane_width, plane_height,
                         cdef_buffer_stride, kRestorationBorder,
                         kRestorationBorder, kRestorationBorder,
@@ -709,10 +878,10 @@
     }
 
     const int num_workers = thread_pool_->num_threads();
-    for (int y = 0; y < plane_height; y += restoration_window_height_) {
-      const int actual_window_height = std::min(
-          restoration_window_height_ - ((y == 0) ? unit_height_offset : 0),
-          plane_height - y);
+    for (int y = 0; y < plane_height; y += window_buffer_height_) {
+      const int actual_window_height =
+          std::min(window_buffer_height_ - ((y == 0) ? unit_height_offset : 0),
+                   plane_height - y);
       int vertical_units_per_window =
           (actual_window_height + plane_process_unit_height[plane] - 1) /
           plane_process_unit_height[plane];
@@ -731,14 +900,13 @@
                 plane_process_unit_height[plane] +
             1;
       }
-      for (int x = 0; x < plane_width; x += kRestorationWindowWidth) {
+      for (int x = 0; x < plane_width; x += window_buffer_width_) {
         const int actual_window_width =
-            std::min(kRestorationWindowWidth, plane_width - x);
-        int pending_jobs = vertical_units_per_window;  // Guarded by |mutex|.
+            std::min(window_buffer_width_, plane_width - x);
         const int jobs_for_threadpool =
-            pending_jobs * num_workers / (num_workers + 1);
-        assert(jobs_for_threadpool < pending_jobs);
-        pending_jobs = jobs_for_threadpool;
+            vertical_units_per_window * num_workers / (num_workers + 1);
+        assert(jobs_for_threadpool < vertical_units_per_window);
+        BlockingCounter pending_jobs(jobs_for_threadpool);
         int job_count = 0;
         int current_process_unit_height;
         for (int row = 0; row < actual_window_height;
@@ -757,20 +925,19 @@
 
           if (job_count < jobs_for_threadpool) {
             thread_pool_->Schedule(
-                [this, &mutex, cdef_buffer, cdef_buffer_stride, deblock_buffer,
+                [this, cdef_buffer, cdef_buffer_stride, deblock_buffer,
                  deblock_buffer_stride, process_unit_width,
                  current_process_unit_height, actual_window_width,
                  plane_unit_size, num_horizontal_units, x, y, row, unit_row,
-                 plane_height, plane_width, plane, &pending_jobs,
-                 &pending_jobs_zero_condvar]() {
+                 plane_height, plane_width, plane, &pending_jobs]() {
                   ApplyLoopRestorationForOneRowInWindow<Pixel>(
                       cdef_buffer, cdef_buffer_stride, deblock_buffer,
                       deblock_buffer_stride, static_cast<Plane>(plane),
                       plane_height, plane_width, x, y, row, unit_row,
                       current_process_unit_height, process_unit_width,
                       actual_window_width, plane_unit_size,
-                      num_horizontal_units, &mutex, &pending_jobs,
-                      &pending_jobs_zero_condvar);
+                      num_horizontal_units);
+                  pending_jobs.Decrement();
                 });
           } else {
             ApplyLoopRestorationForOneRowInWindow<Pixel>(
@@ -778,22 +945,17 @@
                 deblock_buffer_stride, static_cast<Plane>(plane), plane_height,
                 plane_width, x, y, row, unit_row, current_process_unit_height,
                 process_unit_width, actual_window_width, plane_unit_size,
-                num_horizontal_units, nullptr, nullptr, nullptr);
+                num_horizontal_units);
           }
           ++job_count;
         }
-        {
-          // Wait for all jobs of current window to finish.
-          std::unique_lock<std::mutex> lock(mutex);
-          while (pending_jobs > 0) {
-            pending_jobs_zero_condvar.wait(lock);
-          }
-        }
-        // Copy |threaded_loop_restoration_buffer_| to output frame.
-        CopyPlane<Pixel>(threaded_loop_restoration_buffer_,
-                         kRestorationWindowWidth * sizeof(Pixel),
+        // Wait for all jobs of current window to finish.
+        pending_jobs.Wait();
+        // Copy |threaded_window_buffer_| to output frame.
+        CopyPlane<Pixel>(threaded_window_buffer_,
+                         window_buffer_width_ * pixel_size_,
                          actual_window_width, actual_window_height,
-                         src_buffer + y * src_stride + x * sizeof(Pixel) +
+                         src_buffer + y * src_stride + x * pixel_size_ +
                              src_unit_buffer_offset,
                          src_stride);
       }
@@ -811,7 +973,7 @@
 
 bool PostFilter::ApplyLoopRestoration() {
   if (thread_pool_ != nullptr) {
-    assert(threaded_loop_restoration_buffer_ != nullptr);
+    assert(threaded_window_buffer_ != nullptr);
 #if LIBGAV1_MAX_BITDEPTH >= 10
     if (bitdepth_ >= 10) {
       return ApplyLoopRestorationThreaded<uint16_t>();
@@ -821,10 +983,8 @@
   }
 
   if (!DoCdef()) cdef_buffer_ = source_buffer_;
-  const size_t pixel_size =
-      (bitdepth_ == 8) ? sizeof(uint8_t) : sizeof(uint16_t);
   const ptrdiff_t block_buffer_stride =
-      kRestorationProcessingUnitSizeWithBorders * pixel_size;
+      kRestorationProcessingUnitSizeWithBorders * pixel_size_;
   const int plane_process_unit_width[kMaxPlanes] = {
       kRestorationProcessingUnitSize,
       kRestorationProcessingUnitSize >> subsampling_x_,
@@ -833,8 +993,16 @@
       kRestorationProcessingUnitSize,
       kRestorationProcessingUnitSize >> subsampling_y_,
       kRestorationProcessingUnitSize >> subsampling_y_};
-  RestorationBuffer restoration_buffer;
-  PrepareRestorationBuffer(&restoration_buffer);
+  IntermediateBuffers intermediate_buffers;
+  RestorationBuffer restoration_buffer = {
+      {intermediate_buffers.box_filter.output[0],
+       intermediate_buffers.box_filter.output[1]},
+      plane_process_unit_width[kPlaneY],
+      {intermediate_buffers.box_filter.intermediate_a,
+       intermediate_buffers.box_filter.intermediate_b},
+      kRestorationProcessingUnitSizeWithBorders + kRestorationPadding,
+      intermediate_buffers.wiener,
+      kMaxSuperBlockSizeInPixels};
 
   for (int plane = kPlaneY; plane < planes_; ++plane) {
     if (loop_restoration_.type[plane] == kLoopRestorationTypeNone) {
@@ -887,10 +1055,10 @@
     }
 
     int loop_restored_rows = 0;
-    const int horizontal_shift = -source_buffer_->alignment() / pixel_size;
+    const int horizontal_shift = -source_buffer_->alignment() / pixel_size_;
     const int vertical_shift = -kRestorationBorder;
     const ptrdiff_t src_unit_buffer_offset =
-        vertical_shift * src_stride + horizontal_shift * pixel_size;
+        vertical_shift * src_stride + horizontal_shift * pixel_size_;
     for (int unit_row = 0; unit_row < num_vertical_units; ++unit_row) {
       int current_unit_height = plane_unit_size;
       // Note [1]: we need to identify the entire restoration area. So the
@@ -914,11 +1082,11 @@
                 ->loop_restoration_info(static_cast<Plane>(plane), unit_id)
                 .type;
         uint8_t* src_unit_buffer =
-            src_buffer + unit_column * plane_unit_size * pixel_size;
+            src_buffer + unit_column * plane_unit_size * pixel_size_;
         uint8_t* cdef_unit_buffer =
-            cdef_buffer + unit_column * plane_unit_size * pixel_size;
+            cdef_buffer + unit_column * plane_unit_size * pixel_size_;
         uint8_t* deblock_unit_buffer =
-            deblock_buffer + unit_column * plane_unit_size * pixel_size;
+            deblock_buffer + unit_column * plane_unit_size * pixel_size_;
 
         // Take care of the last column. The max width of last column unit
         // could be 3/2 unit_size.
@@ -930,7 +1098,7 @@
         if (type == kLoopRestorationTypeNone) {
           for (int y = 0; y < current_unit_height; ++y) {
             memcpy(src_unit_buffer + src_unit_buffer_offset, cdef_unit_buffer,
-                   current_unit_width * pixel_size);
+                   current_unit_width * pixel_size_);
             src_unit_buffer += src_stride;
             cdef_unit_buffer += cdef_buffer_stride;
           }
@@ -971,14 +1139,14 @@
             // To address this, we store the restored pixels not onto the start
             // of current block on the source frame buffer, say point A,
             // but to its top by three pixels and to the left by
-            // alignment/pixel_size pixels, say point B, such that
+            // alignment/pixel_size_ pixels, say point B, such that
             // next processing unit can fetch 3 pixel border of unrestored
             // values. And we need to adjust the input frame buffer pointer to
             // its left and top corner, point B.
             uint8_t* const cdef_process_unit_buffer =
-                cdef_unit_buffer + column * pixel_size;
+                cdef_unit_buffer + column * pixel_size_;
             uint8_t* const deblock_process_unit_buffer =
-                deblock_unit_buffer + column * pixel_size;
+                deblock_unit_buffer + column * pixel_size_;
             const bool frame_top_border = unit_row + row == 0;
             const bool frame_bottom_border =
                 (unit_row == num_vertical_units - 1) &&
@@ -1001,8 +1169,8 @@
             restoration_func(
                 reinterpret_cast<const uint8_t*>(
                     block_buffer_ + kRestorationBorder * block_buffer_stride +
-                    kRestorationBorder * pixel_size),
-                src_unit_buffer + column * pixel_size + src_unit_buffer_offset,
+                    kRestorationBorder * pixel_size_),
+                src_unit_buffer + column * pixel_size_ + src_unit_buffer_offset,
                 restoration_info_->loop_restoration_info(
                     static_cast<Plane>(plane), unit_id),
                 block_buffer_stride, src_stride, processing_unit_width,
@@ -1043,9 +1211,7 @@
   const int8_t subsampling_y = (plane == kPlaneY) ? 0 : subsampling_y_;
   const int row_step = 1 << subsampling_y;
   const int column_step = 1 << subsampling_x;
-  const size_t pixel_size =
-      (bitdepth_ == 8) ? sizeof(uint8_t) : sizeof(uint16_t);
-  const size_t src_step = 4 * pixel_size;
+  const size_t src_step = 4 * pixel_size_;
   const ptrdiff_t row_stride = MultiplyBy4(source_buffer_->stride(plane));
   const ptrdiff_t src_stride = source_buffer_->stride(plane);
   uint8_t* src = SetBufferOffset(source_buffer_, plane, row4x4_start,
@@ -1083,18 +1249,21 @@
     const int index = GetIndex(row);
     const int shift = GetShift(row, column);
     const int level_offset = LoopFilterMask::GetLevelOffset(row, column);
-    // TODO(chengchen): replace 0, 1, 2 to meaningful enum names.
-    // mask of current row. mask4x4 represents the vertical filter length for
+    // Mask of current row. mask4x4 represents the vertical filter length for
     // the current horizontal edge is 4, and we needs to apply 3-tap filtering.
     // Similarly, mask8x8 and mask16x16 represent filter lengths are 8 and 16.
     uint64_t mask4x4 =
-        (masks_->GetTop(unit_id, plane, 0 /*Tx4x4*/, index) >> shift) &
+        (masks_->GetTop(unit_id, plane, kLoopFilterTransformSizeId4x4, index) >>
+         shift) &
         single_row_mask;
     uint64_t mask8x8 =
-        (masks_->GetTop(unit_id, plane, 1 /*Tx8x8*/, index) >> shift) &
+        (masks_->GetTop(unit_id, plane, kLoopFilterTransformSizeId8x8, index) >>
+         shift) &
         single_row_mask;
     uint64_t mask16x16 =
-        (masks_->GetTop(unit_id, plane, 2 /*Tx16x16*/, index) >> shift) &
+        (masks_->GetTop(unit_id, plane, kLoopFilterTransformSizeId16x16,
+                        index) >>
+         shift) &
         single_row_mask;
     // mask4x4, mask8x8, mask16x16 are mutually exclusive.
     assert((mask4x4 & mask8x8) == 0 && (mask4x4 & mask16x16) == 0 &&
@@ -1133,7 +1302,6 @@
           if ((mask16x16 & two_block_mask) == two_block_mask) {
             edge_count = 2;
             // Apply filtering for two edges.
-            // TODO(chengchen): actual implementation to come.
             filter_func(src_row, src_stride, outer_thresh_0, inner_thresh_0,
                         hev_thresh_0);
             filter_func(src_row + src_step, src_stride, outer_thresh_1,
@@ -1152,7 +1320,6 @@
           if ((mask8x8 & two_block_mask) == two_block_mask) {
             edge_count = 2;
             // Apply filtering for two edges.
-            // TODO(chengchen): actual implementation to come.
             filter_func(src_row, src_stride, outer_thresh_0, inner_thresh_0,
                         hev_thresh_0);
             filter_func(src_row + src_step, src_stride, outer_thresh_1,
@@ -1170,7 +1337,6 @@
           if ((mask4x4 & two_block_mask) == two_block_mask) {
             edge_count = 2;
             // Apply filtering for two edges.
-            // TODO(chengchen): actual implementation to come.
             filter_func(src_row, src_stride, outer_thresh_0, inner_thresh_0,
                         hev_thresh_0);
             filter_func(src_row + src_step, src_stride, outer_thresh_1,
@@ -1189,7 +1355,7 @@
       mask16x16 >>= step;
       mask >>= step;
       column_offset += step;
-      src_row += MultiplyBy4(edge_count) * pixel_size;
+      src_row += MultiplyBy4(edge_count) * pixel_size_;
     }
     src += row_stride;
   }
@@ -1245,13 +1411,19 @@
     // the current vertical edge is 4, and we needs to apply 3-tap filtering.
     // Similarly, mask8x8 and mask16x16 represent filter lengths are 8 and 16.
     uint64_t mask4x4_0 =
-        (masks_->GetLeft(unit_id, plane, 0 /*Tx4x4*/, index) >> shift) &
+        (masks_->GetLeft(unit_id, plane, kLoopFilterTransformSizeId4x4,
+                         index) >>
+         shift) &
         single_row_mask;
     uint64_t mask8x8_0 =
-        (masks_->GetLeft(unit_id, plane, 1 /*Tx8x8*/, index) >> shift) &
+        (masks_->GetLeft(unit_id, plane, kLoopFilterTransformSizeId8x8,
+                         index) >>
+         shift) &
         single_row_mask;
     uint64_t mask16x16_0 =
-        (masks_->GetLeft(unit_id, plane, 2 /*Tx16x16*/, index) >> shift) &
+        (masks_->GetLeft(unit_id, plane, kLoopFilterTransformSizeId16x16,
+                         index) >>
+         shift) &
         single_row_mask;
     // mask4x4, mask8x8, mask16x16 are mutually exclusive.
     assert((mask4x4_0 & mask8x8_0) == 0 && (mask4x4_0 & mask16x16_0) == 0 &&
@@ -1260,15 +1432,18 @@
     // the corresponding SIMD function to apply filtering for two vertical
     // edges together.
     uint64_t mask4x4_1 =
-        (masks_->GetLeft(unit_id, plane, 0 /*Tx4x4*/, index_next) >>
+        (masks_->GetLeft(unit_id, plane, kLoopFilterTransformSizeId4x4,
+                         index_next) >>
          shift_next_row) &
         single_row_mask;
     uint64_t mask8x8_1 =
-        (masks_->GetLeft(unit_id, plane, 1 /*Tx8x8*/, index_next) >>
+        (masks_->GetLeft(unit_id, plane, kLoopFilterTransformSizeId8x8,
+                         index_next) >>
          shift_next_row) &
         single_row_mask;
     uint64_t mask16x16_1 =
-        (masks_->GetLeft(unit_id, plane, 2 /*Tx16x16*/, index_next) >>
+        (masks_->GetLeft(unit_id, plane, kLoopFilterTransformSizeId16x16,
+                         index_next) >>
          shift_next_row) &
         single_row_mask;
     // mask4x4, mask8x8, mask16x16 are mutually exclusive.
@@ -1309,8 +1484,6 @@
           const dsp::LoopFilterFunc filter_func = dsp_.loop_filters[size][type];
           if ((mask16x16_0 & mask16x16_1 & 1) != 0) {
             // Apply dual vertical edge filtering.
-            // TODO(chengchen): actual implementation to come, probably with
-            // functions to support filtering 4 edges.
             filter_func(src_row, src_stride, outer_thresh_0, inner_thresh_0,
                         hev_thresh_0);
             filter_func(src_row_next, src_stride, outer_thresh_1,
@@ -1330,8 +1503,6 @@
                                                : dsp::kLoopFilterSize6;
           const dsp::LoopFilterFunc filter_func = dsp_.loop_filters[size][type];
           if ((mask8x8_0 & mask8x8_1 & 1) != 0) {
-            // TODO(chengchen): actual implementation to come, probably with
-            // functions to support filtering 4 edges.
             filter_func(src_row, src_stride, outer_thresh_0, inner_thresh_0,
                         hev_thresh_0);
             filter_func(src_row_next, src_stride, outer_thresh_1,
@@ -1349,8 +1520,6 @@
           const dsp::LoopFilterSize size = dsp::kLoopFilterSize4;
           const dsp::LoopFilterFunc filter_func = dsp_.loop_filters[size][type];
           if ((mask4x4_0 & mask4x4_1 & 1) != 0) {
-            // TODO(chengchen): actual implementation to come, probably with
-            // functions to support filtering 4 edges.
             filter_func(src_row, src_stride, outer_thresh_0, inner_thresh_0,
                         hev_thresh_0);
             filter_func(src_row_next, src_stride, outer_thresh_1,
@@ -1404,22 +1573,4 @@
   *hev_thresh = hev_thresh_[level];
 }
 
-void PostFilter::PrepareRestorationBuffer(
-    RestorationBuffer* restoration_buffer) {
-  restoration_buffer->box_filter_process_output[0] =
-      intermediate_buffers_.box_filter.output[0];
-  restoration_buffer->box_filter_process_output[1] =
-      intermediate_buffers_.box_filter.output[1];
-  restoration_buffer->box_filter_process_intermediate[0] =
-      intermediate_buffers_.box_filter.intermediate_a;
-  restoration_buffer->box_filter_process_intermediate[1] =
-      intermediate_buffers_.box_filter.intermediate_b;
-  restoration_buffer->box_filter_process_intermediate_stride =
-      kRestorationProcessingUnitSizeWithBorders + kRestorationPadding;
-  restoration_buffer->wiener_buffer = intermediate_buffers_.wiener;
-  restoration_buffer->wiener_buffer_stride = kMaxSuperBlockSizeInPixels;
-  restoration_buffer->inter_round_bits[0] = bitdepth_ == 12 ? 5 : 3;
-  restoration_buffer->inter_round_bits[1] = bitdepth_ == 12 ? 9 : 11;
-}
-
 }  // namespace libgav1
diff --git a/libgav1/src/post_filter.h b/libgav1/src/post_filter.h
index 7120405..b6c9518 100644
--- a/libgav1/src/post_filter.h
+++ b/libgav1/src/post_filter.h
@@ -3,11 +3,11 @@
 
 #include <algorithm>
 #include <array>
-#include <condition_variable>  // NOLINT (unapproved c++11 header)
+#include <atomic>
 #include <cstddef>
 #include <cstdint>
 #include <cstring>
-#include <mutex>  // NOLINT (unapproved c++11 header)
+#include <type_traits>
 
 #include "src/dsp/common.h"
 #include "src/dsp/dsp.h"
@@ -18,6 +18,7 @@
 #include "src/utils/block_parameters_holder.h"
 #include "src/utils/common.h"
 #include "src/utils/constants.h"
+#include "src/utils/memory.h"
 #include "src/utils/threadpool.h"
 #include "src/yuv_buffer.h"
 
@@ -37,8 +38,7 @@
 // and restoration together to only filter frame buffer once.
 class PostFilter {
  public:
-  // Window width used for multi-threaded loop restoration.
-  static constexpr int kRestorationWindowWidth = 512;
+  static constexpr int kCdefLargeValue = 30000;
 
   // This class does not take ownership of the masks/restoration_info, but it
   // may change their values.
@@ -49,8 +49,7 @@
              BlockParametersHolder* block_parameters,
              YuvBuffer* const source_buffer, const dsp::Dsp* dsp,
              ThreadPool* const thread_pool,
-             uint8_t* const threaded_loop_restoration_buffer,
-             int do_post_filter_mask)
+             uint8_t* const threaded_window_buffer, int do_post_filter_mask)
       : frame_header_(frame_header),
         loop_restoration_(frame_header.loop_restoration),
         dsp_(*dsp),
@@ -65,16 +64,21 @@
         planes_(sequence_header.color_config.is_monochrome
                     ? kMaxPlanesMonochrome
                     : kMaxPlanes),
+        pixel_size_(static_cast<int>((bitdepth_ == 8) ? sizeof(uint8_t)
+                                                      : sizeof(uint16_t))),
         masks_(masks),
         cdef_index_(cdef_index),
-        threaded_loop_restoration_buffer_(threaded_loop_restoration_buffer),
+        threaded_window_buffer_(threaded_window_buffer),
         restoration_info_(restoration_info),
-        restoration_window_height_(
-            GetRestorationWindowHeight(thread_pool, frame_header)),
-        block_parameters_(block_parameters),
+        window_buffer_width_(GetWindowBufferWidth(thread_pool, frame_header)),
+        window_buffer_height_(GetWindowBufferHeight(thread_pool, frame_header)),
+        block_parameters_(*block_parameters),
         source_buffer_(source_buffer),
         do_post_filter_mask_(do_post_filter_mask),
-        thread_pool_(thread_pool) {}
+        thread_pool_(thread_pool) {
+    const int8_t zero_delta_lf[kFrameLfCount] = {};
+    ComputeDeblockFilterLevels(zero_delta_lf, deblock_filter_levels_);
+  }
 
   // non copyable/movable.
   PostFilter(const PostFilter&) = delete;
@@ -118,13 +122,15 @@
   //                Inputs are source_buffer_ and cdef_buffer_.
   //                Ouput is source_buffer_.
   bool ApplyFiltering();
-  bool DoCdef() const {
-    return (do_post_filter_mask_ & 0x02) != 0 &&
-           (frame_header_.cdef.bits > 0 ||
-            frame_header_.cdef.y_primary_strength[0] > 0 ||
-            frame_header_.cdef.y_secondary_strength[0] > 0 ||
-            frame_header_.cdef.uv_primary_strength[0] > 0 ||
-            frame_header_.cdef.uv_secondary_strength[0] > 0);
+  bool DoCdef() const { return DoCdef(frame_header_, do_post_filter_mask_); }
+  static bool DoCdef(const ObuFrameHeader& frame_header,
+                     int do_post_filter_mask) {
+    return (do_post_filter_mask & 0x02) != 0 &&
+           (frame_header.cdef.bits > 0 ||
+            frame_header.cdef.y_primary_strength[0] > 0 ||
+            frame_header.cdef.y_secondary_strength[0] > 0 ||
+            frame_header.cdef.uv_primary_strength[0] > 0 ||
+            frame_header.cdef.uv_secondary_strength[0] > 0);
   }
   // If filter levels for Y plane (0 for vertical, 1 for horizontal),
   // are all zero, deblock filter will not be applied.
@@ -137,6 +143,17 @@
   bool DoDeblock() const {
     return DoDeblock(frame_header_, do_post_filter_mask_);
   }
+  uint8_t GetZeroDeltaDeblockFilterLevel(int segment_id, int level_index,
+                                         ReferenceFrameType type,
+                                         int mode_id) const {
+    return deblock_filter_levels_[segment_id][level_index][type][mode_id];
+  }
+  // Computes the deblock filter levels using |delta_lf| and stores them in
+  // |deblock_filter_levels|.
+  void ComputeDeblockFilterLevels(
+      const int8_t delta_lf[kFrameLfCount],
+      uint8_t deblock_filter_levels[kMaxSegments][kFrameLfCount]
+                                   [kNumReferenceFrameTypes][2]) const;
   bool DoRestoration() const;
   // Returns true if loop restoration will be performed for the given parameters
   // and mask.
@@ -172,45 +189,82 @@
                            ptrdiff_t stride, int left, int right, int top,
                            int bottom);
 
-  // For multi-threaded loop restoration, window height is the minimum of the
-  // following two quantities:
+  static int GetWindowBufferWidth(const ThreadPool* const thread_pool,
+                                  const ObuFrameHeader& frame_header) {
+    return (thread_pool == nullptr) ? 0
+                                    : Align(frame_header.upscaled_width, 64);
+  }
+
+  // For multi-threaded cdef and loop restoration, window height is the minimum
+  // of the following two quantities:
   //  1) thread_count * 64
   //  2) frame_height rounded up to the nearest power of 64
-  // Where 64 is the block size for loop restoration.
-  static int GetRestorationWindowHeight(const ThreadPool* const thread_pool,
-                                        const ObuFrameHeader& frame_header) {
+  // Where 64 is the block size for cdef and loop restoration.
+  static int GetWindowBufferHeight(const ThreadPool* const thread_pool,
+                                   const ObuFrameHeader& frame_header) {
     if (thread_pool == nullptr) return 0;
     const int thread_count = 1 + thread_pool->num_threads();
     const int window_height = MultiplyBy64(thread_count);
-    const int adjusted_frame_height = (frame_header.height + 63) & ~63;
+    const int adjusted_frame_height = Align(frame_header.height, 64);
     return std::min(adjusted_frame_height, window_height);
   }
 
  private:
+  // The type of the HorizontalDeblockFilter and VerticalDeblockFilter member
+  // functions.
+  using DeblockFilter = void (PostFilter::*)(Plane plane, int row4x4_start,
+                                             int column4x4_start, int unit_id);
+  // Represents a job for a worker thread to apply the deblock filter.
+  struct DeblockFilterJob : public Allocable {
+    int plane;
+    int row4x4;
+    int row_unit;
+  };
   // Buffers for loop restoration intermediate results. Depending on the filter
   // type, only one member of the union is used.
   union IntermediateBuffers {
     // For Wiener filter.
     // The array |intermediate| in Section 7.17.4, the intermediate results
     // between the horizontal and vertical filters.
-    alignas(
-        32) uint16_t wiener[(kMaxSuperBlockSizeInPixels + kSubPixelTaps - 1) *
-                            kMaxSuperBlockSizeInPixels];
+    alignas(kMaxAlignment)
+        uint16_t wiener[(kMaxSuperBlockSizeInPixels + kSubPixelTaps - 1) *
+                        kMaxSuperBlockSizeInPixels];
     // For self-guided filter.
     struct {
       // The arrays flt0 and flt1 in Section 7.17.2, the outputs of the box
       // filter process in pass 0 and pass 1.
-      alignas(32) int32_t output[2][kMaxBoxFilterProcessOutputPixels];
+      alignas(
+          kMaxAlignment) int32_t output[2][kMaxBoxFilterProcessOutputPixels];
       // The 2d arrays A and B in Section 7.17.3, the intermediate results in
       // the box filter process. Reused for pass 0 and pass 1.
-      alignas(32) uint32_t intermediate_a[kBoxFilterProcessIntermediatePixels];
-      alignas(32) uint32_t intermediate_b[kBoxFilterProcessIntermediatePixels];
+      alignas(kMaxAlignment) uint32_t
+          intermediate_a[kBoxFilterProcessIntermediatePixels];
+      alignas(kMaxAlignment) uint32_t
+          intermediate_b[kBoxFilterProcessIntermediatePixels];
     } box_filter;
   };
 
   bool ApplyDeblockFilter();
+  void DeblockFilterWorker(const DeblockFilterJob* jobs, int num_jobs,
+                           std::atomic<int>* job_counter,
+                           DeblockFilter deblock_filter);
   bool ApplyDeblockFilterThreaded();
+
+  uint8_t* GetCdefBufferAndStride(int start_x, int start_y, int plane,
+                                  int subsampling_x, int subsampling_y,
+                                  int window_buffer_plane_size,
+                                  int vertical_shift, int horizontal_shift,
+                                  int* cdef_stride);
+  template <typename Pixel>
+  void ApplyCdefForOneUnit(uint16_t* cdef_block, int index, int block_width4x4,
+                           int block_height4x4, int row4x4_start,
+                           int column4x4_start);
+  template <typename Pixel>
+  void ApplyCdefForOneRowInWindow(int row, int column);
+  template <typename Pixel>
+  bool ApplyCdefThreaded();
   bool ApplyCdef();  // Sections 7.15 and 7.15.1.
+
   bool ApplySuperRes();
   // Note for ApplyLoopRestoration():
   // First, we must differentiate loop restoration processing unit from loop
@@ -252,8 +306,7 @@
       uint8_t* deblock_buffer, ptrdiff_t deblock_buffer_stride, Plane plane,
       int plane_height, int plane_width, int x, int y, int row, int unit_row,
       int current_process_unit_height, int process_unit_width, int window_width,
-      int plane_unit_size, int num_horizontal_units, std::mutex* mutex,
-      int* pending_jobs, std::condition_variable* pending_jobs_zero_condvar);
+      int plane_unit_size, int num_horizontal_units);
   template <typename Pixel>
   void ApplyLoopRestorationForOneUnit(
       uint8_t* cdef_buffer, ptrdiff_t cdef_buffer_stride,
@@ -273,10 +326,17 @@
                                int column4x4_start, int unit_id);
   void VerticalDeblockFilter(Plane plane, int row4x4_start, int column4x4_start,
                              int unit_id);
+  // HorizontalDeblockFilter and VerticalDeblockFilter must have the correct
+  // signature.
+  static_assert(std::is_same<decltype(&PostFilter::HorizontalDeblockFilter),
+                             DeblockFilter>::value,
+                "");
+  static_assert(std::is_same<decltype(&PostFilter::VerticalDeblockFilter),
+                             DeblockFilter>::value,
+                "");
   void InitDeblockFilterParams();  // Part of 7.14.4.
   void GetDeblockFilterParams(uint8_t level, int* outer_thresh,
                               int* inner_thresh, int* hev_thresh) const;
-  void PrepareRestorationBuffer(RestorationBuffer* restoration_buffer);
   // Applies super resolution and writes result to input_buffer.
   void FrameSuperRes(YuvBuffer* input_buffer);  // Section 7.16.
 
@@ -291,21 +351,31 @@
   const int8_t subsampling_x_;
   const int8_t subsampling_y_;
   const int8_t planes_;
+  const int pixel_size_;
   // This class does not take ownership of the masks/restoration_info, but it
   // could change their values.
   LoopFilterMask* const masks_;
   uint8_t inner_thresh_[kMaxLoopFilterValue + 1] = {};
   uint8_t outer_thresh_[kMaxLoopFilterValue + 1] = {};
   uint8_t hev_thresh_[kMaxLoopFilterValue + 1] = {};
+  // This stores the deblocking filter levels assuming that the delta is zero.
+  // This will be used by all superblocks whose delta is zero (without having to
+  // recompute them). The dimensions (in order) are: segment_id, level_index
+  // (based on plane and direction), reference_frame and mode_id.
+  uint8_t deblock_filter_levels_[kMaxSegments][kFrameLfCount]
+                                [kNumReferenceFrameTypes][2];
   const Array2D<int16_t>& cdef_index_;
-  // Pointer to the data buffer used for multi-threaded loop restoration. The
-  // size of this buffer must be at least kRestorationWindowWidth *
-  // |restoration_window_height_| * sizeof(Pixel). If |thread_pool_| is nullptr,
-  // then this buffer is not used and can be nullptr as well.
-  uint8_t* const threaded_loop_restoration_buffer_;
+  // Pointer to the data buffer used for multi-threaded cdef or loop
+  // restoration. The size of this buffer must be at least
+  // |window_buffer_width_| * |window_buffer_height_| * |pixel_size_|.
+  // Or |planes_| times that for multi-threaded cdef.
+  // If |thread_pool_| is nullptr, then this buffer is not used and can be
+  // nullptr as well.
+  uint8_t* const threaded_window_buffer_;
   LoopRestorationInfo* const restoration_info_;
-  const int restoration_window_height_;
-  BlockParametersHolder* const block_parameters_;
+  const int window_buffer_width_;
+  const int window_buffer_height_;
+  const BlockParametersHolder& block_parameters_;
   // Frame buffer to hold cdef filtered frame.
   YuvBuffer cdef_filtered_buffer_;
   // Frame buffer to hold the copy of the buffer to be upscaled,
@@ -323,7 +393,6 @@
 
   ThreadPool* const thread_pool_;
 
-  IntermediateBuffers intermediate_buffers_;
   // A small buffer to hold input source image block for loop restoration.
   // Its size is one processing unit size + borders.
   // Self-guided filter needs an extra one-pixel border.
@@ -333,6 +402,10 @@
       block_buffer_[kRestorationProcessingUnitSizeWithBorders *
                     kRestorationProcessingUnitSizeWithBorders *
                     sizeof(uint16_t)];
+  // A block buffer to hold the input that is converted to uint16_t before
+  // cdef filtering. Only used in single threaded case.
+  uint16_t cdef_block_[kRestorationProcessingUnitSizeWithBorders *
+                       kRestorationProcessingUnitSizeWithBorders * 3];
 
   template <int bitdepth, typename Pixel>
   friend class PostFilterSuperResTest;
@@ -409,6 +482,104 @@
   }
 }
 
+template <typename Pixel>
+void CopyRows(const Pixel* src, const ptrdiff_t src_stride,
+              const int block_width, const int unit_width,
+              const bool is_frame_top, const bool is_frame_bottom,
+              const bool is_frame_left, const bool is_frame_right,
+              const bool copy_top, const int num_rows, uint16_t* dst,
+              const ptrdiff_t dst_stride) {
+  if (is_frame_top || is_frame_bottom) {
+    if (is_frame_bottom) dst -= kCdefBorder;
+    for (int y = 0; y < num_rows; ++y) {
+      Memset(dst, PostFilter::kCdefLargeValue, unit_width + 2 * kCdefBorder);
+      dst += dst_stride;
+    }
+  } else {
+    if (copy_top) {
+      src -= kCdefBorder * src_stride;
+      dst += kCdefBorder;
+    }
+    for (int y = 0; y < num_rows; ++y) {
+      for (int x = -kCdefBorder; x < 0; ++x) {
+        dst[x] = is_frame_left ? PostFilter::kCdefLargeValue : src[x];
+      }
+      for (int x = 0; x < block_width; ++x) {
+        dst[x] = src[x];
+      }
+      for (int x = block_width; x < unit_width + kCdefBorder; ++x) {
+        dst[x] = is_frame_right ? PostFilter::kCdefLargeValue : src[x];
+      }
+      dst += dst_stride;
+      src += src_stride;
+    }
+  }
+}
+
+// This function prepares the input source block for cdef filtering.
+// The input source block contains a 12x12 block, with the inner 8x8 as the
+// desired filter region.
+// It pads the block if the 12x12 block includes out of frame pixels with
+// a large value.
+// This achieves the required behavior defined in section 5.11.52 of the spec.
+template <typename Pixel>
+void PrepareCdefBlock(const YuvBuffer* const source_buffer, const int planes,
+                      const int subsampling_x, const int subsampling_y,
+                      const int frame_width, const int frame_height,
+                      const int block_width4x4, const int block_height4x4,
+                      const int row_64x64, const int column_64x64,
+                      uint16_t* cdef_source, const ptrdiff_t cdef_stride) {
+  for (int plane = kPlaneY; plane < planes; ++plane) {
+    uint16_t* cdef_src =
+        cdef_source + plane * kRestorationProcessingUnitSizeWithBorders *
+                          kRestorationProcessingUnitSizeWithBorders;
+    const int plane_subsampling_x = (plane == kPlaneY) ? 0 : subsampling_x;
+    const int plane_subsampling_y = (plane == kPlaneY) ? 0 : subsampling_y;
+    const int start_x = MultiplyBy4(column_64x64) >> plane_subsampling_x;
+    const int start_y = MultiplyBy4(row_64x64) >> plane_subsampling_y;
+    const int plane_width =
+        RightShiftWithRounding(frame_width, plane_subsampling_x);
+    const int plane_height =
+        RightShiftWithRounding(frame_height, plane_subsampling_y);
+    const int block_width = MultiplyBy4(block_width4x4) >> plane_subsampling_x;
+    const int block_height =
+        MultiplyBy4(block_height4x4) >> plane_subsampling_y;
+    // unit_width, unit_height are the same as block_width, block_height unless
+    // it reaches the frame boundary, where block_width < 64 or
+    // block_height < 64. unit_width, unit_height guarantee we build blocks on
+    // a multiple of 8.
+    const int unit_width =
+        Align(block_width, (plane_subsampling_x > 0) ? 4 : 8);
+    const int unit_height =
+        Align(block_height, (plane_subsampling_y > 0) ? 4 : 8);
+    const bool is_frame_left = column_64x64 == 0;
+    const bool is_frame_right = start_x + block_width >= plane_width;
+    const bool is_frame_top = row_64x64 == 0;
+    const bool is_frame_bottom = start_y + block_height >= plane_height;
+    const int src_stride = source_buffer->stride(plane) / sizeof(Pixel);
+    const Pixel* src_buffer =
+        reinterpret_cast<const Pixel*>(source_buffer->data(plane)) +
+        start_y * src_stride + start_x;
+    // Copy to the top 2 rows.
+    CopyRows(src_buffer, src_stride, block_width, unit_width, is_frame_top,
+             false, is_frame_left, is_frame_right, true, kCdefBorder, cdef_src,
+             cdef_stride);
+    cdef_src += kCdefBorder * cdef_stride + kCdefBorder;
+
+    // Copy the body.
+    CopyRows(src_buffer, src_stride, block_width, unit_width, false, false,
+             is_frame_left, is_frame_right, false, block_height, cdef_src,
+             cdef_stride);
+    src_buffer += block_height * src_stride;
+    cdef_src += block_height * cdef_stride;
+
+    // Copy to bottom rows.
+    CopyRows(src_buffer, src_stride, block_width, unit_width, false,
+             is_frame_bottom, is_frame_left, is_frame_right, false,
+             kCdefBorder + unit_height - block_height, cdef_src, cdef_stride);
+  }
+}
+
 }  // namespace libgav1
 
 #endif  // LIBGAV1_SRC_POST_FILTER_H_
diff --git a/libgav1/src/prediction_mask.cc b/libgav1/src/prediction_mask.cc
index b77ba0c..8a2a63f 100644
--- a/libgav1/src/prediction_mask.cc
+++ b/libgav1/src/prediction_mask.cc
@@ -17,7 +17,7 @@
 namespace libgav1 {
 namespace {
 
-const int kWedgeDirectionTypes = 16;
+constexpr int kWedgeDirectionTypes = 16;
 
 enum kWedgeDirection : uint8_t {
   kWedgeHorizontal,
@@ -77,7 +77,7 @@
                                                {kWedgeOblique117, 2, 4},
                                                {kWedgeOblique117, 6, 4}}};
 
-const uint8_t kWedgeFlipSignLookup[9][16] = {
+constexpr uint8_t kWedgeFlipSignLookup[9][16] = {
     {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, 1},  // kBlock8x8
     {1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, 1},  // kBlock8x16
     {1, 1, 1, 1, 0, 1, 1, 1, 0, 1, 0, 1, 1, 1, 0, 1},  // kBlock8x32
@@ -89,19 +89,19 @@
     {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, 1},  // kBlock32x32
 };
 
-const uint8_t kWedgeMasterObliqueOdd[kWedgeMaskMasterSize] = {
+constexpr uint8_t kWedgeMasterObliqueOdd[kWedgeMaskMasterSize] = {
     0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,
     0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  1,  2,  6,  18,
     37, 53, 60, 63, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64,
     64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64};
 
-const uint8_t kWedgeMasterObliqueEven[kWedgeMaskMasterSize] = {
+constexpr uint8_t kWedgeMasterObliqueEven[kWedgeMaskMasterSize] = {
     0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,
     0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  1,  4,  11, 27,
     46, 58, 62, 63, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64,
     64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64};
 
-const uint8_t kWedgeMasterVertical[kWedgeMaskMasterSize] = {
+constexpr uint8_t kWedgeMasterVertical[kWedgeMaskMasterSize] = {
     0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,
     0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  2,  7,  21,
     43, 57, 62, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64,
@@ -140,10 +140,14 @@
                           const ptrdiff_t stride_1,
                           const uint16_t* prediction_2,
                           const ptrdiff_t stride_2, const int bitdepth,
-                          const bool mask_is_inverse,
-                          const int inter_post_round_bits, const int width,
+                          const bool mask_is_inverse, const int width,
                           const int height, uint8_t* mask,
                           const ptrdiff_t mask_stride) {
+#if LIBGAV1_MAX_BITDEPTH == 12
+  const int inter_post_round_bits = (bitdepth == 12) ? 2 : 4;
+#else
+  constexpr int inter_post_round_bits = 4;
+#endif
   for (int y = 0; y < height; ++y) {
     for (int x = 0; x < width; ++x) {
       const int rounding_bits = bitdepth - 8 + inter_post_round_bits;
@@ -232,10 +236,14 @@
   int block_size_index = 0;
   int wedge_masks_offset = 0;
   for (int size = kBlock8x8; size <= kBlock32x32; ++size) {
+    if (!kIsWedgeCompoundModeAllowed.Contains(size)) continue;
+
     const int width = kBlockWidthPixels[size];
     const int height = kBlockHeightPixels[size];
-    assert(width >= 8 && width <= 32);
-    if (height < 8 || height > 32) continue;
+    assert(width >= 8);
+    assert(width <= 32);
+    assert(height >= 8);
+    assert(height <= 32);
 
     const auto block_size = static_cast<BlockSize>(size);
     for (int index = 0; index < kWedgeDirectionTypes; ++index) {
@@ -280,12 +288,10 @@
                         const ptrdiff_t stride_1,
                         const uint16_t* const prediction_2,
                         const ptrdiff_t stride_2, const bool mask_is_inverse,
-                        const int inter_post_round_bits, const int width,
-                        const int height, const int bitdepth,
+                        const int width, const int height, const int bitdepth,
                         uint8_t* const mask, const ptrdiff_t mask_stride) {
   DifferenceWeightMask(prediction_1, stride_1, prediction_2, stride_2, bitdepth,
-                       mask_is_inverse, inter_post_round_bits, width, height,
-                       mask, mask_stride);
+                       mask_is_inverse, width, height, mask, mask_stride);
 }
 
 void GenerateInterIntraMask(const int mode, const int width, const int height,
diff --git a/libgav1/src/prediction_mask.h b/libgav1/src/prediction_mask.h
index 86b083d..ce6c5cc 100644
--- a/libgav1/src/prediction_mask.h
+++ b/libgav1/src/prediction_mask.h
@@ -4,10 +4,17 @@
 #include <cstddef>
 #include <cstdint>
 
+#include "src/utils/bit_mask_set.h"
 #include "src/utils/types.h"
 
 namespace libgav1 {
 
+constexpr BitMaskSet kIsWedgeCompoundModeAllowed(kBlock8x8, kBlock8x16,
+                                                 kBlock8x32, kBlock16x8,
+                                                 kBlock16x16, kBlock16x32,
+                                                 kBlock32x8, kBlock32x16,
+                                                 kBlock32x32);
+
 // This function generates wedge masks. It should be called only once for the
 // decoder. If the video is key frame only, we don't have to call this
 // function.
@@ -18,9 +25,8 @@
 // 7.11.3.12.
 void GenerateWeightMask(const uint16_t* prediction_1, ptrdiff_t stride_1,
                         const uint16_t* prediction_2, ptrdiff_t stride_2,
-                        bool mask_is_inverse, int inter_post_round_bits,
-                        int width, int height, int bitdepth, uint8_t* mask,
-                        ptrdiff_t mask_stride);
+                        bool mask_is_inverse, int width, int height,
+                        int bitdepth, uint8_t* mask, ptrdiff_t mask_stride);
 
 // 7.11.3.13.
 void GenerateInterIntraMask(int mode, int width, int height, uint8_t* mask,
diff --git a/libgav1/src/quantizer.h b/libgav1/src/quantizer.h
index 5478167..8e87ab0 100644
--- a/libgav1/src/quantizer.h
+++ b/libgav1/src/quantizer.h
@@ -10,7 +10,8 @@
 
 // Stores the quantization parameters of Section 5.9.12.
 struct QuantizerParameters {
-  int16_t base_index;
+  // base_index is in the range [0, 255].
+  uint8_t base_index;
   int8_t delta_dc[kMaxPlanes];
   // delta_ac[kPlaneY] is always 0.
   int8_t delta_ac[kMaxPlanes];
diff --git a/libgav1/src/reconstruction.cc b/libgav1/src/reconstruction.cc
index a113557..83968ed 100644
--- a/libgav1/src/reconstruction.cc
+++ b/libgav1/src/reconstruction.cc
@@ -38,9 +38,9 @@
 
 template <typename Residual, typename Pixel>
 void Reconstruct(const dsp::Dsp& dsp, TransformType tx_type,
-                 TransformSize tx_size, int8_t bitdepth, bool lossless,
-                 Residual* const buffer, int start_x, int start_y,
-                 Array2DView<Pixel>* frame, int16_t non_zero_coeff_count) {
+                 TransformSize tx_size, bool lossless, Residual* const buffer,
+                 int start_x, int start_y, Array2DView<Pixel>* frame,
+                 int16_t non_zero_coeff_count) {
   static_assert(sizeof(Residual) == 2 || sizeof(Residual) == 4, "");
   const int tx_width_log2 = kTransformWidthLog2[tx_size];
   const int tx_height_log2 = kTransformHeightLog2[tx_size];
@@ -54,8 +54,8 @@
       dsp.inverse_transforms[row_transform_size][row_transform];
   assert(row_transform_func != nullptr);
 
-  row_transform_func(tx_type, tx_size, bitdepth, buffer, start_x, start_y,
-                     frame, /*is_row=*/true, non_zero_coeff_count);
+  row_transform_func(tx_type, tx_size, buffer, start_x, start_y, frame,
+                     /*is_row=*/true, non_zero_coeff_count);
 
   // Column transform.
   const dsp::TransformSize1D column_transform_size =
@@ -66,19 +66,18 @@
       dsp.inverse_transforms[column_transform_size][column_transform];
   assert(column_transform_func != nullptr);
 
-  column_transform_func(tx_type, tx_size, bitdepth, buffer, start_x, start_y,
-                        frame, /*is_row=*/false, non_zero_coeff_count);
+  column_transform_func(tx_type, tx_size, buffer, start_x, start_y, frame,
+                        /*is_row=*/false, non_zero_coeff_count);
 }
 
 template void Reconstruct(const dsp::Dsp& dsp, TransformType tx_type,
-                          TransformSize tx_size, int8_t bitdepth, bool lossless,
-                          int16_t* buffer, int start_x, int start_y,
-                          Array2DView<uint8_t>* frame,
+                          TransformSize tx_size, bool lossless, int16_t* buffer,
+                          int start_x, int start_y, Array2DView<uint8_t>* frame,
                           int16_t non_zero_coeff_count);
 #if LIBGAV1_MAX_BITDEPTH >= 10
 template void Reconstruct(const dsp::Dsp& dsp, TransformType tx_type,
-                          TransformSize tx_size, int8_t bitdepth, bool lossless,
-                          int32_t* buffer, int start_x, int start_y,
+                          TransformSize tx_size, bool lossless, int32_t* buffer,
+                          int start_x, int start_y,
                           Array2DView<uint16_t>* frame,
                           int16_t non_zero_coeff_count);
 #endif
diff --git a/libgav1/src/reconstruction.h b/libgav1/src/reconstruction.h
index 8b4849e..e73f567 100644
--- a/libgav1/src/reconstruction.h
+++ b/libgav1/src/reconstruction.h
@@ -17,20 +17,20 @@
 // transform block size |tx_size| starting at position |start_x| and |start_y|.
 template <typename Residual, typename Pixel>
 void Reconstruct(const dsp::Dsp& dsp, TransformType tx_type,
-                 TransformSize tx_size, int8_t bitdepth, bool lossless,
-                 Residual* buffer, int start_x, int start_y,
-                 Array2DView<Pixel>* frame, int16_t non_zero_coeff_count);
+                 TransformSize tx_size, bool lossless, Residual* buffer,
+                 int start_x, int start_y, Array2DView<Pixel>* frame,
+                 int16_t non_zero_coeff_count);
 
 extern template void Reconstruct(const dsp::Dsp& dsp, TransformType tx_type,
-                                 TransformSize tx_size, int8_t bitdepth,
-                                 bool lossless, int16_t* buffer, int start_x,
-                                 int start_y, Array2DView<uint8_t>* frame,
+                                 TransformSize tx_size, bool lossless,
+                                 int16_t* buffer, int start_x, int start_y,
+                                 Array2DView<uint8_t>* frame,
                                  int16_t non_zero_coeff_count);
 #if LIBGAV1_MAX_BITDEPTH >= 10
 extern template void Reconstruct(const dsp::Dsp& dsp, TransformType tx_type,
-                                 TransformSize tx_size, int8_t bitdepth,
-                                 bool lossless, int32_t* buffer, int start_x,
-                                 int start_y, Array2DView<uint16_t>* frame,
+                                 TransformSize tx_size, bool lossless,
+                                 int32_t* buffer, int start_x, int start_y,
+                                 Array2DView<uint16_t>* frame,
                                  int16_t non_zero_coeff_count);
 #endif
 
diff --git a/libgav1/src/residual_buffer_pool.cc b/libgav1/src/residual_buffer_pool.cc
index 3e16a4b..c21bb9a 100644
--- a/libgav1/src/residual_buffer_pool.cc
+++ b/libgav1/src/residual_buffer_pool.cc
@@ -1,7 +1,6 @@
 #include "src/residual_buffer_pool.h"
 
-#include <algorithm>
-#include <new>
+#include <mutex>  // NOLINT (unapproved c++11 header)
 #include <utility>
 
 namespace libgav1 {
@@ -37,6 +36,36 @@
 
 }  // namespace
 
+ResidualBufferStack::~ResidualBufferStack() {
+  while (top_ != nullptr) {
+    ResidualBuffer* top = top_;
+    top_ = top_->next_;
+    delete top;
+  }
+}
+
+void ResidualBufferStack::Push(std::unique_ptr<ResidualBuffer> buffer) {
+  buffer->next_ = top_;
+  top_ = buffer.release();
+  ++num_buffers_;
+}
+
+std::unique_ptr<ResidualBuffer> ResidualBufferStack::Pop() {
+  std::unique_ptr<ResidualBuffer> top;
+  if (top_ != nullptr) {
+    top.reset(top_);
+    top_ = top_->next_;
+    top->next_ = nullptr;
+    --num_buffers_;
+  }
+  return top;
+}
+
+void ResidualBufferStack::Swap(ResidualBufferStack* other) {
+  std::swap(top_, other->top_);
+  std::swap(num_buffers_, other->num_buffers_);
+}
+
 ResidualBufferPool::ResidualBufferPool(bool use_128x128_superblock,
                                        int subsampling_x, int subsampling_y,
                                        size_t residual_size)
@@ -61,36 +90,39 @@
   queue_size_ = queue_size;
   // The existing buffers (if any) are no longer valid since the buffer size or
   // the queue size has changed. Clear the stack.
-  std::lock_guard<std::mutex> lock(mutex_);
-  while (!buffers_.empty()) {
-    buffers_.pop();
+  ResidualBufferStack buffers;
+  {
+    std::lock_guard<std::mutex> lock(mutex_);
+    // Move the buffers in the stack to the local variable |buffers| and clear
+    // the stack.
+    buffers.Swap(&buffers_);
+    // Release mutex_ before freeing the buffers.
   }
+  // As the local variable |buffers| goes out of scope, its destructor frees
+  // the buffers that were in the stack.
 }
 
 std::unique_ptr<ResidualBuffer> ResidualBufferPool::Get() {
   std::unique_ptr<ResidualBuffer> buffer = nullptr;
   {
     std::lock_guard<std::mutex> lock(mutex_);
-    if (!buffers_.empty()) {
-      buffer = std::move(buffers_.top());
-      buffers_.pop();
-    }
+    buffer = buffers_.Pop();
   }
   if (buffer == nullptr) {
-    buffer.reset(new (std::nothrow) ResidualBuffer(buffer_size_, queue_size_));
+    buffer = ResidualBuffer::Create(buffer_size_, queue_size_);
   }
   return buffer;
 }
 
 void ResidualBufferPool::Release(std::unique_ptr<ResidualBuffer> buffer) {
-  buffer->transform_parameters.Reset();
+  buffer->transform_parameters()->Reset();
   std::lock_guard<std::mutex> lock(mutex_);
-  buffers_.push(std::move(buffer));
+  buffers_.Push(std::move(buffer));
 }
 
 size_t ResidualBufferPool::Size() const {
   std::lock_guard<std::mutex> lock(mutex_);
-  return buffers_.size();
+  return buffers_.Size();
 }
 
 }  // namespace libgav1
diff --git a/libgav1/src/residual_buffer_pool.h b/libgav1/src/residual_buffer_pool.h
index a2751bd..ab9c1e6 100644
--- a/libgav1/src/residual_buffer_pool.h
+++ b/libgav1/src/residual_buffer_pool.h
@@ -6,7 +6,6 @@
 #include <memory>
 #include <mutex>  // NOLINT (unapproved c++11 header)
 #include <new>
-#include <stack>
 
 #include "src/utils/common.h"
 #include "src/utils/compiler_attributes.h"
@@ -21,17 +20,21 @@
 // boundary checks since we always push data into the queue before accessing it.
 class TransformParameterQueue {
  public:
-  explicit TransformParameterQueue(int max_size) : max_size_(max_size) {
-    // No initialization is necessary since the data will be always written to
-    // before being read.
-    non_zero_coeff_count_.reset(new (std::nothrow) int16_t[max_size_]);
-    tx_type_.reset(new (std::nothrow) TransformType[max_size_]);
-  }
+  TransformParameterQueue() = default;
 
   // Move only.
   TransformParameterQueue(TransformParameterQueue&& other) = default;
   TransformParameterQueue& operator=(TransformParameterQueue&& other) = default;
 
+  LIBGAV1_MUST_USE_RESULT bool Init(int max_size) {
+    max_size_ = max_size;
+    // No initialization is necessary since the data will be always written to
+    // before being read.
+    non_zero_coeff_count_.reset(new (std::nothrow) int16_t[max_size_]);
+    tx_type_.reset(new (std::nothrow) TransformType[max_size_]);
+    return non_zero_coeff_count_ != nullptr && tx_type_ != nullptr;
+  }
+
   // Adds the |non_zero_coeff_count| and the |tx_type| to the back of the queue.
   void Push(int16_t non_zero_coeff_count, TransformType tx_type) {
     assert(back_ < max_size_);
@@ -65,22 +68,31 @@
   }
 
   // Used only in the tests. Returns the number of elements in the queue.
-  int Size() { return back_ - front_; }
+  int Size() const { return back_ - front_; }
 
  private:
-  const int max_size_;
+  int max_size_ = 0;
   std::unique_ptr<int16_t[]> non_zero_coeff_count_;
   std::unique_ptr<TransformType[]> tx_type_;
   int front_ = 0;
   int back_ = 0;
 };
 
-// This struct is used for parsing and decoding a superblock. Members of this
-// struct are populated in the "parse" step and consumed in the "decode" step.
-struct ResidualBuffer : public Allocable {
-  ResidualBuffer(size_t buffer_size, int queue_size)
-      : transform_parameters(queue_size) {
-    buffer = MakeAlignedUniquePtr<uint8_t>(32, buffer_size);
+// This class is used for parsing and decoding a superblock. Members of this
+// class are populated in the "parse" step and consumed in the "decode" step.
+class ResidualBuffer : public Allocable {
+ public:
+  static std::unique_ptr<ResidualBuffer> Create(size_t buffer_size,
+                                                int queue_size) {
+    std::unique_ptr<ResidualBuffer> buffer(new (std::nothrow) ResidualBuffer);
+    if (buffer != nullptr) {
+      buffer->buffer_ = MakeAlignedUniquePtr<uint8_t>(32, buffer_size);
+      if (buffer->buffer_ == nullptr ||
+          !buffer->transform_parameters_.Init(queue_size)) {
+        buffer = nullptr;
+      }
+    }
+    return buffer;
   }
 
   // Move only.
@@ -88,9 +100,52 @@
   ResidualBuffer& operator=(ResidualBuffer&& other) = default;
 
   // Buffer used to store the residual values.
-  AlignedUniquePtr<uint8_t> buffer;
+  uint8_t* buffer() { return buffer_.get(); }
   // Queue used to store the transform parameters.
-  TransformParameterQueue transform_parameters;
+  TransformParameterQueue* transform_parameters() {
+    return &transform_parameters_;
+  }
+
+ private:
+  friend class ResidualBufferStack;
+
+  ResidualBuffer() = default;
+
+  AlignedUniquePtr<uint8_t> buffer_;
+  TransformParameterQueue transform_parameters_;
+  // Used by ResidualBufferStack to form a chain of ResidualBuffers.
+  ResidualBuffer* next_ = nullptr;
+};
+
+// A LIFO stack of ResidualBuffers. Owns the buffers in the stack.
+class ResidualBufferStack {
+ public:
+  ResidualBufferStack() = default;
+
+  // Not copyable or movable
+  ResidualBufferStack(const ResidualBufferStack&) = delete;
+  ResidualBufferStack& operator=(const ResidualBufferStack&) = delete;
+
+  ~ResidualBufferStack();
+
+  // Pushes |buffer| to the top of the stack.
+  void Push(std::unique_ptr<ResidualBuffer> buffer);
+
+  // If the stack is non-empty, returns the buffer at the top of the stack and
+  // removes it from the stack. If the stack is empty, returns nullptr.
+  std::unique_ptr<ResidualBuffer> Pop();
+
+  // Swaps the contents of this stack and |other|.
+  void Swap(ResidualBufferStack* other);
+
+  // Returns the number of buffers in the stack.
+  size_t Size() const { return num_buffers_; }
+
+ private:
+  // A singly-linked list of ResidualBuffers, chained together using the next_
+  // field of ResidualBuffer.
+  ResidualBuffer* top_ = nullptr;
+  size_t num_buffers_ = 0;
 };
 
 // Utility class used to manage the residual buffers (and the transform
@@ -122,8 +177,7 @@
 
  private:
   mutable std::mutex mutex_;
-  std::stack<std::unique_ptr<ResidualBuffer>> buffers_
-      LIBGAV1_GUARDED_BY(mutex_);
+  ResidualBufferStack buffers_ LIBGAV1_GUARDED_BY(mutex_);
   size_t buffer_size_;
   int queue_size_;
 };
diff --git a/libgav1/src/utils/scan.cc b/libgav1/src/scan_tables.inc
similarity index 92%
rename from libgav1/src/utils/scan.cc
rename to libgav1/src/scan_tables.inc
index 3cc096d..3caacde 100644
--- a/libgav1/src/utils/scan.cc
+++ b/libgav1/src/scan_tables.inc
@@ -1,10 +1,4 @@
-#include "src/utils/scan.h"
-
-#include "src/utils/common.h"
-#include "src/utils/constants.h"
-
-namespace libgav1 {
-namespace {
+// This file contains all the scan order tables.
 
 const uint16_t kDefaultScan4x4[16] = {0, 1,  4,  8,  5, 2,  3,  6,
                                       9, 12, 13, 10, 7, 11, 14, 15};
@@ -409,38 +403,24 @@
     95,  251, 220, 189, 158, 127, 252, 221, 190, 159, 253, 222, 191, 254, 223,
     255};
 
-const uint16_t* kDefaultScan[kNumTransformSizes] = {
-    kDefaultScan4x4,   kDefaultScan4x8,   kDefaultScan4x16,  kDefaultScan8x4,
-    kDefaultScan8x8,   kDefaultScan8x16,  kDefaultScan8x32,  kDefaultScan16x4,
-    kDefaultScan16x8,  kDefaultScan16x16, kDefaultScan16x32, kDefaultScan32x32,
-    kDefaultScan32x8,  kDefaultScan32x16, kDefaultScan32x32, kDefaultScan32x32,
-    kDefaultScan32x32, kDefaultScan32x32, kDefaultScan32x32};
-
-const uint16_t* kColumnScan[kNumTransformSizes] = {
-    kColumnScan4x4,  kColumnScan4x8,   kColumnScan4x16, kColumnScan8x4,
-    kColumnScan8x8,  kColumnScan8x16,  kColumnScan16x4, kColumnScan16x4,
-    kColumnScan16x8, kColumnScan16x16, kColumnScan16x4, kColumnScan16x4,
-    kColumnScan16x4, kColumnScan16x4,  kColumnScan16x4, kColumnScan16x4,
-    kColumnScan16x4, kColumnScan16x4,  kColumnScan16x4};
-
-const uint16_t* kRowScan[kNumTransformSizes] = {
-    kRowScan4x4,  kRowScan4x8,  kRowScan4x16, kRowScan8x4,  kRowScan8x8,
-    kRowScan8x16, kRowScan16x4, kRowScan16x4, kRowScan16x8, kRowScan16x16,
-    kRowScan16x4, kRowScan16x4, kRowScan16x4, kRowScan16x4, kRowScan16x4,
-    kRowScan16x4, kRowScan16x4, kRowScan16x4, kRowScan16x4};
-
-}  // namespace
-
-const uint16_t* GetScan(TransformSize tx_size, TransformType tx_type) {
-  if (tx_size == kTransformSize16x64) return kDefaultScan16x32;
-  if (tx_size == kTransformSize64x16) return kDefaultScan32x16;
-  if (kTransformSizeSquareMax[tx_size] == kTransformSize64x64) {
-    return kDefaultScan32x32;
-  }
-  const TransformClass tx_class = GetTransformClass(tx_type);
-  if (tx_class == kTransformClassVertical) return kRowScan[tx_size];
-  if (tx_class == kTransformClassHorizontal) return kColumnScan[tx_size];
-  return kDefaultScan[tx_size];
-}
-
-}  // namespace libgav1
+// 5.11.41 (implemented as a simple look up of transform class and transform
+// size).
+const uint16_t* kScan[3][kNumTransformSizes] = {
+    // kTransformClass2D
+    {kDefaultScan4x4, kDefaultScan4x8, kDefaultScan4x16, kDefaultScan8x4,
+     kDefaultScan8x8, kDefaultScan8x16, kDefaultScan8x32, kDefaultScan16x4,
+     kDefaultScan16x8, kDefaultScan16x16, kDefaultScan16x32, kDefaultScan16x32,
+     kDefaultScan32x8, kDefaultScan32x16, kDefaultScan32x32, kDefaultScan32x32,
+     kDefaultScan32x16, kDefaultScan32x32, kDefaultScan32x32},
+    // kTransformClassHorizontal
+    {kColumnScan4x4, kColumnScan4x8, kColumnScan4x16, kColumnScan8x4,
+     kColumnScan8x8, kColumnScan8x16, kColumnScan16x4, kColumnScan16x4,
+     kColumnScan16x8, kColumnScan16x16, kColumnScan16x4, kDefaultScan16x32,
+     kColumnScan16x4, kColumnScan16x4, kColumnScan16x4, kDefaultScan32x32,
+     kDefaultScan32x16, kDefaultScan32x32, kDefaultScan32x32},
+    // kTransformClassVertical
+    {kRowScan4x4, kRowScan4x8, kRowScan4x16, kRowScan8x4, kRowScan8x8,
+     kRowScan8x16, kRowScan16x4, kRowScan16x4, kRowScan16x8, kRowScan16x16,
+     kRowScan16x4, kDefaultScan16x32, kRowScan16x4, kRowScan16x4, kRowScan16x4,
+     kDefaultScan32x32, kDefaultScan32x16, kDefaultScan32x32,
+     kDefaultScan32x32}};
diff --git a/libgav1/src/symbol_decoder_context.cc b/libgav1/src/symbol_decoder_context.cc
index aa65be9..c7bef4a 100644
--- a/libgav1/src/symbol_decoder_context.cc
+++ b/libgav1/src/symbol_decoder_context.cc
@@ -10,7 +10,7 @@
 // Import all the constants in the anonymous namespace.
 #include "src/symbol_decoder_context_cdfs.inc"
 
-inline uint8_t GetQuantizerContext(int base_quantizer_index) {
+uint8_t GetQuantizerContext(int base_quantizer_index) {
   if (base_quantizer_index <= 20) return 0;
   if (base_quantizer_index <= 60) return 1;
   if (base_quantizer_index <= 120) return 2;
@@ -84,11 +84,6 @@
 
 }  // namespace
 
-const int kEobPtSymbolCount[7] = {kEobPt16SymbolCount,  kEobPt32SymbolCount,
-                                  kEobPt64SymbolCount,  kEobPt128SymbolCount,
-                                  kEobPt256SymbolCount, kEobPt512SymbolCount,
-                                  kEobPt1024SymbolCount};
-
 #define CDF_COPY(source, destination) \
   memcpy(destination, source, sizeof(source))
 
diff --git a/libgav1/src/symbol_decoder_context.h b/libgav1/src/symbol_decoder_context.h
index 6a44a3f..63f37ce 100644
--- a/libgav1/src/symbol_decoder_context.h
+++ b/libgav1/src/symbol_decoder_context.h
@@ -15,8 +15,7 @@
   kSkipContexts = 3,
   kSkipModeContexts = 3,
   kBooleanFieldCdfSize = 3,
-  kDeltaQSymbolCount = 4,
-  kDeltaLfSymbolCount = 4,
+  kDeltaSymbolCount = 4,  // Used for both delta_q and delta_lf.
   kIntraModeContexts = 5,
   kYModeContexts = 4,
   kAngleDeltaSymbolCount = 2 * kMaxAngleDelta + 1,
@@ -73,8 +72,6 @@
   kNumMvComponents = 2,
 };  // anonymous enum
 
-extern const int kEobPtSymbolCount[7];
-
 struct SymbolDecoderContext {
   SymbolDecoderContext() = default;
   explicit SymbolDecoderContext(int base_quantizer_index) {
@@ -104,9 +101,9 @@
                                        [kBooleanFieldCdfSize];
   uint16_t skip_cdf[kSkipContexts][kBooleanFieldCdfSize];
   uint16_t skip_mode_cdf[kSkipModeContexts][kBooleanFieldCdfSize];
-  uint16_t delta_q_cdf[kDeltaQSymbolCount + 1];
-  uint16_t delta_lf_cdf[kDeltaLfSymbolCount + 1];
-  uint16_t delta_lf_multi_cdf[kFrameLfCount][kDeltaLfSymbolCount + 1];
+  uint16_t delta_q_cdf[kDeltaSymbolCount + 1];
+  uint16_t delta_lf_cdf[kDeltaSymbolCount + 1];
+  uint16_t delta_lf_multi_cdf[kFrameLfCount][kDeltaSymbolCount + 1];
   uint16_t intra_block_copy_cdf[kBooleanFieldCdfSize];
   uint16_t intra_frame_y_mode_cdf[kIntraModeContexts][kIntraModeContexts]
                                  [kIntraPredictionModesY + 1];
diff --git a/libgav1/src/symbol_decoder_context_cdfs.inc b/libgav1/src/symbol_decoder_context_cdfs.inc
index 6c57ce8..9b720ff 100644
--- a/libgav1/src/symbol_decoder_context_cdfs.inc
+++ b/libgav1/src/symbol_decoder_context_cdfs.inc
@@ -47,7 +47,7 @@
     {147, 0, 0}, {12060, 0, 0}, {24641, 0, 0}};
 
 // This constant is also used for DeltaLf and DeltaLfMulti.
-const uint16_t kDefaultDeltaQCdf[kDeltaQSymbolCount + 1] = {4608, 648, 91, 0,
+const uint16_t kDefaultDeltaQCdf[kDeltaSymbolCount + 1] = {4608, 648, 91, 0,
                                                             0};
 
 const uint16_t kDefaultIntraBlockCopyCdf[kBooleanFieldCdfSize] = {2237, 0, 0};
diff --git a/libgav1/src/symbol_visibility.h b/libgav1/src/symbol_visibility.h
new file mode 100644
index 0000000..5c79687
--- /dev/null
+++ b/libgav1/src/symbol_visibility.h
@@ -0,0 +1,72 @@
+#ifndef LIBGAV1_SRC_SYMBOL_VISIBILITY_H_
+#define LIBGAV1_SRC_SYMBOL_VISIBILITY_H_
+
+// This module defines the LIBGAV1_PUBLIC macro. LIBGAV1_PUBLIC, when combined
+// with the flags -fvisibility=hidden and -fvisibility-inlines-hidden, restricts
+// symbol availability when users use the shared object form of libgav1. The
+// intent is to prevent exposure of libgav1 internals to users of the library,
+// and to avoid ABI compatibility problems that changes to libgav1 internals
+// would cause for users of the libgav1 shared object.
+//
+// Examples:
+//
+// This form makes a class and all of its members part of the public API:
+//
+// class LIBGAV1_PUBLIC A {
+//  public:
+//   A();
+//   ~A();
+//   void Foo();
+//   int Bar();
+// };
+//
+// A::A(), A::~A(), A::Foo(), and A::Bar() are all available to code linking to
+// the shared object when this form is used.
+//
+// This form exposes a single class method as part of the public API:
+//
+// class B {
+//  public:
+//   B();
+//   ~B();
+//   LIBGAV1_PUBLIC int Foo();
+// };
+//
+// In this examples only B::Foo() is available to the user of the shared object.
+//
+// Non-class member functions can also be exposed individually:
+//
+// LIBGAV1_PUBLIC void Bar();
+//
+// In this example Bar() would be available to users of the shared object.
+//
+// Much of the above information and more can be found at
+// https://gcc.gnu.org/wiki/Visibility
+
+#if !defined(LIBGAV1_PUBLIC)
+#if defined(_WIN32)
+#if defined(LIBGAV1_BUILDING_DLL) && LIBGAV1_BUILDING_DLL
+#if defined(__GNUC__)
+#define LIBGAV1_PUBLIC __attribute__((dllexport))
+#else
+#define LIBGAV1_PUBLIC __declspec(dllexport)
+#endif  // defined(__GNUC__)
+#elif defined(LIBGAV1_BUILDING_DLL)
+#ifdef __GNUC__
+#define LIBGAV1_PUBLIC __attribute__((dllimport))
+#else
+#define LIBGAV1_PUBLIC __declspec(dllimport)
+#endif  // defined(__GNUC__)
+#else
+#define LIBGAV1_PUBLIC
+#endif  // defined(LIBGAV1_BUILDING_DLL) && LIBGAV1_BUILDING_DLL
+#else
+#if defined(__GNUC__) && __GNUC__ >= 4
+#define LIBGAV1_PUBLIC __attribute__((visibility("default")))
+#else
+#define LIBGAV1_PUBLIC
+#endif
+#endif  // defined(_WIN32)
+#endif  // defined(LIBGAV1_PUBLIC)
+
+#endif  // LIBGAV1_SRC_SYMBOL_VISIBILITY_H_
diff --git a/libgav1/src/threading_strategy.cc b/libgav1/src/threading_strategy.cc
index 36d9802..440cad0 100644
--- a/libgav1/src/threading_strategy.cc
+++ b/libgav1/src/threading_strategy.cc
@@ -1,6 +1,7 @@
 #include "src/threading_strategy.h"
 
 #include <algorithm>
+#include <cassert>
 
 #include "src/utils/logging.h"
 
@@ -14,14 +15,15 @@
 
 bool ThreadingStrategy::Reset(const ObuFrameHeader& frame_header,
                               int thread_count) {
-  if (thread_count <= 1) {
+  assert(thread_count > 0);
+  if (thread_count == 1) {
     thread_pool_.reset(nullptr);
-    use_tile_threads_ = false;
+    tile_thread_count_ = 0;
     max_tile_index_for_row_threads_ = 0;
     return true;
   }
 
-  // We do work in the main thread, so it is sufficient to create
+  // We do work in the current thread, so it is sufficient to create
   // |thread_count|-1 threads in the threadpool.
   thread_count = std::min(thread_count - 1, kMaxThreads);
 
@@ -30,7 +32,7 @@
     if (thread_pool_ == nullptr) {
       LIBGAV1_DLOG(ERROR, "Failed to create a thread pool with %d threads.",
                    thread_count);
-      use_tile_threads_ = false;
+      tile_thread_count_ = 0;
       max_tile_index_for_row_threads_ = 0;
       return false;
     }
@@ -39,16 +41,44 @@
   // Prefer tile threads first (but only if there is more than one tile).
   const int tile_count = frame_header.tile_info.tile_count;
   if (tile_count > 1) {
-    use_tile_threads_ = true;
-    thread_count -= tile_count;
-    if (thread_count <= 0) {
+    // We want 1 + tile_thread_count_ <= tile_count because the current thread
+    // is also used to decode tiles. This is equivalent to
+    // tile_thread_count_ <= tile_count - 1.
+    tile_thread_count_ = std::min(thread_count, tile_count - 1);
+    thread_count -= tile_thread_count_;
+    if (thread_count == 0) {
       max_tile_index_for_row_threads_ = 0;
       return true;
     }
   } else {
-    use_tile_threads_ = false;
+    tile_thread_count_ = 0;
   }
 
+#if defined(__ANDROID__)
+  // Assign the remaining threads for each Tile. The heuristic used here is that
+  // we will assign two threads for each Tile. So for example, if |thread_count|
+  // is 2, for a stream with 2 tiles the first tile would get both the threads
+  // and the second tile would have row multi-threading turned off. This
+  // heuristic is based on the fact that row multi-threading is fast enough only
+  // when there are at least two threads to do the decoding (since one thread
+  // always does the parsing).
+  //
+  // This heuristic might stop working when SIMD optimizations make the decoding
+  // much faster and the parsing thread is only as fast as the decoding threads.
+  // So we will have to revisit this later to make sure that this is still
+  // optimal.
+  //
+  // Note that while this heuristic significantly improves performance on high
+  // end devices (like the Pixel 3), there are some performance regressions in
+  // some lower end devices (in some cases) and that needs to be revisited as we
+  // bring in more optimizations. Overall, the gains because of this heuristic
+  // seems to be much larger than the regressions.
+  for (int i = 0; i < tile_count; ++i) {
+    max_tile_index_for_row_threads_ = i + 1;
+    thread_count -= 2;
+    if (thread_count <= 0) break;
+  }
+#else   // !defined(__ANDROID__)
   // Assign the remaining threads to each Tile.
   for (int i = 0; i < tile_count; ++i) {
     const int count = thread_count / tile_count +
@@ -60,6 +90,7 @@
     }
     max_tile_index_for_row_threads_ = i + 1;
   }
+#endif  // defined(__ANDROID__)
   return true;
 }
 
diff --git a/libgav1/src/threading_strategy.h b/libgav1/src/threading_strategy.h
index eebd8b5..2745195 100644
--- a/libgav1/src/threading_strategy.h
+++ b/libgav1/src/threading_strategy.h
@@ -35,9 +35,11 @@
   // Returns a pointer to the ThreadPool that is to be used for Tile
   // multi-threading.
   ThreadPool* tile_thread_pool() const {
-    return use_tile_threads_ ? thread_pool_.get() : nullptr;
+    return (tile_thread_count_ != 0) ? thread_pool_.get() : nullptr;
   }
 
+  int tile_thread_count() const { return tile_thread_count_; }
+
   // Returns a pointer to the ThreadPool that is to be used within the Tile at
   // index |tile_index| for superblock row multi-threading.
   ThreadPool* row_thread_pool(int tile_index) const {
@@ -51,7 +53,7 @@
 
  private:
   std::unique_ptr<ThreadPool> thread_pool_;
-  bool use_tile_threads_;
+  int tile_thread_count_;
   int max_tile_index_for_row_threads_;
 };
 
diff --git a/libgav1/src/tile.h b/libgav1/src/tile.h
index 524229b..3895f5e 100644
--- a/libgav1/src/tile.h
+++ b/libgav1/src/tile.h
@@ -24,6 +24,7 @@
 #include "src/symbol_decoder_context.h"
 #include "src/utils/array_2d.h"
 #include "src/utils/block_parameters_holder.h"
+#include "src/utils/blocking_counter.h"
 #include "src/utils/common.h"
 #include "src/utils/compiler_attributes.h"
 #include "src/utils/constants.h"
@@ -64,7 +65,8 @@
        const SegmentationMap* prev_segment_ids, PostFilter* post_filter,
        BlockParametersHolder* block_parameters, Array2D<int16_t>* cdef_index,
        Array2D<TransformSize>* inter_transform_sizes, const dsp::Dsp* dsp,
-       ThreadPool* thread_pool, ResidualBufferPool* residual_buffer_pool);
+       ThreadPool* thread_pool, ResidualBufferPool* residual_buffer_pool,
+       BlockingCounterWithStatus* pending_tiles);
 
   // Move only.
   Tile(Tile&& tile) noexcept;
@@ -74,7 +76,7 @@
 
   struct Block;  // Defined after this class.
 
-  bool Decode();  // 5.11.2.
+  bool Decode(bool is_main_thread);  // 5.11.2.
   const ObuSequenceHeader& sequence_header() const { return sequence_header_; }
   const ObuFrameHeader& frame_header() const { return frame_header_; }
   const RefCountedBuffer& current_frame() const { return current_frame_; }
@@ -92,51 +94,42 @@
   int superblock_columns() const { return superblock_columns_; }
 
  private:
-  struct EntropyContext : public Allocable {
-    static const int kLeft = 0;
-    static const int kTop = 1;
-
-    EntropyContext() = default;
-    ~EntropyContext() = default;
-    EntropyContext(const EntropyContext&) = default;
-    EntropyContext& operator=(const EntropyContext&) = default;
-
-    uint8_t coefficient_level = 0;
-    uint8_t dc_category = 0;
-  };
-
-  // Stores the transform tree state when reading variable size transform trees.
+  // Stores the transform tree state when reading variable size transform trees
+  // and when applying the transform tree. When applying the transform tree,
+  // |depth| is not used.
   struct TransformTreeNode {
-    TransformTreeNode(int row4x4, int column4x4, TransformSize tx_size,
-                      int depth)
-        : row4x4(row4x4),
-          column4x4(column4x4),
-          tx_size(tx_size),
-          depth(depth) {}
+    // The default constructor is invoked by the Stack<TransformTreeNode, n>
+    // constructor. Stack<> does not use the default-constructed elements, so it
+    // is safe for the default constructor to not initialize the members.
+    TransformTreeNode() = default;
+    TransformTreeNode(int x, int y, TransformSize tx_size, int depth = -1)
+        : x(x), y(y), tx_size(tx_size), depth(depth) {}
 
-    int row4x4;
-    int column4x4;
+    int x;
+    int y;
     TransformSize tx_size;
     int depth;
   };
 
+  static constexpr int kBlockDecodedStride = 34;
   // Buffer to facilitate decoding a superblock. When |split_parse_and_decode_|
   // is true, each superblock that is being decoded will get its own instance of
   // this buffer.
   struct SuperBlockBuffer {
-    uint8_t prediction_mask[kMaxSuperBlockSizeInPixels *
-                            kMaxSuperBlockSizeInPixels];
-    // This stores the decoded state of every 4x4 block in a superblock. It has
-    // 1 row/column border on all 4 sides. The left and top borders are handled
-    // by the |BlockDecoded()| function. The bottom and right borders are
-    // included in the array itself (hence the 33x33 dimension instead of
-    // 32x32).
-    bool block_decoded[kMaxPlanes][33][33];
-    // Stores the thresholds for determining if top-right and bottom-left pixels
-    // are available. Equivalent to the sbWidth4 and sbHeight4 variables in
-    // section 5.11.3 of the spec.
-    int block_decoded_width_threshold;
-    int block_decoded_height_threshold;
+    // The size of mask is 128x128.
+    AlignedUniquePtr<uint8_t> prediction_mask;
+    // Buffer used for inter prediction process. The buffers have an alignment
+    // of 8 bytes when allocated.
+    AlignedUniquePtr<uint16_t> prediction_buffer[2];
+    size_t prediction_buffer_size[2] = {};
+    // Equivalent to BlockDecoded array in the spec. This stores the decoded
+    // state of every 4x4 block in a superblock. It has 1 row/column border on
+    // all 4 sides (hence the 34x34 dimension instead of 32x32). Note that the
+    // spec uses "-1" as an index to access the left and top borders. In the
+    // code, we treat the index (1, 1) as equivalent to the spec's (0, 0). So
+    // all accesses into this array will be offset by +1 when compared with the
+    // spec.
+    bool block_decoded[kMaxPlanes][kBlockDecodedStride][kBlockDecodedStride];
     // Buffer used for storing subsampled luma samples needed for CFL
     // prediction. This buffer is used to avoid repetition of the subsampling
     // for the V plane when it is already done for the U plane.
@@ -176,27 +169,28 @@
     std::condition_variable pending_jobs_zero_condvar;
   };
 
+  // Performs member initializations that may fail. Called by Decode().
+  LIBGAV1_MUST_USE_RESULT bool Init();
+
   // Entry point for multi-threaded decoding. This function performs the same
   // functionality as Decode(). The current thread does the "parse" step while
   // the worker threads do the "decode" step.
   bool ThreadedDecode();
 
   // Returns whether or not the prerequisites for decoding the superblock at
-  // |row_index| and |column_index| are satisfied. |threading_parameters.mutex|
-  // must be held when calling this function.
-  bool CanDecode(int row_index, int column_index,
-                 const Array2D<SuperBlockState>& sb_state);
+  // |row_index| and |column_index| are satisfied. |threading_.mutex| must be
+  // held when calling this function.
+  bool CanDecode(int row_index, int column_index) const;
 
   // This function is run by the worker threads when multi-threaded decoding is
   // enabled. Once a superblock is decoded, this function will set the
-  // corresponding |threading->sb_state| entry to kSuperBlockStateDecoded. On
-  // failure, |threading->abort| will be set to true. If at any point
-  // |threading->abort| becomes true, this function will return as early as it
+  // corresponding |threading_.sb_state| entry to kSuperBlockStateDecoded. On
+  // failure, |threading_.abort| will be set to true. If at any point
+  // |threading_.abort| becomes true, this function will return as early as it
   // can. If the decoding succeeds, this function will also schedule the
   // decoding jobs for the superblock to the bottom-left and the superblock to
   // the right of this superblock (if it is allowed).
-  void DecodeSuperBlock(int row_index, int column_index, int block_width4x4,
-                        ThreadingParameters* threading);
+  void DecodeSuperBlock(int row_index, int column_index, int block_width4x4);
 
   uint16_t* GetPartitionCdf(int row4x4, int column4x4, BlockSize block_size);
   bool ReadPartition(int row4x4, int column4x4, BlockSize block_size,
@@ -219,6 +213,8 @@
   // function in the spec is equivalent to ProcessBlock() in the code.
   bool DecodeBlock(ParameterTree* tree, SuperBlockBuffer* sb_buffer);
 
+  void ClearBlockDecoded(SuperBlockBuffer* sb_buffer, int row4x4,
+                         int column4x4);  // 5.11.3.
   bool ProcessSuperBlock(int row4x4, int column4x4, int block_width4x4,
                          SuperBlockBuffer* sb_buffer, ProcessingMode mode);
   void ResetLoopRestorationParams();
@@ -239,12 +235,14 @@
   void ReadSkip(const Block& block);            // 5.11.11.
   void ReadSkipMode(const Block& block);        // 5.11.10.
   void ReadCdef(const Block& block);            // 5.11.56.
-  // Returns the new value.
-  int ReadAndClipDelta(uint16_t* cdf, int symbol_count, int delta_small,
-                       int scale, int min_value, int max_value, int value);
+  // Returns the new value. |cdf| is an array of size kDeltaSymbolCount + 1.
+  int ReadAndClipDelta(uint16_t* cdf, int delta_small, int scale, int min_value,
+                       int max_value, int value);
   void ReadQuantizerIndexDelta(const Block& block);  // 5.11.12.
   void ReadLoopFilterDelta(const Block& block);      // 5.11.13.
-  void ComputeDeblockFilterLevel(const Block& block);
+  // Populates |BlockParameters::deblock_filter_level| for the given |block|
+  // using |deblock_filter_levels_|.
+  void PopulateDeblockFilterLevel(const Block& block);
   void ReadPredictionModeY(const Block& block, bool intra_y_mode);
   void ReadIntraAngleInfo(const Block& block,
                           PlaneType plane_type);  // 5.11.42 and 5.11.43.
@@ -267,18 +265,26 @@
                               bool intra_y_mode);  // 5.11.22.
   int GetUseCompoundReferenceContext(const Block& block);
   CompoundReferenceType ReadCompoundReferenceType(const Block& block);
-  int GetReferenceContext(const Block& block,
-                          const std::vector<ReferenceFrameType>& types1,
-                          const std::vector<ReferenceFrameType>& types2) const;
-  uint16_t* GetReferenceCdf(
-      const Block& block, bool is_single, bool is_backward, int index,
-      CompoundReferenceType type = kNumCompoundReferenceTypes);
+  // Calculates count0 by calling block.CountReferences() on the frame types
+  // from type0_start to type0_end, inclusive, and summing the results.
+  // Calculates count1 by calling block.CountReferences() on the frame types
+  // from type1_start to type1_end, inclusive, and summing the results.
+  // Compares count0 with count1 and returns 0, 1 or 2.
+  //
+  // See count_refs and ref_count_ctx in 8.3.2.
+  int GetReferenceContext(const Block& block, ReferenceFrameType type0_start,
+                          ReferenceFrameType type0_end,
+                          ReferenceFrameType type1_start,
+                          ReferenceFrameType type1_end) const;
+  template <bool is_single, bool is_backward, int index>
+  uint16_t* GetReferenceCdf(const Block& block, CompoundReferenceType type =
+                                                    kNumCompoundReferenceTypes);
   void ReadReferenceFrames(const Block& block);  // 5.11.25.
   void ReadInterPredictionModeY(const Block& block,
                                 const MvContexts& mode_contexts);
   void ReadRefMvIndex(const Block& block);
   void ReadInterIntraMode(const Block& block, bool is_compound);  // 5.11.28.
-  bool IsScaled(ReferenceFrameType type);  // Part of 5.11.27.
+  bool IsScaled(ReferenceFrameType type) const;  // Part of 5.11.27.
   void ReadMotionMode(const Block& block, bool is_compound);  // 5.11.27.
   uint16_t* GetIsExplicitCompoundTypeCdf(const Block& block);
   uint16_t* GetIsCompoundTypeAverageCdf(const Block& block);
@@ -300,7 +306,7 @@
                                  TransformSize tx_size);
   void DecodeTransformSize(const Block& block);  // 5.11.16.
   bool ComputePrediction(const Block& block);    // 5.11.33.
-  // |x4| and |y4| are the row and column positions of the 4x4 block. |w4| and
+  // |x4| and |y4| are the column and row positions of the 4x4 block. |w4| and
   // |h4| are the width and height in 4x4 units of |tx_size|.
   int GetTransformAllZeroContext(const Block& block, Plane plane,
                                  TransformSize tx_size, int x4, int y4, int w4,
@@ -324,21 +330,35 @@
   int GetCoeffBaseRangeContextVertical(int adjusted_tx_width_log2, int pos);
   int GetDcSignContext(int x4, int y4, int w4, int h4, Plane plane);
   void SetEntropyContexts(int x4, int y4, int w4, int h4, Plane plane,
-                          uint8_t coefficient_level, uint8_t dc_category);
-  void InterIntraPrediction(
+                          uint8_t coefficient_level, int8_t dc_category);
+  bool InterIntraPrediction(
       uint16_t* prediction[2], ptrdiff_t prediction_stride,
       const uint8_t* prediction_mask, ptrdiff_t prediction_mask_stride,
       const PredictionParameters& prediction_parameters, int prediction_width,
       int prediction_height, int subsampling_x, int subsampling_y,
-      uint8_t post_round_bits, uint8_t* dest,
+      uint8_t* dest,
       ptrdiff_t dest_stride);  // Part of section 7.11.3.1 in the spec.
-  void CompoundInterPrediction(
-      const Block& block, uint16_t* prediction[2], ptrdiff_t prediction_stride,
+  // Several prediction modes need a prediction mask:
+  // kCompoundPredictionTypeDiffWeighted, kCompoundPredictionTypeWedge,
+  // kCompoundPredictionTypeIntra. They are mutually exclusive. So the mask is
+  // allocated in each case. The mask only needs to be allocated for kPlaneY
+  // and then used for other planes.
+  LIBGAV1_MUST_USE_RESULT static bool AllocatePredictionMask(
+      SuperBlockBuffer* sb_buffer);
+  bool CompoundInterPrediction(
+      const Block& block, ptrdiff_t prediction_stride,
       ptrdiff_t prediction_mask_stride, int prediction_width,
       int prediction_height, Plane plane, int subsampling_x, int subsampling_y,
       int bitdepth, int candidate_row, int candidate_column, uint8_t* dest,
-      ptrdiff_t dest_stride,
-      uint8_t post_round_bits);  // Part of section 7.11.3.1 in the spec.
+      ptrdiff_t dest_stride);  // Part of section 7.11.3.1 in the spec.
+  GlobalMotion* GetWarpParams(const Block& block, Plane plane,
+                              int prediction_width, int prediction_height,
+                              const PredictionParameters& prediction_parameters,
+                              ReferenceFrameType reference_type,
+                              bool* is_local_valid,
+                              GlobalMotion* global_motion_params,
+                              GlobalMotion* local_warp_params)
+      const;  // Part of section 7.11.3.1 in the spec.
   bool InterPrediction(const Block& block, Plane plane, int x, int y,
                        int prediction_width, int prediction_height,
                        int candidate_row, int candidate_column,
@@ -369,24 +389,42 @@
                             const uint8_t* round_bits, bool is_compound,
                             bool is_inter_intra, uint8_t* dest,
                             ptrdiff_t dest_stride);  // 7.11.3.4.
-  bool BlockWarpProcess(const Block& block, Plane plane, int index, int width,
-                        int height, uint16_t* prediction,
-                        ptrdiff_t prediction_stride, GlobalMotion* warp_params,
-                        const uint8_t* round_bits, bool is_compound,
-                        bool is_inter_intra, uint8_t* dest,
+  bool BlockWarpProcess(const Block& block, Plane plane, int index,
+                        int block_start_x, int block_start_y, int width,
+                        int height, ptrdiff_t prediction_stride,
+                        GlobalMotion* warp_params, const uint8_t* round_bits,
+                        bool is_compound, bool is_inter_intra, uint8_t* dest,
                         ptrdiff_t dest_stride);  // 7.11.3.5.
   void ObmcBlockPrediction(const MotionVector& mv, Plane plane,
                            int reference_frame_index, int width, int height,
                            int x, int y, int candidate_row,
-                           int candidate_column, const uint8_t* mask,
-                           int blending_direction, const uint8_t* round_bits);
+                           int candidate_column,
+                           ObmcDirection blending_direction,
+                           const uint8_t* round_bits);
   void ObmcPrediction(const Block& block, Plane plane, int width, int height,
                       const uint8_t* round_bits);  // 7.11.3.9.
-  void DistanceWeightedPrediction(
-      uint16_t* prediction_0, ptrdiff_t prediction_stride_0,
-      uint16_t* prediction_1, ptrdiff_t prediction_stride_1, int width,
-      int height, int candidate_row, int candidate_column, uint8_t* dest,
-      ptrdiff_t dest_stride, uint8_t post_round_bits);  // 7.11.3.15.
+  void DistanceWeightedPrediction(uint16_t* prediction_0,
+                                  ptrdiff_t prediction_stride_0,
+                                  uint16_t* prediction_1,
+                                  ptrdiff_t prediction_stride_1, int width,
+                                  int height, int candidate_row,
+                                  int candidate_column, uint8_t* dest,
+                                  ptrdiff_t dest_stride);  // 7.11.3.15.
+  // This function specializes the parsing of DC coefficient by removing some of
+  // the branches when i == 0 (since scan[0] is always 0 and scan[i] is always
+  // non-zero for all other possible values of i). |dc_category| is an output
+  // parameter that is populated when |is_dc_coefficient| is true.
+  // |coefficient_level| is an output parameter which accumulates the
+  // coefficient level.
+  template <bool is_dc_coefficient>
+  bool ReadSignAndApplyDequantization(
+      const Block& block, const uint16_t* scan, int i,
+      int adjusted_tx_width_log2, int tx_width, int q_value,
+      const uint8_t* quantizer_matrix, int shift, int min_value, int max_value,
+      uint16_t* dc_sign_cdf, int8_t* dc_category,
+      int* coefficient_level);  // Part of 5.11.39.
+  int ReadCoeffBaseRange(int clamped_tx_size_context, int cdf_context,
+                         int plane_type);  // Part of 5.11.39.
   // Returns the number of non-zero coefficients that were read. |tx_type| is an
   // output parameter that stores the computed transform type for the plane
   // whose coefficients were read. Returns -1 on failure.
@@ -397,8 +435,9 @@
   bool TransformBlock(const Block& block, Plane plane, int base_x, int base_y,
                       TransformSize tx_size, int x, int y,
                       ProcessingMode mode);  // 5.11.35.
-  bool TransformTree(const Block& block, int start_x, int start_y, int width,
-                     int height, ProcessingMode mode);  // 5.11.36.
+  // Iterative implementation of 5.11.36.
+  bool TransformTree(const Block& block, int start_x, int start_y,
+                     BlockSize plane_size, ProcessingMode mode);
   void ReconstructBlock(const Block& block, Plane plane, int start_x,
                         int start_y, TransformSize tx_size,
                         TransformType tx_type,
@@ -406,21 +445,19 @@
   bool Residual(const Block& block, ProcessingMode mode);  // 5.11.34.
   // part of 5.11.5 (reset_block_context() in the spec).
   void ResetEntropyContext(const Block& block);
-  int GetPaletteColorContext(const Block& block, PlaneType plane_type, int row,
-                             int column, int palette_size,
-                             uint8_t color_order[kMaxPaletteSize]);  // 5.11.50.
-  void ReadPaletteTokens(const Block& block);                        // 5.11.49.
-  // Helper function for handling the border cases in 5.11.3 of the spec.
-  // Early return if has_top_or_left is false, since has_bottom_left
-  // (has_top_right) must be false if has_left(has_top) is false.
-  bool BlockDecoded(const Block& block, Plane plane, int row4x4, int column4x4,
-                    bool has_top_or_left) const;
+  // Populates the |color_context| and |color_order| for the |i|th iteration
+  // with entries counting down from |start| to |end| (|start| > |end|).
+  void PopulatePaletteColorContexts(
+      const Block& block, PlaneType plane_type, int i, int start, int end,
+      uint8_t color_order[kMaxPaletteSquare][kMaxPaletteSize],
+      uint8_t color_context[kMaxPaletteSquare]);                 // 5.11.50.
+  bool ReadPaletteTokens(const Block& block);                    // 5.11.49.
   template <typename Pixel>
   void IntraPrediction(const Block& block, Plane plane, int x, int y,
                        bool has_left, bool has_top, bool has_top_right,
                        bool has_bottom_left, PredictionMode mode,
                        TransformSize tx_size);
-  bool UsesSmoothPrediction(int row, int column, Plane plane) const;
+  bool IsSmoothPrediction(int row, int column, Plane plane) const;
   int GetIntraEdgeFilterType(const Block& block,
                              Plane plane) const;  // 7.11.2.8.
   template <typename Pixel>
@@ -449,12 +486,6 @@
     return sequence_header_.color_config.is_monochrome ? kMaxPlanesMonochrome
                                                        : kMaxPlanes;
   }
-  int SubsamplingX(Plane plane) const {
-    return (plane > kPlaneY) ? sequence_header_.color_config.subsampling_x : 0;
-  }
-  int SubsamplingY(Plane plane) const {
-    return (plane > kPlaneY) ? sequence_header_.color_config.subsampling_y : 0;
-  }
 
   const int number_;
   int row_;
@@ -468,11 +499,46 @@
   int superblock_rows_;
   int superblock_columns_;
   bool read_deltas_;
+  const int8_t subsampling_x_[kMaxPlanes];
+  const int8_t subsampling_y_[kMaxPlanes];
+
+  // The dimensions (in order) are: segment_id, level_index (based on plane and
+  // direction), reference_frame and mode_id.
+  uint8_t deblock_filter_levels_[kMaxSegments][kFrameLfCount]
+                                [kNumReferenceFrameTypes][2];
+
   // current_quantizer_index_ is in the range [0, 255].
-  int current_quantizer_index_;
-  // First dimension: left/top; Second dimension: plane; Third dimension:
-  // row4x4/column4x4.
-  std::array<Array2D<EntropyContext>, 2> entropy_contexts_;
+  uint8_t current_quantizer_index_;
+  // These two arrays (|coefficient_levels_| and |dc_categories_|) are used to
+  // store the entropy context. Their dimensions are as follows: First -
+  // left/top; Second - plane; Third - row4x4 (if first dimension is
+  // left)/column4x4 (if first dimension is top).
+  //
+  // This is equivalent to the LeftLevelContext and AboveLevelContext arrays in
+  // the spec. In the spec, it stores values from 0 through 63 (inclusive). The
+  // stored values are used to compute the left and top contexts in
+  // GetTransformAllZeroContext. In that function, we only care about the
+  // following values: 0, 1, 2, 3 and >= 4. So instead of clamping to 63, we
+  // clamp to 4 (i.e.) all the values greater than 4 are stored as 4.
+  std::array<Array2D<uint8_t>, 2> coefficient_levels_;
+  // This is equivalent to the LeftDcContext and AboveDcContext arrays in the
+  // spec. In the spec, it can store 3 possible values: 0, 1 and 2 (where 1
+  // means the value is < 0, 2 means the value is > 0 and 0 means the value is
+  // equal to 0).
+  //
+  // The stored values are used in two places:
+  //  * GetTransformAllZeroContext: Here, we only care about whether the
+  //  value is 0 or not (whether it is 1 or 2 is irrelevant).
+  //  * GetDcSignContext: Here, we do the following computation: if the
+  //  stored value is 1, we decrement a counter. If the stored value is 2
+  //  we increment a counter.
+  //
+  // Based on this usage, we can simply replace 1 with -1 and 2 with 1 and
+  // use that value to compute the counter.
+  //
+  // The usage on GetTransformAllZeroContext is unaffected since there we
+  // only care about whether it is 0 or not.
+  std::array<Array2D<int8_t>, 2> dc_categories_;
   const ObuSequenceHeader& sequence_header_;
   const ObuFrameHeader& frame_header_;
   RefCountedBuffer& current_frame_;
@@ -496,6 +562,18 @@
   // array will be used by each call to ReadTransformCoefficients() depending on
   // the transform size.
   int32_t quantized_[kQuantizedCoefficientBufferSize];
+  // Stores the "color order" for a block for each iteration in
+  // ReadPaletteTokens(). The "color order" is used to populate the
+  // |color_index_map| used for palette prediction. This is declared as a class
+  // variable because only the first few values in each dimension are used by
+  // each call depending on the block size and the palette size.
+  uint8_t color_order_[kMaxPaletteSquare][kMaxPaletteSize];
+  // Stores the "color context" for a block for each iteration in
+  // ReadPaletteTokens(). The "color context" is the cdf context index used to
+  // read the |palette_color_idx_y| variable in the spec. This is declared as a
+  // class variable because only the first few values in each dimension are used
+  // by each call depending on the block size and the palette size.
+  uint8_t color_context_[kMaxPaletteSquare];
   // When there is no multi-threading within the Tile, |residual_buffer_| is
   // used. When there is multi-threading within the Tile,
   // |residual_buffer_threaded_| is used. In the following comment,
@@ -542,7 +620,9 @@
   // thread will do the parsing while the thread pool workers will do the
   // decoding.
   ThreadPool* const thread_pool_;
+  ThreadingParameters threading_;
   ResidualBufferPool* const residual_buffer_pool_;
+  BlockingCounterWithStatus* const pending_tiles_;
   bool split_parse_and_decode_;
   // This is used only when |split_parse_and_decode_| is false.
   std::unique_ptr<PredictionParameters> prediction_parameters_ = nullptr;
@@ -552,6 +632,9 @@
   TransformType transform_types_[32][32];
   // delta_lf_[i] is in the range [-63, 63].
   int8_t delta_lf_[kFrameLfCount];
+  // True if all the values in |delta_lf_| are zero. False otherwise.
+  bool delta_lf_all_zero_;
+  bool build_bit_mask_when_parsing_;
 };
 
 struct Tile::Block {
@@ -590,7 +673,7 @@
 
   bool TopAvailableChroma() const {
     if (!HasChroma()) return false;
-    if ((tile.sequence_header_.color_config.subsampling_y |
+    if ((tile.sequence_header_.color_config.subsampling_y &
          kNum4x4BlocksHigh[size]) == 1) {
       return tile.IsInside(row4x4 - 2, column4x4);
     }
@@ -599,7 +682,7 @@
 
   bool LeftAvailableChroma() const {
     if (!HasChroma()) return false;
-    if ((tile.sequence_header_.color_config.subsampling_x |
+    if ((tile.sequence_header_.color_config.subsampling_x &
          kNum4x4BlocksWide[size]) == 1) {
       return tile.IsInside(row4x4, column4x4 - 2);
     }
diff --git a/libgav1/src/tile/bitstream/mode_info.cc b/libgav1/src/tile/bitstream/mode_info.cc
index be71f86..b2e6039 100644
--- a/libgav1/src/tile/bitstream/mode_info.cc
+++ b/libgav1/src/tile/bitstream/mode_info.cc
@@ -11,9 +11,11 @@
 #include "src/dsp/constants.h"
 #include "src/motion_vector.h"
 #include "src/obu_parser.h"
+#include "src/prediction_mask.h"
 #include "src/symbol_decoder_context.h"
 #include "src/tile.h"
 #include "src/utils/array_2d.h"
+#include "src/utils/bit_mask_set.h"
 #include "src/utils/block_parameters_holder.h"
 #include "src/utils/common.h"
 #include "src/utils/constants.h"
@@ -30,41 +32,42 @@
 constexpr int kDeltaLfSmall = 3;
 constexpr int kNoScale = 1 << kReferenceFrameScalePrecision;
 
-const uint8_t kIntraYModeContext[kIntraPredictionModesY] = {0, 1, 2, 3, 4, 4, 4,
-                                                            4, 3, 0, 1, 2, 0};
+constexpr uint8_t kIntraYModeContext[kIntraPredictionModesY] = {
+    0, 1, 2, 3, 4, 4, 4, 4, 3, 0, 1, 2, 0};
 
-const uint8_t kSizeGroup[kMaxBlockSizes] = {0, 0, 0, 0, 1, 1, 1, 0, 1, 2, 2,
-                                            2, 1, 2, 3, 3, 2, 3, 3, 3, 3, 3};
+constexpr uint8_t kSizeGroup[kMaxBlockSizes] = {
+    0, 0, 0, 0, 1, 1, 1, 0, 1, 2, 2, 2, 1, 2, 3, 3, 2, 3, 3, 3, 3, 3};
 
-const int kCompoundModeNewMvContexts = 5;
-const uint8_t kCompoundModeContextMap[3][kCompoundModeNewMvContexts] = {
+constexpr int kCompoundModeNewMvContexts = 5;
+constexpr uint8_t kCompoundModeContextMap[3][kCompoundModeNewMvContexts] = {
     {0, 1, 1, 1, 1}, {1, 2, 3, 4, 4}, {4, 4, 5, 6, 7}};
 
-const uint8_t kWedgeBits[kMaxBlockSizes] = {0, 0, 0, 0, 4, 4, 4, 0, 4, 4, 4,
-                                            0, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0};
-
 enum CflSign : uint8_t {
   kCflSignZero = 0,
   kCflSignNegative = 1,
   kCflSignPositive = 2
 };
 
-inline bool IsBackwardReference(ReferenceFrameType type) {
+constexpr BitMaskSet kPredictionModeHasNearMvMask(kPredictionModeNearMv,
+                                                  kPredictionModeNearNearMv,
+                                                  kPredictionModeNearNewMv,
+                                                  kPredictionModeNewNearMv);
+
+constexpr BitMaskSet kIsInterIntraModeAllowedMask(kBlock8x8, kBlock8x16,
+                                                  kBlock16x8, kBlock16x16,
+                                                  kBlock16x32, kBlock32x16,
+                                                  kBlock32x32);
+
+bool IsBackwardReference(ReferenceFrameType type) {
   return type >= kReferenceFrameBackward && type <= kReferenceFrameAlternate;
 }
 
-inline bool IsSameDirectionReferencePair(ReferenceFrameType type1,
-                                         ReferenceFrameType type2) {
+bool IsSameDirectionReferencePair(ReferenceFrameType type1,
+                                  ReferenceFrameType type2) {
   return (type1 >= kReferenceFrameBackward) ==
          (type2 >= kReferenceFrameBackward);
 }
 
-inline bool IsInterIntraModeAllowed(BlockSize size) {
-  return size == kBlock8x8 || size == kBlock8x16 || size == kBlock16x8 ||
-         size == kBlock16x16 || size == kBlock16x32 || size == kBlock32x16 ||
-         size == kBlock32x32;
-}
-
 // This is called neg_deinterleave() in the spec.
 int DecodeSegmentId(int diff, int reference, int max) {
   if (reference == 0) return diff;
@@ -98,6 +101,17 @@
   return 0;
 }
 
+// Returns true if the either the width or the height of the block is equal to
+// four.
+bool IsBlockDimension4(BlockSize size) {
+  return size < kBlock8x8 || size == kBlock16x4;
+}
+
+// Returns true if both the width and height of the block is less than 64.
+bool IsBlockDimensionLessThan64(BlockSize size) {
+  return size <= kBlock32x32 && size != kBlock16x64;
+}
+
 }  // namespace
 
 bool Tile::ReadSegmentId(const Block& block) {
@@ -139,7 +153,7 @@
   uint16_t* const segment_id_cdf =
       symbol_decoder_context_.segment_id_cdf[context];
   const int encoded_segment_id =
-      reader_.ReadSymbol(segment_id_cdf, kMaxSegments);
+      reader_.ReadSymbol<kMaxSegments>(segment_id_cdf);
   bp.segment_id =
       DecodeSegmentId(encoded_segment_id, pred,
                       frame_header_.segmentation.last_active_segment_id + 1);
@@ -181,7 +195,7 @@
     ++context;
   }
   uint16_t* const skip_cdf = symbol_decoder_context_.skip_cdf[context];
-  bp.skip = static_cast<bool>(reader_.ReadSymbol(skip_cdf, 2));
+  bp.skip = reader_.ReadSymbol(skip_cdf);
 }
 
 void Tile::ReadSkipMode(const Block& block) {
@@ -193,7 +207,7 @@
                                                kSegmentFeatureReferenceFrame) ||
       frame_header_.segmentation.FeatureActive(bp.segment_id,
                                                kSegmentFeatureGlobalMv) ||
-      kBlockWidthPixels[block.size] < 8 || kBlockHeightPixels[block.size] < 8) {
+      IsBlockDimension4(block.size)) {
     bp.skip_mode = false;
     return;
   }
@@ -229,10 +243,9 @@
   }
 }
 
-int Tile::ReadAndClipDelta(uint16_t* const cdf, int symbol_count,
-                           int delta_small, int scale, int min_value,
-                           int max_value, int value) {
-  int abs = reader_.ReadSymbol(cdf, symbol_count);
+int Tile::ReadAndClipDelta(uint16_t* const cdf, int delta_small, int scale,
+                           int min_value, int max_value, int value) {
+  int abs = reader_.ReadSymbol<kDeltaSymbolCount>(cdf);
   if (abs == delta_small) {
     const int remaining_bit_count =
         static_cast<int>(reader_.ReadLiteral(3)) + 1;
@@ -251,19 +264,21 @@
 }
 
 void Tile::ReadQuantizerIndexDelta(const Block& block) {
+  assert(read_deltas_);
   BlockParameters& bp = *block.bp;
-  if (!read_deltas_ || (block.size == SuperBlockSize() && bp.skip)) {
+  if ((block.size == SuperBlockSize() && bp.skip)) {
     return;
   }
-  current_quantizer_index_ = ReadAndClipDelta(
-      symbol_decoder_context_.delta_q_cdf, kDeltaQSymbolCount, kDeltaQSmall,
-      frame_header_.delta_q.scale, kMinLossyQuantizer, kMaxQuantizer,
-      current_quantizer_index_);
+  current_quantizer_index_ =
+      ReadAndClipDelta(symbol_decoder_context_.delta_q_cdf, kDeltaQSmall,
+                       frame_header_.delta_q.scale, kMinLossyQuantizer,
+                       kMaxQuantizer, current_quantizer_index_);
 }
 
 void Tile::ReadLoopFilterDelta(const Block& block) {
+  assert(read_deltas_);
   BlockParameters& bp = *block.bp;
-  if (!read_deltas_ || !frame_header_.delta_lf.present ||
+  if (!frame_header_.delta_lf.present ||
       (block.size == SuperBlockSize() && bp.skip)) {
     return;
   }
@@ -271,15 +286,23 @@
   if (frame_header_.delta_lf.multi) {
     frame_lf_count = kFrameLfCount - (PlaneCount() > 1 ? 0 : 2);
   }
+  bool recompute_deblock_filter_levels = false;
   for (int i = 0; i < frame_lf_count; ++i) {
     uint16_t* const delta_lf_abs_cdf =
         frame_header_.delta_lf.multi
             ? symbol_decoder_context_.delta_lf_multi_cdf[i]
             : symbol_decoder_context_.delta_lf_cdf;
-    delta_lf_[i] =
-        ReadAndClipDelta(delta_lf_abs_cdf, kDeltaLfSymbolCount, kDeltaLfSmall,
-                         frame_header_.delta_lf.scale, -kMaxLoopFilterValue,
-                         kMaxLoopFilterValue, delta_lf_[i]);
+    const int8_t old_delta_lf = delta_lf_[i];
+    delta_lf_[i] = ReadAndClipDelta(
+        delta_lf_abs_cdf, kDeltaLfSmall, frame_header_.delta_lf.scale,
+        -kMaxLoopFilterValue, kMaxLoopFilterValue, delta_lf_[i]);
+    recompute_deblock_filter_levels =
+        recompute_deblock_filter_levels || (old_delta_lf != delta_lf_[i]);
+  }
+  delta_lf_all_zero_ =
+      (delta_lf_[0] | delta_lf_[1] | delta_lf_[2] | delta_lf_[3]) == 0;
+  if (!delta_lf_all_zero_ && recompute_deblock_filter_levels) {
+    post_filter_.ComputeDeblockFilterLevels(delta_lf_, deblock_filter_levels_);
   }
 }
 
@@ -298,7 +321,7 @@
     cdf = symbol_decoder_context_.y_mode_cdf[kSizeGroup[block.size]];
   }
   block.bp->y_mode = static_cast<PredictionMode>(
-      reader_.ReadSymbol(cdf, static_cast<int>(kIntraPredictionModesY)));
+      reader_.ReadSymbol<kIntraPredictionModesY>(cdf));
 }
 
 void Tile::ReadIntraAngleInfo(const Block& block, PlaneType plane_type) {
@@ -312,13 +335,13 @@
   uint16_t* const cdf =
       symbol_decoder_context_.angle_delta_cdf[mode - kPredictionModeVertical];
   prediction_parameters.angle_delta[plane_type] =
-      reader_.ReadSymbol(cdf, kAngleDeltaSymbolCount);
+      reader_.ReadSymbol<kAngleDeltaSymbolCount>(cdf);
   prediction_parameters.angle_delta[plane_type] -= kMaxAngleDelta;
 }
 
 void Tile::ReadCflAlpha(const Block& block) {
-  const int signs = reader_.ReadSymbol(
-      symbol_decoder_context_.cfl_alpha_signs_cdf, kCflAlphaSignsSymbolCount);
+  const int signs = reader_.ReadSymbol<kCflAlphaSignsSymbolCount>(
+      symbol_decoder_context_.cfl_alpha_signs_cdf);
   const auto sign_u = static_cast<CflSign>((signs + 1) / 3);
   const auto sign_v = static_cast<CflSign>((signs + 1) % 3);
   PredictionParameters& prediction_parameters =
@@ -326,8 +349,8 @@
   prediction_parameters.cfl_alpha_u = 0;
   if (sign_u != kCflSignZero) {
     prediction_parameters.cfl_alpha_u =
-        reader_.ReadSymbol(symbol_decoder_context_.cfl_alpha_cdf[signs - 2],
-                           kCflAlphaSymbolCount) +
+        reader_.ReadSymbol<kCflAlphaSymbolCount>(
+            symbol_decoder_context_.cfl_alpha_cdf[signs - 2]) +
         1;
     if (sign_u == kCflSignNegative) prediction_parameters.cfl_alpha_u *= -1;
   }
@@ -335,8 +358,8 @@
   if (sign_v != kCflSignZero) {
     const int context = (sign_v - 1) * 3 + sign_u;
     prediction_parameters.cfl_alpha_v =
-        reader_.ReadSymbol(symbol_decoder_context_.cfl_alpha_cdf[context],
-                           kCflAlphaSymbolCount) +
+        reader_.ReadSymbol<kCflAlphaSymbolCount>(
+            symbol_decoder_context_.cfl_alpha_cdf[context]) +
         1;
     if (sign_v == kCflSignNegative) prediction_parameters.cfl_alpha_v *= -1;
   }
@@ -347,11 +370,10 @@
   bool chroma_from_luma_allowed;
   if (frame_header_.segmentation.lossless[bp.segment_id]) {
     chroma_from_luma_allowed =
-        kPlaneResidualSize[block.size][SubsamplingX(kPlaneU)]
-                          [SubsamplingY(kPlaneU)] == kBlock4x4;
+        kPlaneResidualSize[block.size][subsampling_x_[kPlaneU]]
+                          [subsampling_y_[kPlaneU]] == kBlock4x4;
   } else {
-    chroma_from_luma_allowed = kBlockWidthPixels[block.size] <= 32 &&
-                               kBlockHeightPixels[block.size] <= 32;
+    chroma_from_luma_allowed = IsBlockDimensionLessThan64(block.size);
   }
   uint16_t* const cdf =
       symbol_decoder_context_
@@ -367,9 +389,8 @@
       static_cast<int>(block.bp->prediction_parameters->use_intra_block_copy);
   const bool sign = reader_.ReadSymbol(
       symbol_decoder_context_.mv_sign_cdf[component][context]);
-  const int mv_class = reader_.ReadSymbol(
-      symbol_decoder_context_.mv_class_cdf[component][context],
-      kMvClassSymbolCount);
+  const int mv_class = reader_.ReadSymbol<kMvClassSymbolCount>(
+      symbol_decoder_context_.mv_class_cdf[component][context]);
   int magnitude = 1;
   int value;
   uint16_t* fraction_cdf;
@@ -396,7 +417,7 @@
   }
   const int fraction =
       (frame_header_.force_integer_mv == 0)
-          ? reader_.ReadSymbol(fraction_cdf, kMvFractionSymbolCount)
+          ? reader_.ReadSymbol<kMvFractionSymbolCount>(fraction_cdf)
           : 3;
   const int precision =
       frame_header_.allow_high_precision_mv
@@ -430,16 +451,15 @@
   prediction_parameters.use_filter_intra = false;
   if (!sequence_header_.enable_filter_intra || bp.y_mode != kPredictionModeDc ||
       bp.palette_mode_info.size[kPlaneTypeY] != 0 ||
-      kBlockWidthPixels[block.size] > 32 ||
-      kBlockHeightPixels[block.size] > 32) {
+      !IsBlockDimensionLessThan64(block.size)) {
     return;
   }
   prediction_parameters.use_filter_intra = reader_.ReadSymbol(
       symbol_decoder_context_.use_filter_intra_cdf[block.size]);
   if (prediction_parameters.use_filter_intra) {
     prediction_parameters.filter_intra_mode = static_cast<FilterIntraPredictor>(
-        reader_.ReadSymbol(symbol_decoder_context_.filter_intra_mode_cdf,
-                           static_cast<int>(kNumFilterIntraPredictors)));
+        reader_.ReadSymbol<kNumFilterIntraPredictors>(
+            symbol_decoder_context_.filter_intra_mode_cdf));
   }
 }
 
@@ -457,9 +477,11 @@
     return false;
   }
   ReadCdef(block);
-  ReadQuantizerIndexDelta(block);
-  ReadLoopFilterDelta(block);
-  read_deltas_ = false;
+  if (read_deltas_) {
+    ReadQuantizerIndexDelta(block);
+    ReadLoopFilterDelta(block);
+    read_deltas_ = false;
+  }
   PredictionParameters& prediction_parameters =
       *block.bp->prediction_parameters;
   prediction_parameters.use_intra_block_copy = false;
@@ -551,29 +573,32 @@
   BlockParameters& bp = *block.bp;
   if (bp.skip_mode) {
     bp.is_inter = true;
-  } else if (frame_header_.segmentation.FeatureActive(
-                 bp.segment_id, kSegmentFeatureReferenceFrame)) {
+    return;
+  }
+  if (frame_header_.segmentation.FeatureActive(bp.segment_id,
+                                               kSegmentFeatureReferenceFrame)) {
     bp.is_inter =
         frame_header_.segmentation
             .feature_data[bp.segment_id][kSegmentFeatureReferenceFrame] !=
         kReferenceFrameIntra;
-  } else if (frame_header_.segmentation.FeatureActive(
-                 bp.segment_id, kSegmentFeatureGlobalMv)) {
-    bp.is_inter = true;
-  } else {
-    int context = 0;
-    if (block.top_available && block.left_available) {
-      context =
-          (block.IsTopIntra() && block.IsLeftIntra())
-              ? 3
-              : static_cast<int>(block.IsTopIntra() || block.IsLeftIntra());
-    } else if (block.top_available || block.left_available) {
-      context = 2 * static_cast<int>(block.top_available ? block.IsTopIntra()
-                                                         : block.IsLeftIntra());
-    }
-    bp.is_inter =
-        reader_.ReadSymbol(symbol_decoder_context_.is_inter_cdf[context]);
+    return;
   }
+  if (frame_header_.segmentation.FeatureActive(bp.segment_id,
+                                               kSegmentFeatureGlobalMv)) {
+    bp.is_inter = true;
+    return;
+  }
+  int context = 0;
+  if (block.top_available && block.left_available) {
+    context = (block.IsTopIntra() && block.IsLeftIntra())
+                  ? 3
+                  : static_cast<int>(block.IsTopIntra() || block.IsLeftIntra());
+  } else if (block.top_available || block.left_available) {
+    context = 2 * static_cast<int>(block.top_available ? block.IsTopIntra()
+                                                       : block.IsLeftIntra());
+  }
+  bp.is_inter =
+      reader_.ReadSymbol(symbol_decoder_context_.is_inter_cdf[context]);
 }
 
 bool Tile::ReadIntraBlockModeInfo(const Block& block, bool intra_y_mode) {
@@ -677,62 +702,67 @@
       symbol_decoder_context_.compound_reference_type_cdf[context]));
 }
 
-int Tile::GetReferenceContext(
-    const Block& block, const std::vector<ReferenceFrameType>& types1,
-    const std::vector<ReferenceFrameType>& types2) const {
-  int count[2] = {};
-  for (int i = 0; i < 2; ++i) {
-    for (const auto& type : (i == 0) ? types1 : types2) {
-      count[i] += block.CountReferences(type);
-    }
+int Tile::GetReferenceContext(const Block& block,
+                              ReferenceFrameType type0_start,
+                              ReferenceFrameType type0_end,
+                              ReferenceFrameType type1_start,
+                              ReferenceFrameType type1_end) const {
+  int count0 = 0;
+  int count1 = 0;
+  for (int type = type0_start; type <= type0_end; ++type) {
+    count0 += block.CountReferences(static_cast<ReferenceFrameType>(type));
   }
-  return (count[0] < count[1]) ? 0 : (count[0] == count[1] ? 1 : 2);
+  for (int type = type1_start; type <= type1_end; ++type) {
+    count1 += block.CountReferences(static_cast<ReferenceFrameType>(type));
+  }
+  return (count0 < count1) ? 0 : (count0 == count1 ? 1 : 2);
 }
 
+template <bool is_single, bool is_backward, int index>
 uint16_t* Tile::GetReferenceCdf(
-    const Block& block, bool is_single, bool is_backward, int index,
+    const Block& block,
     CompoundReferenceType type /*= kNumCompoundReferenceTypes*/) {
   int context = 0;
   if ((type == kCompoundReferenceUnidirectional && index == 0) ||
       (is_single && index == 1)) {
     // uni_comp_ref and single_ref_p1.
     context =
-        GetReferenceContext(block,
-                            {kReferenceFrameLast, kReferenceFrameLast2,
-                             kReferenceFrameLast3, kReferenceFrameGolden},
-                            {kReferenceFrameBackward, kReferenceFrameAlternate2,
-                             kReferenceFrameAlternate});
+        GetReferenceContext(block, kReferenceFrameLast, kReferenceFrameGolden,
+                            kReferenceFrameBackward, kReferenceFrameAlternate);
   } else if (type == kCompoundReferenceUnidirectional && index == 1) {
     // uni_comp_ref_p1.
     context =
-        GetReferenceContext(block, {kReferenceFrameLast2},
-                            {kReferenceFrameLast3, kReferenceFrameGolden});
+        GetReferenceContext(block, kReferenceFrameLast2, kReferenceFrameLast2,
+                            kReferenceFrameLast3, kReferenceFrameGolden);
   } else if ((type == kCompoundReferenceUnidirectional && index == 2) ||
              (type == kCompoundReferenceBidirectional && index == 2) ||
              (is_single && index == 5)) {
     // uni_comp_ref_p2, comp_ref_p2 and single_ref_p5.
-    context = GetReferenceContext(block, {kReferenceFrameLast3},
-                                  {kReferenceFrameGolden});
+    context =
+        GetReferenceContext(block, kReferenceFrameLast3, kReferenceFrameLast3,
+                            kReferenceFrameGolden, kReferenceFrameGolden);
   } else if ((type == kCompoundReferenceBidirectional && index == 0) ||
              (is_single && index == 3)) {
     // comp_ref and single_ref_p3.
     context =
-        GetReferenceContext(block, {kReferenceFrameLast, kReferenceFrameLast2},
-                            {kReferenceFrameLast3, kReferenceFrameGolden});
+        GetReferenceContext(block, kReferenceFrameLast, kReferenceFrameLast2,
+                            kReferenceFrameLast3, kReferenceFrameGolden);
   } else if ((type == kCompoundReferenceBidirectional && index == 1) ||
              (is_single && index == 4)) {
     // comp_ref_p1 and single_ref_p4.
-    context = GetReferenceContext(block, {kReferenceFrameLast},
-                                  {kReferenceFrameLast2});
+    context =
+        GetReferenceContext(block, kReferenceFrameLast, kReferenceFrameLast,
+                            kReferenceFrameLast2, kReferenceFrameLast2);
   } else if ((is_single && index == 2) || (is_backward && index == 0)) {
     // single_ref_p2 and comp_bwdref.
     context = GetReferenceContext(
-        block, {kReferenceFrameBackward, kReferenceFrameAlternate2},
-        {kReferenceFrameAlternate});
+        block, kReferenceFrameBackward, kReferenceFrameAlternate2,
+        kReferenceFrameAlternate, kReferenceFrameAlternate);
   } else if ((is_single && index == 6) || (is_backward && index == 1)) {
     // single_ref_p6 and comp_bwdref_p1.
-    context = GetReferenceContext(block, {kReferenceFrameBackward},
-                                  {kReferenceFrameAlternate2});
+    context = GetReferenceContext(
+        block, kReferenceFrameBackward, kReferenceFrameBackward,
+        kReferenceFrameAlternate2, kReferenceFrameAlternate2);
   }
   if (is_single) {
     // The index parameter for single references is offset by one since the spec
@@ -781,21 +811,21 @@
     if (reference_type == kCompoundReferenceUnidirectional) {
       // uni_comp_ref.
       if (reader_.ReadSymbol(
-              GetReferenceCdf(block, false, false, 0, reference_type))) {
+              GetReferenceCdf<false, false, 0>(block, reference_type))) {
         bp.reference_frame[0] = kReferenceFrameBackward;
         bp.reference_frame[1] = kReferenceFrameAlternate;
         return;
       }
       // uni_comp_ref_p1.
       if (!reader_.ReadSymbol(
-              GetReferenceCdf(block, false, false, 1, reference_type))) {
+              GetReferenceCdf<false, false, 1>(block, reference_type))) {
         bp.reference_frame[0] = kReferenceFrameLast;
         bp.reference_frame[1] = kReferenceFrameLast2;
         return;
       }
       // uni_comp_ref_p2.
       if (reader_.ReadSymbol(
-              GetReferenceCdf(block, false, false, 2, reference_type))) {
+              GetReferenceCdf<false, false, 2>(block, reference_type))) {
         bp.reference_frame[0] = kReferenceFrameLast;
         bp.reference_frame[1] = kReferenceFrameGolden;
         return;
@@ -807,26 +837,28 @@
     assert(reference_type == kCompoundReferenceBidirectional);
     // comp_ref.
     if (reader_.ReadSymbol(
-            GetReferenceCdf(block, false, false, 0, reference_type))) {
+            GetReferenceCdf<false, false, 0>(block, reference_type))) {
       // comp_ref_p2.
-      bp.reference_frame[0] = reader_.ReadSymbol(GetReferenceCdf(
-                                  block, false, false, 2, reference_type))
-                                  ? kReferenceFrameGolden
-                                  : kReferenceFrameLast3;
+      bp.reference_frame[0] =
+          reader_.ReadSymbol(
+              GetReferenceCdf<false, false, 2>(block, reference_type))
+              ? kReferenceFrameGolden
+              : kReferenceFrameLast3;
     } else {
       // comp_ref_p1.
-      bp.reference_frame[0] = reader_.ReadSymbol(GetReferenceCdf(
-                                  block, false, false, 1, reference_type))
-                                  ? kReferenceFrameLast2
-                                  : kReferenceFrameLast;
+      bp.reference_frame[0] =
+          reader_.ReadSymbol(
+              GetReferenceCdf<false, false, 1>(block, reference_type))
+              ? kReferenceFrameLast2
+              : kReferenceFrameLast;
     }
     // comp_bwdref.
-    if (reader_.ReadSymbol(GetReferenceCdf(block, false, true, 0))) {
+    if (reader_.ReadSymbol(GetReferenceCdf<false, true, 0>(block))) {
       bp.reference_frame[1] = kReferenceFrameAlternate;
     } else {
       // comp_bwdref_p1.
       bp.reference_frame[1] =
-          reader_.ReadSymbol(GetReferenceCdf(block, false, true, 1))
+          reader_.ReadSymbol(GetReferenceCdf<false, true, 1>(block))
               ? kReferenceFrameAlternate2
               : kReferenceFrameBackward;
     }
@@ -835,31 +867,31 @@
   assert(!use_compound_reference);
   bp.reference_frame[1] = kReferenceFrameNone;
   // single_ref_p1.
-  if (reader_.ReadSymbol(GetReferenceCdf(block, true, false, 1))) {
+  if (reader_.ReadSymbol(GetReferenceCdf<true, false, 1>(block))) {
     // single_ref_p2.
-    if (reader_.ReadSymbol(GetReferenceCdf(block, true, false, 2))) {
+    if (reader_.ReadSymbol(GetReferenceCdf<true, false, 2>(block))) {
       bp.reference_frame[0] = kReferenceFrameAlternate;
       return;
     }
     // single_ref_p6.
     bp.reference_frame[0] =
-        reader_.ReadSymbol(GetReferenceCdf(block, true, false, 6))
+        reader_.ReadSymbol(GetReferenceCdf<true, false, 6>(block))
             ? kReferenceFrameAlternate2
             : kReferenceFrameBackward;
     return;
   }
   // single_ref_p3.
-  if (reader_.ReadSymbol(GetReferenceCdf(block, true, false, 3))) {
+  if (reader_.ReadSymbol(GetReferenceCdf<true, false, 3>(block))) {
     // single_ref_p5.
     bp.reference_frame[0] =
-        reader_.ReadSymbol(GetReferenceCdf(block, true, false, 5))
+        reader_.ReadSymbol(GetReferenceCdf<true, false, 5>(block))
             ? kReferenceFrameGolden
             : kReferenceFrameLast3;
     return;
   }
   // single_ref_p4.
   bp.reference_frame[0] =
-      reader_.ReadSymbol(GetReferenceCdf(block, true, false, 4))
+      reader_.ReadSymbol(GetReferenceCdf<true, false, 4>(block))
           ? kReferenceFrameLast2
           : kReferenceFrameLast;
 }
@@ -883,9 +915,8 @@
     const int idx1 =
         std::min(mode_contexts.new_mv, kCompoundModeNewMvContexts - 1);
     const int context = kCompoundModeContextMap[idx0][idx1];
-    const int offset = reader_.ReadSymbol(
-        symbol_decoder_context_.compound_prediction_mode_cdf[context],
-        static_cast<int>(kNumCompoundInterPredictionModes));
+    const int offset = reader_.ReadSymbol<kNumCompoundInterPredictionModes>(
+        symbol_decoder_context_.compound_prediction_mode_cdf[context]);
     bp.y_mode =
         static_cast<PredictionMode>(kPredictionModeNearestNearestMv + offset);
     return;
@@ -916,10 +947,12 @@
       *block.bp->prediction_parameters;
   prediction_parameters.ref_mv_index = 0;
   if (bp.y_mode != kPredictionModeNewMv &&
-      bp.y_mode != kPredictionModeNewNewMv && !HasNearMv(bp.y_mode)) {
+      bp.y_mode != kPredictionModeNewNewMv &&
+      !kPredictionModeHasNearMvMask.Contains(bp.y_mode)) {
     return;
   }
-  const int start = static_cast<int>(HasNearMv(bp.y_mode));
+  const int start =
+      static_cast<int>(kPredictionModeHasNearMvMask.Contains(bp.y_mode));
   prediction_parameters.ref_mv_index = start;
   for (int i = start; i < start + 2; ++i) {
     if (prediction_parameters.ref_mv_count <= i + 1) continue;
@@ -940,7 +973,7 @@
   prediction_parameters.inter_intra_mode = kNumInterIntraModes;
   prediction_parameters.is_wedge_inter_intra = false;
   if (bp.skip_mode || !sequence_header_.enable_interintra_compound ||
-      is_compound || !IsInterIntraModeAllowed(block.size)) {
+      is_compound || !kIsInterIntraModeAllowedMask.Contains(block.size)) {
     return;
   }
   // kSizeGroup[block.size] is guaranteed to be non-zero because of the block
@@ -952,10 +985,10 @@
     prediction_parameters.inter_intra_mode = kNumInterIntraModes;
     return;
   }
-  prediction_parameters
-      .inter_intra_mode = static_cast<InterIntraMode>(reader_.ReadSymbol(
-      symbol_decoder_context_.inter_intra_mode_cdf[kSizeGroup[block.size] - 1],
-      static_cast<int>(kNumInterIntraModes)));
+  prediction_parameters.inter_intra_mode =
+      static_cast<InterIntraMode>(reader_.ReadSymbol<kNumInterIntraModes>(
+          symbol_decoder_context_
+              .inter_intra_mode_cdf[kSizeGroup[block.size] - 1]));
   bp.reference_frame[1] = kReferenceFrameIntra;
   prediction_parameters.angle_delta[kPlaneTypeY] = 0;
   prediction_parameters.angle_delta[kPlaneTypeUV] = 0;
@@ -964,12 +997,12 @@
       symbol_decoder_context_.is_wedge_inter_intra_cdf[block.size]);
   if (!prediction_parameters.is_wedge_inter_intra) return;
   prediction_parameters.wedge_index =
-      reader_.ReadSymbol(symbol_decoder_context_.wedge_index_cdf[block.size],
-                         kWedgeIndexSymbolCount);
+      reader_.ReadSymbol<kWedgeIndexSymbolCount>(
+          symbol_decoder_context_.wedge_index_cdf[block.size]);
   prediction_parameters.wedge_sign = 0;
 }
 
-bool Tile::IsScaled(ReferenceFrameType type) {
+bool Tile::IsScaled(ReferenceFrameType type) const {
   const int index =
       frame_header_.reference_frame_index[type - kReferenceFrameLast];
   const int x_scale = ((reference_frames_[index]->upscaled_width()
@@ -991,7 +1024,7 @@
   const auto global_motion_type =
       frame_header_.global_motion[bp.reference_frame[0]].type;
   if (bp.skip_mode || !frame_header_.is_motion_mode_switchable ||
-      kBlockWidthPixels[block.size] < 8 || kBlockHeightPixels[block.size] < 8 ||
+      IsBlockDimension4(block.size) ||
       (frame_header_.force_integer_mv == 0 &&
        (bp.y_mode == kPredictionModeGlobalMv ||
         bp.y_mode == kPredictionModeGlobalGlobalMv) &&
@@ -1017,9 +1050,9 @@
             : kMotionModeSimple;
     return;
   }
-  prediction_parameters.motion_mode = static_cast<MotionMode>(
-      reader_.ReadSymbol(symbol_decoder_context_.motion_mode_cdf[block.size],
-                         static_cast<int>(kNumMotionModes)));
+  prediction_parameters.motion_mode =
+      static_cast<MotionMode>(reader_.ReadSymbol<kNumMotionModes>(
+          symbol_decoder_context_.motion_mode_cdf[block.size]));
 }
 
 uint16_t* Tile::GetIsExplicitCompoundTypeCdf(const Block& block) {
@@ -1087,16 +1120,15 @@
           reader_.ReadSymbol(GetIsExplicitCompoundTypeCdf(block));
     }
     if (bp.is_explicit_compound_type) {
-      if (kWedgeBits[block.size] == 0) {
-        prediction_parameters.compound_prediction_type =
-            kCompoundPredictionTypeDiffWeighted;
-      } else {
+      if (kIsWedgeCompoundModeAllowed.Contains(block.size)) {
         // Only kCompoundPredictionTypeWedge and
         // kCompoundPredictionTypeDiffWeighted are signaled explicitly.
         prediction_parameters.compound_prediction_type =
             static_cast<CompoundPredictionType>(reader_.ReadSymbol(
-                symbol_decoder_context_.compound_type_cdf[block.size],
-                static_cast<int>(kNumExplicitCompoundPredictionTypes)));
+                symbol_decoder_context_.compound_type_cdf[block.size]));
+      } else {
+        prediction_parameters.compound_prediction_type =
+            kCompoundPredictionTypeDiffWeighted;
       }
     } else {
       if (sequence_header_.enable_jnt_comp) {
@@ -1113,9 +1145,9 @@
     }
     if (prediction_parameters.compound_prediction_type ==
         kCompoundPredictionTypeWedge) {
-      prediction_parameters.wedge_index = reader_.ReadSymbol(
-          symbol_decoder_context_.wedge_index_cdf[block.size],
-          kWedgeIndexSymbolCount);
+      prediction_parameters.wedge_index =
+          reader_.ReadSymbol<kWedgeIndexSymbolCount>(
+              symbol_decoder_context_.wedge_index_cdf[block.size]);
       prediction_parameters.wedge_sign =
           static_cast<int>(reader_.ReadLiteral(1));
     } else if (prediction_parameters.compound_prediction_type ==
@@ -1179,18 +1211,16 @@
     }
     return;
   }
-  const bool is_block_larger_than_8x8 =
-      std::min(kBlockWidthPixels[block.size], kBlockHeightPixels[block.size]) >=
-      8;
   bool interpolation_filter_present = true;
   if (bp.skip_mode ||
       block.bp->prediction_parameters->motion_mode == kMotionModeLocalWarp) {
     interpolation_filter_present = false;
-  } else if (is_block_larger_than_8x8 && bp.y_mode == kPredictionModeGlobalMv) {
+  } else if (!IsBlockDimension4(block.size) &&
+             bp.y_mode == kPredictionModeGlobalMv) {
     interpolation_filter_present =
         frame_header_.global_motion[bp.reference_frame[0]].type ==
         kGlobalMotionTransformationTypeTranslation;
-  } else if (is_block_larger_than_8x8 &&
+  } else if (!IsBlockDimension4(block.size) &&
              bp.y_mode == kPredictionModeGlobalGlobalMv) {
     interpolation_filter_present =
         frame_header_.global_motion[bp.reference_frame[0]].type ==
@@ -1201,9 +1231,9 @@
   for (int i = 0; i < (sequence_header_.enable_dual_filter ? 2 : 1); ++i) {
     bp.interpolation_filter[i] =
         interpolation_filter_present
-            ? static_cast<InterpolationFilter>(reader_.ReadSymbol(
-                  GetInterpolationFilterCdf(block, i),
-                  static_cast<int>(kNumExplicitInterpolationFilters)))
+            ? static_cast<InterpolationFilter>(
+                  reader_.ReadSymbol<kNumExplicitInterpolationFilters>(
+                      GetInterpolationFilterCdf(block, i)))
             : kInterpolationFilterEightTap;
   }
   if (!sequence_header_.enable_dual_filter) {
@@ -1250,9 +1280,11 @@
     return false;
   }
   ReadCdef(block);
-  ReadQuantizerIndexDelta(block);
-  ReadLoopFilterDelta(block);
-  read_deltas_ = false;
+  if (read_deltas_) {
+    ReadQuantizerIndexDelta(block);
+    ReadLoopFilterDelta(block);
+    read_deltas_ = false;
+  }
   ReadIsInter(block);
   return bp.is_inter ? ReadInterBlockModeInfo(block)
                      : ReadIntraBlockModeInfo(block, /*intra_y_mode=*/false);
diff --git a/libgav1/src/tile/bitstream/palette.cc b/libgav1/src/tile/bitstream/palette.cc
index 9c9bd4e..133b016 100644
--- a/libgav1/src/tile/bitstream/palette.cc
+++ b/libgav1/src/tile/bitstream/palette.cc
@@ -8,6 +8,7 @@
 #include "src/obu_parser.h"
 #include "src/symbol_decoder_context.h"
 #include "src/tile.h"
+#include "src/utils/bit_mask_set.h"
 #include "src/utils/common.h"
 #include "src/utils/constants.h"
 #include "src/utils/entropy_decoder.h"
@@ -16,14 +17,9 @@
 namespace libgav1 {
 namespace {
 
-const int kNumPaletteNeighbors = 3;
-const uint8_t kPaletteColorHashMultiplier[kNumPaletteNeighbors] = {1, 2, 2};
-const int kPaletteColorIndexContext[kPaletteColorIndexSymbolCount + 1] = {
-    -1, -1, 0, -1, -1, 4, 3, 2, 1};
-
 // Add |value| to the |cache| if it doesn't already exist.
-inline void MaybeAddToPaletteCache(uint16_t value, uint16_t* const cache,
-                                   int* const n) {
+void MaybeAddToPaletteCache(uint16_t value, uint16_t* const cache,
+                            int* const n) {
   assert(cache != nullptr);
   assert(n != nullptr);
   if (*n > 0 && value == cache[*n - 1]) return;
@@ -170,8 +166,7 @@
 
 void Tile::ReadPaletteModeInfo(const Block& block) {
   BlockParameters& bp = *block.bp;
-  if (IsBlockSmallerThan8x8(block.size) || kBlockWidthPixels[block.size] > 64 ||
-      kBlockHeightPixels[block.size] > 64 ||
+  if (IsBlockSmallerThan8x8(block.size) || block.size > kBlock64x64 ||
       !frame_header_.allow_screen_content_tools) {
     bp.palette_mode_info.size[kPlaneTypeY] = 0;
     bp.palette_mode_info.size[kPlaneTypeUV] = 0;
@@ -186,9 +181,8 @@
     if (has_palette_y) {
       bp.palette_mode_info.size[kPlaneTypeY] =
           kMinPaletteSize +
-          reader_.ReadSymbol(
-              symbol_decoder_context_.palette_y_size_cdf[block_size_context],
-              kPaletteSizeSymbolCount);
+          reader_.ReadSymbol<kPaletteSizeSymbolCount>(
+              symbol_decoder_context_.palette_y_size_cdf[block_size_context]);
       ReadPaletteColors(block, kPlaneY);
     }
   }
@@ -201,60 +195,86 @@
     if (has_palette_uv) {
       bp.palette_mode_info.size[kPlaneTypeUV] =
           kMinPaletteSize +
-          reader_.ReadSymbol(
-              symbol_decoder_context_.palette_uv_size_cdf[block_size_context],
-              kPaletteSizeSymbolCount);
+          reader_.ReadSymbol<kPaletteSizeSymbolCount>(
+              symbol_decoder_context_.palette_uv_size_cdf[block_size_context]);
       ReadPaletteColors(block, kPlaneU);
     }
   }
 }
 
-int Tile::GetPaletteColorContext(const Block& block, PlaneType plane_type,
-                                 int row, int column, int palette_size,
-                                 uint8_t color_order[kMaxPaletteSize]) {
-  for (int i = 0; i < kMaxPaletteSize; ++i) {
-    color_order[i] = i;
-  }
-  int scores[kMaxPaletteSize] = {};
+void Tile::PopulatePaletteColorContexts(
+    const Block& block, PlaneType plane_type, int i, int start, int end,
+    uint8_t color_order[kMaxPaletteSquare][kMaxPaletteSize],
+    uint8_t color_context[kMaxPaletteSquare]) {
   const PredictionParameters& prediction_parameters =
       *block.bp->prediction_parameters;
-  if (row > 0 && column > 0) {
-    ++scores[prediction_parameters
-                 .color_index_map[plane_type][row - 1][column - 1]];
-  }
-  if (row > 0) {
-    scores[prediction_parameters
-               .color_index_map[plane_type][row - 1][column]] += 2;
-  }
-  if (column > 0) {
-    scores[prediction_parameters
-               .color_index_map[plane_type][row][column - 1]] += 2;
-  }
-  // Move the top 3 scores (largest first) and the corresponding color_order
-  // entry to the front of the array.
-  for (int i = 0; i < kNumPaletteNeighbors; ++i) {
-    const auto max_element =
-        std::max_element(scores + i, scores + palette_size);
-    const auto max_score = *max_element;
-    const auto max_index = static_cast<int>(std::distance(scores, max_element));
-    if (max_index != i) {
-      const uint8_t max_color_order = color_order[max_index];
-      for (int j = max_index; j > i; --j) {
-        scores[j] = scores[j - 1];
-        color_order[j] = color_order[j - 1];
+  for (int column = start, counter = 0; column >= end; --column, ++counter) {
+    const int row = i - column;
+    assert(row > 0 || column > 0);
+    const uint8_t top =
+        (row > 0)
+            ? prediction_parameters.color_index_map[plane_type][row - 1][column]
+            : 0;
+    const uint8_t left =
+        (column > 0)
+            ? prediction_parameters.color_index_map[plane_type][row][column - 1]
+            : 0;
+    uint8_t index_mask;
+    static_assert(kMaxPaletteSize <= 8, "");
+    int index;
+    if (column <= 0) {
+      color_context[counter] = 0;
+      color_order[counter][0] = top;
+      index_mask = 1 << top;
+      index = 1;
+    } else if (row <= 0) {
+      color_context[counter] = 0;
+      color_order[counter][0] = left;
+      index_mask = 1 << left;
+      index = 1;
+    } else {
+      const uint8_t top_left =
+          prediction_parameters
+              .color_index_map[plane_type][row - 1][column - 1];
+      index_mask = (1 << top) | (1 << left) | (1 << top_left);
+      if (top == left && top == top_left) {
+        color_context[counter] = 4;
+        color_order[counter][0] = top;
+        index = 1;
+      } else if (top == left) {
+        color_context[counter] = 3;
+        color_order[counter][0] = top;
+        color_order[counter][1] = top_left;
+        index = 2;
+      } else if (top == top_left) {
+        color_context[counter] = 2;
+        color_order[counter][0] = top_left;
+        color_order[counter][1] = left;
+        index = 2;
+      } else if (left == top_left) {
+        color_context[counter] = 2;
+        color_order[counter][0] = top_left;
+        color_order[counter][1] = top;
+        index = 2;
+      } else {
+        color_context[counter] = 1;
+        color_order[counter][0] = std::min(top, left);
+        color_order[counter][1] = std::max(top, left);
+        color_order[counter][2] = top_left;
+        index = 3;
       }
-      scores[i] = max_score;
-      color_order[i] = max_color_order;
+    }
+    // Even though only the first |palette_size| entries of this array are ever
+    // used, it is faster to populate all 8 because of the vectorization of the
+    // constant sized loop.
+    for (uint8_t j = 0; j < kMaxPaletteSize; ++j) {
+      if (BitMaskSet::MaskContainsValue(index_mask, j)) continue;
+      color_order[counter][index++] = j;
     }
   }
-  int context = 0;
-  for (int i = 0; i < kNumPaletteNeighbors; ++i) {
-    context += scores[i] * kPaletteColorHashMultiplier[i];
-  }
-  return kPaletteColorIndexContext[context];
 }
 
-void Tile::ReadPaletteTokens(const Block& block) {
+bool Tile::ReadPaletteTokens(const Block& block) {
   const PaletteModeInfo& palette_mode_info = block.bp->palette_mode_info;
   PredictionParameters& prediction_parameters =
       *block.bp->prediction_parameters;
@@ -283,24 +303,29 @@
         screen_width += 2;
       }
     }
-    uint8_t color_order[kMaxPaletteSize];
+    if (!prediction_parameters.color_index_map[plane_type].Reset(
+            block_height, block_width, /*zero_initialize=*/false)) {
+      return false;
+    }
     int first_value = 0;
     reader_.DecodeUniform(palette_size, &first_value);
     prediction_parameters.color_index_map[plane_type][0][0] = first_value;
     for (int i = 1; i < screen_height + screen_width - 1; ++i) {
-      for (int j = std::min(i, screen_width - 1);
-           j >= std::max(0, i - screen_height + 1); --j) {
-        const int context =
-            GetPaletteColorContext(block, static_cast<PlaneType>(plane_type),
-                                   i - j, j, palette_size, color_order);
-        assert(context >= 0);
+      const int start = std::min(i, screen_width - 1);
+      const int end = std::max(0, i - screen_height + 1);
+      uint8_t color_order[kMaxPaletteSquare][kMaxPaletteSize];
+      uint8_t color_context[kMaxPaletteSquare];
+      PopulatePaletteColorContexts(block, static_cast<PlaneType>(plane_type), i,
+                                   start, end, color_order, color_context);
+      for (int j = start, counter = 0; j >= end; --j, ++counter) {
         uint16_t* const cdf =
             symbol_decoder_context_
-                .palette_color_index_cdf[plane_type][palette_size -
-                                                     kMinPaletteSize][context];
+                .palette_color_index_cdf[plane_type]
+                                        [palette_size - kMinPaletteSize]
+                                        [color_context[counter]];
         const int color_order_index = reader_.ReadSymbol(cdf, palette_size);
         prediction_parameters.color_index_map[plane_type][i - j][j] =
-            color_order[color_order_index];
+            color_order[counter][color_order_index];
       }
     }
     if (screen_width < block_width) {
@@ -319,6 +344,7 @@
           block_width);
     }
   }
+  return true;
 }
 
 }  // namespace libgav1
diff --git a/libgav1/src/tile/bitstream/partition.cc b/libgav1/src/tile/bitstream/partition.cc
index 0709b2f..3a31095 100644
--- a/libgav1/src/tile/bitstream/partition.cc
+++ b/libgav1/src/tile/bitstream/partition.cc
@@ -12,12 +12,11 @@
 namespace libgav1 {
 namespace {
 
-inline uint16_t InverseCdfProbability(uint16_t probability) {
+uint16_t InverseCdfProbability(uint16_t probability) {
   return kCdfMaxProbability - probability;
 }
 
-inline uint16_t CdfElementProbability(const uint16_t* const cdf,
-                                      uint8_t element) {
+uint16_t CdfElementProbability(const uint16_t* const cdf, uint8_t element) {
   return (element > 0 ? cdf[element - 1] : uint16_t{kCdfMaxProbability}) -
          cdf[element];
 }
diff --git a/libgav1/src/tile/bitstream/transform_size.cc b/libgav1/src/tile/bitstream/transform_size.cc
index f11fb10..36867f1 100644
--- a/libgav1/src/tile/bitstream/transform_size.cc
+++ b/libgav1/src/tile/bitstream/transform_size.cc
@@ -1,7 +1,6 @@
 #include <algorithm>
 #include <cstdint>
 #include <cstring>
-#include <vector>
 
 #include "src/dsp/constants.h"
 #include "src/obu_parser.h"
@@ -13,14 +12,30 @@
 #include "src/utils/constants.h"
 #include "src/utils/entropy_decoder.h"
 #include "src/utils/segmentation.h"
+#include "src/utils/stack.h"
 #include "src/utils/types.h"
 
 namespace libgav1 {
 namespace {
 
-const uint8_t kMaxVariableTransformTreeDepth = 2;
+constexpr uint8_t kMaxVariableTransformTreeDepth = 2;
+// Max_Tx_Depth array from section 5.11.5 in the spec with the following
+// modification: If the element is not zero, it is subtracted by one. That is
+// the only way in which this array is being used.
+constexpr int kTxDepthCdfIndex[kMaxBlockSizes] = {
+    0, 0, 1, 0, 0, 1, 2, 1, 1, 1, 2, 3, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3};
 
-inline TransformSize GetSquareTransformSize(uint8_t pixels) {
+constexpr TransformSize kMaxTransformSizeRectangle[kMaxBlockSizes] = {
+    kTransformSize4x4,   kTransformSize4x8,   kTransformSize4x16,
+    kTransformSize8x4,   kTransformSize8x8,   kTransformSize8x16,
+    kTransformSize8x32,  kTransformSize16x4,  kTransformSize16x8,
+    kTransformSize16x16, kTransformSize16x32, kTransformSize16x64,
+    kTransformSize32x8,  kTransformSize32x16, kTransformSize32x32,
+    kTransformSize32x64, kTransformSize64x16, kTransformSize64x32,
+    kTransformSize64x64, kTransformSize64x64, kTransformSize64x64,
+    kTransformSize64x64};
+
+TransformSize GetSquareTransformSize(uint8_t pixels) {
   switch (pixels) {
     case 128:
     case 64:
@@ -71,7 +86,7 @@
   }
   const TransformSize max_rect_tx_size = kMaxTransformSizeRectangle[block.size];
   const bool allow_select = !bp.skip || !bp.is_inter;
-  if (block.size <= kBlock4x4 || !allow_select ||
+  if (block.size == kBlock4x4 || !allow_select ||
       frame_header_.tx_mode != kTxModeSelect) {
     return max_rect_tx_size;
   }
@@ -87,16 +102,16 @@
           : 0;
   const auto context = static_cast<int>(top_width >= max_tx_width) +
                        static_cast<int>(left_height >= max_tx_height);
-  const int max_tx_depth = kMaxTransformDepth[block.size];
-  const int cdf_index = (max_tx_depth > 0) ? max_tx_depth - 1 : 0;
-  const int symbol_count = (cdf_index == 0) ? 2 : 3;
+  const int cdf_index = kTxDepthCdfIndex[block.size];
+  const int symbol_count = 3 - static_cast<int>(cdf_index == 0);
   const int tx_depth = reader_.ReadSymbol(
       symbol_decoder_context_.tx_depth_cdf[cdf_index][context], symbol_count);
+  assert(tx_depth < 3);
   TransformSize tx_size = max_rect_tx_size;
-  for (int i = 0; i < tx_depth; ++i) {
-    tx_size = kSplitTransformSize[tx_size];
-  }
-  return tx_size;
+  if (tx_depth == 0) return tx_size;
+  tx_size = kSplitTransformSize[tx_size];
+  if (tx_depth == 1) return tx_size;
+  return kSplitTransformSize[tx_size];
 }
 
 void Tile::ReadVariableTransformTree(const Block& block, int row4x4,
@@ -108,24 +123,22 @@
                              TransformSizeToSquareTransformIndex(max_tx_size)) *
                             6;
 
-  std::vector<TransformTreeNode> stack;
   // Branching factor is 4 and maximum depth is 2. So the maximum stack size
-  // necessary is 8.
-  stack.reserve(8);
-  stack.emplace_back(row4x4, column4x4, tx_size, 0);
+  // necessary is (4 - 1) + 4 = 7.
+  Stack<TransformTreeNode, 7> stack;
+  stack.Push(TransformTreeNode(column4x4, row4x4, tx_size, 0));
 
-  while (!stack.empty()) {
-    TransformTreeNode node = stack.back();
-    stack.pop_back();
-    const int tx_width4x4 = DivideBy4(kTransformWidth[node.tx_size]);
-    const int tx_height4x4 = DivideBy4(kTransformHeight[node.tx_size]);
+  while (!stack.Empty()) {
+    TransformTreeNode node = stack.Pop();
+    const int tx_width4x4 = kTransformWidth4x4[node.tx_size];
+    const int tx_height4x4 = kTransformHeight4x4[node.tx_size];
     if (node.tx_size != kTransformSize4x4 &&
         node.depth != kMaxVariableTransformTreeDepth) {
-      const auto top = static_cast<int>(
-          GetTopTransformWidth(block, node.row4x4, node.column4x4, false) <
-          kTransformWidth[node.tx_size]);
+      const auto top =
+          static_cast<int>(GetTopTransformWidth(block, node.y, node.x, false) <
+                           kTransformWidth[node.tx_size]);
       const auto left = static_cast<int>(
-          GetLeftTransformHeight(block, node.row4x4, node.column4x4, false) <
+          GetLeftTransformHeight(block, node.y, node.x, false) <
           kTransformHeight[node.tx_size]);
       const int context =
           static_cast<int>(max_tx_size > kTransformSize8x8 &&
@@ -136,20 +149,20 @@
       // tx_split.
       if (reader_.ReadSymbol(symbol_decoder_context_.tx_split_cdf[context])) {
         const TransformSize sub_tx_size = kSplitTransformSize[node.tx_size];
-        const int step_width4x4 = DivideBy4(kTransformWidth[sub_tx_size]);
-        const int step_height4x4 = DivideBy4(kTransformHeight[sub_tx_size]);
+        const int step_width4x4 = kTransformWidth4x4[sub_tx_size];
+        const int step_height4x4 = kTransformHeight4x4[sub_tx_size];
         // The loops have to run in reverse order because we use a stack for
         // DFS.
         for (int i = tx_height4x4 - step_height4x4; i >= 0;
              i -= step_height4x4) {
           for (int j = tx_width4x4 - step_width4x4; j >= 0;
                j -= step_width4x4) {
-            if (node.row4x4 + i >= frame_header_.rows4x4 ||
-                node.column4x4 + j >= frame_header_.columns4x4) {
+            if (node.y + i >= frame_header_.rows4x4 ||
+                node.x + j >= frame_header_.columns4x4) {
               continue;
             }
-            stack.emplace_back(node.row4x4 + i, node.column4x4 + j, sub_tx_size,
-                               node.depth + 1);
+            stack.Push(TransformTreeNode(node.x + j, node.y + i, sub_tx_size,
+                                         node.depth + 1));
           }
         }
         continue;
@@ -158,10 +171,10 @@
     // tx_split is false.
     for (int i = 0; i < tx_height4x4; ++i) {
       static_assert(sizeof(TransformSize) == 1, "");
-      memset(&inter_transform_sizes_[node.row4x4 + i][node.column4x4],
-             node.tx_size, tx_width4x4);
+      memset(&inter_transform_sizes_[node.y + i][node.x], node.tx_size,
+             tx_width4x4);
     }
-    block_parameters_holder_.Find(node.row4x4, node.column4x4)->transform_size =
+    block_parameters_holder_.Find(node.y, node.x)->transform_size =
         node.tx_size;
   }
 }
@@ -174,8 +187,8 @@
       bp.is_inter && !bp.skip &&
       !frame_header_.segmentation.lossless[bp.segment_id]) {
     const TransformSize max_tx_size = kMaxTransformSizeRectangle[block.size];
-    const int tx_width4x4 = kTransformWidth[max_tx_size] / 4;
-    const int tx_height4x4 = kTransformHeight[max_tx_size] / 4;
+    const int tx_width4x4 = kTransformWidth4x4[max_tx_size];
+    const int tx_height4x4 = kTransformHeight4x4[max_tx_size];
     for (int row = block.row4x4; row < block.row4x4 + block_height4x4;
          row += tx_height4x4) {
       for (int column = block.column4x4;
diff --git a/libgav1/src/tile/prediction.cc b/libgav1/src/tile/prediction.cc
index b180516..6759838 100644
--- a/libgav1/src/tile/prediction.cc
+++ b/libgav1/src/tile/prediction.cc
@@ -15,6 +15,7 @@
 #include "src/prediction_mask.h"
 #include "src/tile.h"
 #include "src/utils/array_2d.h"
+#include "src/utils/bit_mask_set.h"
 #include "src/utils/block_parameters_holder.h"
 #include "src/utils/common.h"
 #include "src/utils/constants.h"
@@ -27,7 +28,8 @@
 namespace libgav1 {
 namespace {
 
-const int kAngleStep = 3;
+constexpr int kObmcBufferSize = 4096;  // 64x64
+constexpr int kAngleStep = 3;
 constexpr int kPredictionModeToAngle[kIntraPredictionModesUV] = {
     0, 90, 180, 45, 135, 113, 157, 203, 67, 0, 0, 0, 0};
 
@@ -39,33 +41,22 @@
 // The values for directional and dc modes are not used since the left/top
 // requirement for those modes depend on the prediction angle and the type of dc
 // mode.
-constexpr uint8_t kPredictionModeNeeds[kIntraPredictionModesY] = {
-    0,                       // kPredictionModeDc
-    kNeedsTop,               // kPredictionModeVertical
-    kNeedsLeft,              // kPredictionModeHorizontal
-    kNeedsTop,               // kPredictionModeD45
-    kNeedsLeft | kNeedsTop,  // kPredictionModeD135
-    kNeedsLeft | kNeedsTop,  // kPredictionModeD113
-    kNeedsLeft | kNeedsTop,  // kPredictionModeD157
-    kNeedsLeft,              // kPredictionModeD203
-    kNeedsTop,               // kPredictionModeD67
-    kNeedsLeft | kNeedsTop,  // kPredictionModeSmooth
-    kNeedsLeft | kNeedsTop,  // kPredictionModeSmoothVertical
-    kNeedsLeft | kNeedsTop,  // kPredictionModeSmoothHorizontal
-    kNeedsLeft | kNeedsTop   // kPredictionModePaeth
+constexpr BitMaskSet kPredictionModeNeedsMask[kIntraPredictionModesY] = {
+    BitMaskSet(0),                      // kPredictionModeDc
+    BitMaskSet(kNeedsTop),              // kPredictionModeVertical
+    BitMaskSet(kNeedsLeft),             // kPredictionModeHorizontal
+    BitMaskSet(kNeedsTop),              // kPredictionModeD45
+    BitMaskSet(kNeedsLeft, kNeedsTop),  // kPredictionModeD135
+    BitMaskSet(kNeedsLeft, kNeedsTop),  // kPredictionModeD113
+    BitMaskSet(kNeedsLeft, kNeedsTop),  // kPredictionModeD157
+    BitMaskSet(kNeedsLeft),             // kPredictionModeD203
+    BitMaskSet(kNeedsTop),              // kPredictionModeD67
+    BitMaskSet(kNeedsLeft, kNeedsTop),  // kPredictionModeSmooth
+    BitMaskSet(kNeedsLeft, kNeedsTop),  // kPredictionModeSmoothVertical
+    BitMaskSet(kNeedsLeft, kNeedsTop),  // kPredictionModeSmoothHorizontal
+    BitMaskSet(kNeedsLeft, kNeedsTop)   // kPredictionModePaeth
 };
 
-const int kBlendFromAbove = 0;
-const int kBlendFromLeft = 1;
-constexpr uint8_t kObmcMask2[2] = {45, 64};
-constexpr uint8_t kObmcMask4[4] = {39, 50, 59, 64};
-constexpr uint8_t kObmcMask8[8] = {36, 42, 48, 53, 57, 61, 64, 64};
-constexpr uint8_t kObmcMask16[16] = {34, 37, 40, 43, 46, 49, 52, 54,
-                                     56, 58, 60, 61, 64, 64, 64, 64};
-constexpr uint8_t kObmcMask32[32] = {33, 35, 36, 38, 40, 41, 43, 44, 45, 47, 48,
-                                     50, 51, 52, 53, 55, 56, 57, 58, 59, 60, 60,
-                                     61, 62, 64, 64, 64, 64, 64, 64, 64, 64};
-
 int16_t GetDirectionalIntraPredictorDerivative(const int angle) {
   assert(angle >= 3);
   assert(angle <= 87);
@@ -98,21 +89,6 @@
   }
 }
 
-const uint8_t* GetObmcMask(int length) {
-  switch (length) {
-    case 2:
-      return kObmcMask2;
-    case 4:
-      return kObmcMask4;
-    case 8:
-      return kObmcMask8;
-    case 16:
-      return kObmcMask16;
-    default:
-      return kObmcMask32;
-  }
-}
-
 // 7.11.2.9.
 int GetIntraEdgeFilterStrength(int width, int height, int filter_type,
                                int delta) {
@@ -236,16 +212,18 @@
 }
 
 // 7.11.3.2.
-void SetInterRoundingBits(const bool is_compound, const int bitdepth,
-                          uint8_t round_bits[2],
-                          uint8_t* const post_round_bits) {
+void GetInterRoundingBits(const bool is_compound, const int bitdepth,
+                          uint8_t round_bits[2]) {
   round_bits[0] = 3;
   round_bits[1] = is_compound ? 7 : 11;
+#if LIBGAV1_MAX_BITDEPTH == 12
   if (bitdepth == 12) {
     round_bits[0] += 2;
     if (!is_compound) round_bits[1] -= 2;
   }
-  *post_round_bits = 2 * kFilterBits - round_bits[0] - round_bits[1];
+#else
+  static_cast<void>(bitdepth);
+#endif
 }
 
 uint8_t* GetStartPoint(Array2DView<uint8_t>* const buffer, const int plane,
@@ -262,31 +240,36 @@
   return &buffer[plane][y][x];
 }
 
-inline int GetPixelPositionFromHighScale(int start, int step, int offset) {
+int GetPixelPositionFromHighScale(int start, int step, int offset) {
   return (start + step * offset) >> kScaleSubPixelBits;
 }
 
-}  // namespace
-
-bool Tile::BlockDecoded(const Block& block, Plane plane, int row4x4,
-                        int column4x4, bool has_top_or_left) const {
-  if (!has_top_or_left) return false;
-  if (row4x4 >= 0 && column4x4 >= 0) {
-    return block.sb_buffer->block_decoded[plane][row4x4][column4x4];
-  }
-  if (row4x4 < 0) {
-    return column4x4 < (block.sb_buffer->block_decoded_width_threshold >>
-                        SubsamplingX(plane));
-  }
-  assert(column4x4 < 0);
-  const int sb_size4x4 =
-      kNum4x4BlocksWide[sequence_header_.use_128x128_superblock ? kBlock128x128
-                                                                : kBlock64x64];
-  return row4x4 < (block.sb_buffer->block_decoded_height_threshold >>
-                   SubsamplingY(plane)) &&
-         row4x4 != (sb_size4x4 >> SubsamplingY(plane));
+dsp::MaskBlendFunc GetMaskBlendFunc(const dsp::Dsp& dsp,
+                                    InterIntraMode inter_intra_mode,
+                                    bool is_wedge_inter_intra,
+                                    int subsampling_x, int subsampling_y) {
+  const int is_inter_intra =
+      static_cast<int>(inter_intra_mode != kNumInterIntraModes);
+  return (is_inter_intra == 1 && !is_wedge_inter_intra)
+             ? dsp.mask_blend[0][is_inter_intra]
+             : dsp.mask_blend[subsampling_x + subsampling_y][is_inter_intra];
 }
 
+void PopulatePredictionMaskFromWedgeMask(const uint8_t* wedge_mask,
+                                         int wedge_mask_stride,
+                                         int prediction_width,
+                                         int prediction_height,
+                                         uint8_t* prediction_mask,
+                                         int prediction_mask_stride) {
+  for (int y = 0; y < prediction_height; ++y) {
+    memcpy(prediction_mask, wedge_mask, prediction_width);
+    prediction_mask += prediction_mask_stride;
+    wedge_mask += wedge_mask_stride;
+  }
+}
+
+}  // namespace
+
 template <typename Pixel>
 void Tile::IntraPrediction(const Block& block, Plane plane, int x, int y,
                            bool has_left, bool has_top, bool has_top_right,
@@ -294,14 +277,12 @@
                            TransformSize tx_size) {
   const int width = 1 << kTransformWidthLog2[tx_size];
   const int height = 1 << kTransformHeightLog2[tx_size];
-  const int x_shift =
-      (plane == kPlaneY) ? 0 : sequence_header_.color_config.subsampling_x;
-  const int y_shift =
-      (plane == kPlaneY) ? 0 : sequence_header_.color_config.subsampling_y;
+  const int x_shift = subsampling_x_[plane];
+  const int y_shift = subsampling_y_[plane];
   const int max_x = (MultiplyBy4(frame_header_.columns4x4) >> x_shift) - 1;
   const int max_y = (MultiplyBy4(frame_header_.rows4x4) >> y_shift) - 1;
-  alignas(16) Pixel top_row_data[160] = {};
-  alignas(16) Pixel left_column_data[160] = {};
+  alignas(kMaxAlignment) Pixel top_row_data[160] = {};
+  alignas(kMaxAlignment) Pixel left_column_data[160] = {};
   // Some predictors use |top_row_data| and |left_column_data| with a negative
   // offset to access pixels to the top-left of the current block. So have some
   // space before the arrays to allow populating those without having to move
@@ -322,7 +303,7 @@
                     kAngleStep
           : 0;
   const bool needs_top = use_filter_intra ||
-                         ((kPredictionModeNeeds[mode] & kNeedsTop) != 0) ||
+                         kPredictionModeNeedsMask[mode].Contains(kNeedsTop) ||
                          (is_directional_mode && prediction_angle < 180) ||
                          (mode == kPredictionModeDc && has_top);
   Array2DView<Pixel> buffer(buffer_[plane].rows(),
@@ -346,7 +327,7 @@
     }
   }
   const bool needs_left = use_filter_intra ||
-                          ((kPredictionModeNeeds[mode] & kNeedsLeft) != 0) ||
+                          kPredictionModeNeedsMask[mode].Contains(kNeedsLeft) ||
                           (is_directional_mode && prediction_angle > 90) ||
                           (mode == kPredictionModeDc && has_left);
   if (needs_left) {
@@ -404,7 +385,11 @@
                                               TransformSize tx_size);
 #endif
 
-bool Tile::UsesSmoothPrediction(int row, int column, Plane plane) const {
+constexpr BitMaskSet kPredictionModeSmoothMask(kPredictionModeSmooth,
+                                               kPredictionModeSmoothHorizontal,
+                                               kPredictionModeSmoothVertical);
+
+bool Tile::IsSmoothPrediction(int row, int column, Plane plane) const {
   const BlockParameters& bp = *block_parameters_holder_.Find(row, column);
   PredictionMode mode;
   if (plane == kPlaneY) {
@@ -413,14 +398,12 @@
     if (bp.reference_frame[0] > kReferenceFrameIntra) return false;
     mode = bp.uv_mode;
   }
-  return mode == kPredictionModeSmooth ||
-         mode == kPredictionModeSmoothHorizontal ||
-         mode == kPredictionModeSmoothVertical;
+  return kPredictionModeSmoothMask.Contains(mode);
 }
 
 int Tile::GetIntraEdgeFilterType(const Block& block, Plane plane) const {
-  const int subsampling_x = SubsamplingX(plane);
-  const int subsampling_y = SubsamplingY(plane);
+  const int subsampling_x = subsampling_x_[plane];
+  const int subsampling_y = subsampling_y_[plane];
   if ((plane == kPlaneY && block.top_available) ||
       (plane != kPlaneY && block.TopAvailableChroma())) {
     const int row =
@@ -429,7 +412,7 @@
     const int column =
         block.column4x4 +
         static_cast<int>(subsampling_x != 0 && (block.column4x4 & 1) == 0);
-    if (UsesSmoothPrediction(row, column, plane)) return 1;
+    if (IsSmoothPrediction(row, column, plane)) return 1;
   }
   if ((plane == kPlaneY && block.left_available) ||
       (plane != kPlaneY && block.LeftAvailableChroma())) {
@@ -438,7 +421,7 @@
     const int column =
         block.column4x4 - 1 -
         static_cast<int>(subsampling_x != 0 && (block.column4x4 & 1) != 0);
-    if (UsesSmoothPrediction(row, column, plane)) return 1;
+    if (IsSmoothPrediction(row, column, plane)) return 1;
   }
   return 0;
 }
@@ -544,6 +527,8 @@
                             buffer_[plane].columns() / sizeof(Pixel),
                             reinterpret_cast<Pixel*>(&buffer_[plane][0][0]));
   for (int row = 0; row < tx_height; ++row) {
+    assert(block.bp->prediction_parameters
+               ->color_index_map[plane_type][y4 + row] != nullptr);
     for (int column = 0; column < tx_width; ++column) {
       buffer[start_y + row][start_x + column] =
           palette[block.bp->prediction_parameters
@@ -565,20 +550,20 @@
 void Tile::ChromaFromLumaPrediction(const Block& block, const Plane plane,
                                     const int start_x, const int start_y,
                                     const TransformSize tx_size) {
-  const int subsampling_x = SubsamplingX(plane);
-  const int subsampling_y = SubsamplingY(plane);
+  const int subsampling_x = subsampling_x_[plane];
+  const int subsampling_y = subsampling_y_[plane];
   const PredictionParameters& prediction_parameters =
       *block.bp->prediction_parameters;
-  const int luma_y = start_y << subsampling_y;
-  const int luma_x = start_x << subsampling_x;
   Array2DView<Pixel> y_buffer(
       buffer_[kPlaneY].rows(), buffer_[kPlaneY].columns() / sizeof(Pixel),
       reinterpret_cast<Pixel*>(&buffer_[kPlaneY][0][0]));
   if (!block.sb_buffer->cfl_luma_buffer_valid) {
+    const int luma_x = start_x << subsampling_x;
+    const int luma_y = start_y << subsampling_y;
     dsp_.cfl_subsamplers[tx_size][subsampling_x + subsampling_y](
         block.sb_buffer->cfl_luma_buffer,
-        prediction_parameters.max_luma_width - (start_x << subsampling_x),
-        prediction_parameters.max_luma_height - (start_y << subsampling_y),
+        prediction_parameters.max_luma_width - luma_x,
+        prediction_parameters.max_luma_height - luma_y,
         reinterpret_cast<uint8_t*>(&y_buffer[luma_y][luma_x]),
         buffer_[kPlaneY].columns());
     block.sb_buffer->cfl_luma_buffer_valid = true;
@@ -602,15 +587,15 @@
     const TransformSize tx_size);
 #endif
 
-void Tile::InterIntraPrediction(
+bool Tile::InterIntraPrediction(
     uint16_t* prediction[2], const ptrdiff_t prediction_stride,
     const uint8_t* const prediction_mask,
     const ptrdiff_t prediction_mask_stride,
     const PredictionParameters& prediction_parameters,
     const int prediction_width, const int prediction_height,
-    const int subsampling_x, const int subsampling_y,
-    const uint8_t post_round_bits, uint8_t* const dest,
+    const int subsampling_x, const int subsampling_y, uint8_t* const dest,
     const ptrdiff_t dest_stride) {
+  assert(prediction_mask != nullptr);
   assert(prediction_parameters.compound_prediction_type ==
              kCompoundPredictionTypeIntra ||
          prediction_parameters.compound_prediction_type ==
@@ -628,7 +613,7 @@
     if (!intra_prediction.Reset(prediction_height, prediction_width)) {
       LIBGAV1_DLOG(ERROR,
                    "Can't allocate memory for the intra prediction block.");
-      return;
+      return false;
     }
     uint8_t* dest_ptr = dest;
     for (int r = 0; r < prediction_height; ++r) {
@@ -643,70 +628,127 @@
     prediction[1] = reinterpret_cast<uint16_t*>(dest);
     intra_stride = dest_stride / sizeof(uint16_t);
   }
-  dsp_.mask_blend(prediction[0], prediction_stride, prediction[1], intra_stride,
-                  prediction_mask, prediction_mask_stride, prediction_width,
-                  prediction_height, subsampling_x, subsampling_y,
-                  prediction_parameters.inter_intra_mode != kNumInterIntraModes,
-                  prediction_parameters.is_wedge_inter_intra, post_round_bits,
-                  dest, dest_stride);
+  GetMaskBlendFunc(dsp_, prediction_parameters.inter_intra_mode,
+                   prediction_parameters.is_wedge_inter_intra, subsampling_x,
+                   subsampling_y)(prediction[0], prediction_stride,
+                                  prediction[1], intra_stride, prediction_mask,
+                                  prediction_mask_stride, prediction_width,
+                                  prediction_height, dest, dest_stride);
+  return true;
 }
 
-void Tile::CompoundInterPrediction(
-    const Block& block, uint16_t* prediction[2],
-    const ptrdiff_t prediction_stride, const ptrdiff_t prediction_mask_stride,
-    const int prediction_width, const int prediction_height, const Plane plane,
-    const int subsampling_x, const int subsampling_y, const int bitdepth,
-    const int candidate_row, const int candidate_column, uint8_t* dest,
-    const ptrdiff_t dest_stride, const uint8_t post_round_bits) {
+bool Tile::AllocatePredictionMask(SuperBlockBuffer* sb_buffer) {
+  if (sb_buffer->prediction_mask != nullptr) {
+    return true;
+  }
+  sb_buffer->prediction_mask = MakeAlignedUniquePtr<uint8_t>(
+      16, kMaxSuperBlockSizeInPixels * kMaxSuperBlockSizeInPixels);
+  if (sb_buffer->prediction_mask == nullptr) {
+    LIBGAV1_DLOG(ERROR, "Allocation of prediction_mask failed.");
+    return false;
+  }
+  return true;
+}
+
+bool Tile::CompoundInterPrediction(
+    const Block& block, const ptrdiff_t prediction_stride,
+    const ptrdiff_t prediction_mask_stride, const int prediction_width,
+    const int prediction_height, const Plane plane, const int subsampling_x,
+    const int subsampling_y, const int bitdepth, const int candidate_row,
+    const int candidate_column, uint8_t* dest, const ptrdiff_t dest_stride) {
   const PredictionParameters& prediction_parameters =
       *block.bp->prediction_parameters;
+  uint16_t* prediction[2] = {block.sb_buffer->prediction_buffer[0].get(),
+                             block.sb_buffer->prediction_buffer[1].get()};
   switch (prediction_parameters.compound_prediction_type) {
     case kCompoundPredictionTypeWedge:
-      dsp_.mask_blend(
+      GetMaskBlendFunc(dsp_, prediction_parameters.inter_intra_mode,
+                       prediction_parameters.is_wedge_inter_intra,
+                       subsampling_x, subsampling_y)(
           prediction[0], prediction_stride, prediction[1], prediction_stride,
-          block.sb_buffer->prediction_mask, prediction_mask_stride,
-          prediction_width, prediction_height, subsampling_x, subsampling_y,
-          prediction_parameters.inter_intra_mode != kNumInterIntraModes,
-          prediction_parameters.is_wedge_inter_intra, post_round_bits, dest,
-          dest_stride);
+          block.sb_buffer->prediction_mask.get(), prediction_mask_stride,
+          prediction_width, prediction_height, dest, dest_stride);
       break;
     case kCompoundPredictionTypeDiffWeighted:
       if (plane == kPlaneY) {
+        if (!AllocatePredictionMask(block.sb_buffer)) {
+          return false;
+        }
         GenerateWeightMask(
             prediction[0], prediction_stride, prediction[1], prediction_stride,
-            prediction_parameters.mask_is_inverse, post_round_bits,
-            prediction_width, prediction_height, bitdepth,
-            block.sb_buffer->prediction_mask, prediction_mask_stride);
+            prediction_parameters.mask_is_inverse, prediction_width,
+            prediction_height, bitdepth, block.sb_buffer->prediction_mask.get(),
+            prediction_mask_stride);
       }
-      dsp_.mask_blend(
+      GetMaskBlendFunc(dsp_, prediction_parameters.inter_intra_mode,
+                       prediction_parameters.is_wedge_inter_intra,
+                       subsampling_x, subsampling_y)(
           prediction[0], prediction_stride, prediction[1], prediction_stride,
-          block.sb_buffer->prediction_mask, prediction_mask_stride,
-          prediction_width, prediction_height, subsampling_x, subsampling_y,
-          prediction_parameters.inter_intra_mode != kNumInterIntraModes,
-          prediction_parameters.is_wedge_inter_intra, post_round_bits, dest,
-          dest_stride);
+          block.sb_buffer->prediction_mask.get(), prediction_mask_stride,
+          prediction_width, prediction_height, dest, dest_stride);
       break;
     case kCompoundPredictionTypeDistance:
       DistanceWeightedPrediction(
           prediction[0], prediction_stride, prediction[1], prediction_stride,
           prediction_width, prediction_height, candidate_row, candidate_column,
-          dest, dest_stride, post_round_bits);
+          dest, dest_stride);
       break;
     case kCompoundPredictionTypeAverage:
       dsp_.average_blend(prediction[0], prediction_stride, prediction[1],
-                         prediction_stride, post_round_bits, prediction_width,
-                         prediction_height, dest, dest_stride);
+                         prediction_stride, prediction_width, prediction_height,
+                         dest, dest_stride);
       break;
     default:
       assert(false && "This is not a compound type.\n");
-      return;
+      return false;
   }
+  return true;
 }
 
-bool Tile::InterPrediction(const Block& block, Plane plane, int x, int y,
-                           int prediction_width, int prediction_height,
-                           int candidate_row, int candidate_column,
-                           bool* const is_local_valid,
+GlobalMotion* Tile::GetWarpParams(
+    const Block& block, const Plane plane, const int prediction_width,
+    const int prediction_height,
+    const PredictionParameters& prediction_parameters,
+    const ReferenceFrameType reference_type, bool* const is_local_valid,
+    GlobalMotion* const global_motion_params,
+    GlobalMotion* const local_warp_params) const {
+  if (prediction_width < 8 || prediction_height < 8 ||
+      frame_header_.force_integer_mv == 1) {
+    return nullptr;
+  }
+  if (plane == kPlaneY) {
+    *is_local_valid =
+        prediction_parameters.motion_mode == kMotionModeLocalWarp &&
+        WarpEstimation(
+            prediction_parameters.num_warp_samples, DivideBy4(prediction_width),
+            DivideBy4(prediction_height), block.row4x4, block.column4x4,
+            block.bp->mv[0], prediction_parameters.warp_estimate_candidates,
+            local_warp_params) &&
+        SetupShear(local_warp_params);
+  }
+  if (prediction_parameters.motion_mode == kMotionModeLocalWarp &&
+      *is_local_valid) {
+    return local_warp_params;
+  }
+  if (!IsScaled(reference_type)) {
+    GlobalMotionTransformationType global_motion_type =
+        (reference_type != kReferenceFrameIntra)
+            ? global_motion_params->type
+            : kNumGlobalMotionTransformationTypes;
+    const bool is_global_valid =
+        IsGlobalMvBlock(block.bp->y_mode, global_motion_type, block.size) &&
+        SetupShear(global_motion_params);
+    // Valid global motion type implies reference type can't be intra.
+    assert(!is_global_valid || reference_type != kReferenceFrameIntra);
+    if (is_global_valid) return global_motion_params;
+  }
+  return nullptr;
+}
+
+bool Tile::InterPrediction(const Block& block, const Plane plane, const int x,
+                           const int y, const int prediction_width,
+                           const int prediction_height, int candidate_row,
+                           int candidate_column, bool* const is_local_valid,
                            GlobalMotion* const local_warp_params) {
   const int bitdepth = sequence_header_.color_config.bitdepth;
   const BlockParameters& bp = *block.bp;
@@ -716,73 +758,54 @@
       bp_reference.reference_frame[1] > kReferenceFrameIntra;
   const bool is_inter_intra =
       bp.is_inter && bp.reference_frame[1] == kReferenceFrameIntra;
-  const ptrdiff_t prediction_stride = kMaxSuperBlockSizeInPixels;
-  AlignedUniquePtr<uint16_t> prediction[2] = {
-      MakeAlignedUniquePtr<uint16_t>(
-          8, kMaxSuperBlockSizeInPixels * prediction_stride),
-      AlignedUniquePtr<uint16_t>()};
-  if (prediction[0] == nullptr) {
-    LIBGAV1_DLOG(ERROR,
-                 "Can't allocate memory for the first prediction block.");
-    return false;
+  const ptrdiff_t prediction_stride = prediction_width;
+
+  const size_t prediction_buffer_size = prediction_height * prediction_stride;
+  if (block.sb_buffer->prediction_buffer_size[0] < prediction_buffer_size) {
+    block.sb_buffer->prediction_buffer[0] =
+        MakeAlignedUniquePtr<uint16_t>(8, prediction_buffer_size);
+    if (block.sb_buffer->prediction_buffer[0] == nullptr) {
+      block.sb_buffer->prediction_buffer_size[0] = 0;
+      LIBGAV1_DLOG(ERROR,
+                   "Can't allocate memory for the first prediction block.");
+      return false;
+    }
+    block.sb_buffer->prediction_buffer_size[0] = prediction_buffer_size;
   }
-  if (is_compound) {
-    prediction[1] = MakeAlignedUniquePtr<uint16_t>(
-        8, kMaxSuperBlockSizeInPixels * prediction_stride);
-    if (prediction[1] == nullptr) {
+  if (is_compound &&
+      block.sb_buffer->prediction_buffer_size[1] < prediction_buffer_size) {
+    block.sb_buffer->prediction_buffer[1] =
+        MakeAlignedUniquePtr<uint16_t>(8, prediction_buffer_size);
+    if (block.sb_buffer->prediction_buffer[1] == nullptr) {
+      block.sb_buffer->prediction_buffer_size[1] = 0;
       LIBGAV1_DLOG(ERROR,
                    "Can't allocate memory for the second prediction block.");
       return false;
     }
+    block.sb_buffer->prediction_buffer_size[1] = prediction_buffer_size;
   }
+
+  const PredictionParameters& prediction_parameters =
+      *block.bp->prediction_parameters;
   uint8_t* const dest = GetStartPoint(buffer_, plane, x, y, bitdepth);
   const ptrdiff_t dest_stride = buffer_[plane].columns();  // In bytes.
   uint8_t round_bits[2];
-  uint8_t post_round_bits;
-  SetInterRoundingBits(is_compound, sequence_header_.color_config.bitdepth,
-                       round_bits, &post_round_bits);
-  const PredictionParameters& prediction_parameters =
-      *block.bp->prediction_parameters;
-  if (plane == kPlaneY) {
-    *is_local_valid =
-        prediction_parameters.motion_mode == kMotionModeLocalWarp &&
-        WarpEstimation(
-            prediction_parameters.num_warp_samples, DivideBy4(prediction_width),
-            DivideBy4(prediction_height), block.row4x4, block.column4x4,
-            bp.mv[0], prediction_parameters.warp_estimate_candidates,
-            local_warp_params) &&
-        SetupShear(local_warp_params);
-  }
-
+  GetInterRoundingBits(is_compound, sequence_header_.color_config.bitdepth,
+                       round_bits);
   for (int index = 0; index < 1 + static_cast<int>(is_compound); ++index) {
     const ReferenceFrameType reference_type =
         bp_reference.reference_frame[index];
     GlobalMotion global_motion_params =
         frame_header_.global_motion[reference_type];
-    GlobalMotionTransformationType global_motion_type =
-        (reference_type != kReferenceFrameIntra)
-            ? global_motion_params.type
-            : kNumGlobalMotionTransformationTypes;
-    const bool is_global_valid =
-        IsGlobalMvBlock(bp.y_mode, global_motion_type, block.size) &&
-        SetupShear(&global_motion_params);
-    // Valid global motion type implies reference type can't be intra.
-    assert(!is_global_valid || reference_type != kReferenceFrameIntra);
-    GlobalMotion* warp_params = nullptr;
-    if (prediction_width < 8 || prediction_height < 8 ||
-        frame_header_.force_integer_mv == 1) {
-      warp_params = nullptr;
-    } else if (prediction_parameters.motion_mode == kMotionModeLocalWarp &&
-               *is_local_valid) {
-      warp_params = local_warp_params;
-    } else if (is_global_valid && !IsScaled(reference_type)) {
-      warp_params = &global_motion_params;
-    }
+    GlobalMotion* warp_params =
+        GetWarpParams(block, plane, prediction_width, prediction_height,
+                      prediction_parameters, reference_type, is_local_valid,
+                      &global_motion_params, local_warp_params);
     if (warp_params != nullptr) {
-      if (!BlockWarpProcess(block, plane, index, prediction_width,
-                            prediction_height, prediction[index].get(),
-                            prediction_stride, warp_params, round_bits,
-                            is_compound, is_inter_intra, dest, dest_stride)) {
+      if (!BlockWarpProcess(block, plane, index, x, y, prediction_width,
+                            prediction_height, prediction_stride, warp_params,
+                            round_bits, is_compound, is_inter_intra, dest,
+                            dest_stride)) {
         return false;
       }
     } else {
@@ -791,17 +814,17 @@
               ? -1
               : frame_header_.reference_frame_index[reference_type -
                                                     kReferenceFrameLast];
-      BlockInterPrediction(plane, reference_index, bp_reference.mv[index], x, y,
-                           prediction_width, prediction_height, candidate_row,
-                           candidate_column, prediction[index].get(),
-                           prediction_stride, round_bits, is_compound,
-                           is_inter_intra, dest, dest_stride);
+      BlockInterPrediction(
+          plane, reference_index, bp_reference.mv[index], x, y,
+          prediction_width, prediction_height, candidate_row, candidate_column,
+          block.sb_buffer->prediction_buffer[index].get(), prediction_stride,
+          round_bits, is_compound, is_inter_intra, dest, dest_stride);
     }
   }
 
   const ptrdiff_t prediction_mask_stride = kMaxSuperBlockSizeInPixels;
-  const int subsampling_x = SubsamplingX(plane);
-  const int subsampling_y = SubsamplingY(plane);
+  const int subsampling_x = subsampling_x_[plane];
+  const int subsampling_y = subsampling_y_[plane];
   if (prediction_parameters.compound_prediction_type ==
           kCompoundPredictionTypeWedge &&
       plane == kPlaneY) {
@@ -815,29 +838,32 @@
     const int offset = block_size_index * wedge_mask_stride_3 +
                        prediction_parameters.wedge_sign * wedge_mask_stride_2 +
                        prediction_parameters.wedge_index * wedge_mask_stride_1;
-    const int stride = kWedgeMaskMasterSize;
-    uint8_t* mask_ptr = block.sb_buffer->prediction_mask;
-    const uint8_t* wedge_mask_ptr = &wedge_masks_[offset];
-    for (int y = 0; y < prediction_height; ++y) {
-      memcpy(mask_ptr, wedge_mask_ptr, prediction_width);
-      mask_ptr += kMaxSuperBlockSizeInPixels;
-      wedge_mask_ptr += stride;
+    if (!AllocatePredictionMask(block.sb_buffer)) {
+      return false;
     }
+    PopulatePredictionMaskFromWedgeMask(
+        &wedge_masks_[offset], kWedgeMaskMasterSize, prediction_width,
+        prediction_height, block.sb_buffer->prediction_mask.get(),
+        kMaxSuperBlockSizeInPixels);
   } else if (prediction_parameters.compound_prediction_type ==
              kCompoundPredictionTypeIntra) {
+    if (plane == kPlaneY) {
+      if (!AllocatePredictionMask(block.sb_buffer)) {
+        return false;
+      }
+    }
     GenerateInterIntraMask(prediction_parameters.inter_intra_mode,
                            prediction_width, prediction_height,
-                           block.sb_buffer->prediction_mask,
+                           block.sb_buffer->prediction_mask.get(),
                            prediction_mask_stride);
   }
 
-  uint16_t* prediction_ptr[2] = {prediction[0].get(), prediction[1].get()};
+  bool ok = true;
   if (is_compound) {
-    CompoundInterPrediction(
-        block, prediction_ptr, prediction_stride, prediction_mask_stride,
-        prediction_width, prediction_height, plane, subsampling_x,
-        subsampling_y, bitdepth, candidate_row, candidate_column, dest,
-        dest_stride, post_round_bits);
+    ok = CompoundInterPrediction(
+        block, prediction_stride, prediction_mask_stride, prediction_width,
+        prediction_height, plane, subsampling_x, subsampling_y, bitdepth,
+        candidate_row, candidate_column, dest, dest_stride);
   } else {
     if (prediction_parameters.motion_mode == kMotionModeObmc) {
       // Obmc mode is allowed only for single reference (!is_compound).
@@ -845,14 +871,17 @@
                      round_bits);
     } else if (is_inter_intra) {
       // InterIntra and obmc must be mutually exclusive.
-      InterIntraPrediction(prediction_ptr, prediction_stride,
-                           block.sb_buffer->prediction_mask,
-                           prediction_mask_stride, prediction_parameters,
-                           prediction_width, prediction_height, subsampling_x,
-                           subsampling_y, post_round_bits, dest, dest_stride);
+      uint16_t* prediction_ptr[2] = {
+          block.sb_buffer->prediction_buffer[0].get(),
+          block.sb_buffer->prediction_buffer[1].get()};
+      ok = InterIntraPrediction(
+          prediction_ptr, prediction_stride,
+          block.sb_buffer->prediction_mask.get(), prediction_mask_stride,
+          prediction_parameters, prediction_width, prediction_height,
+          subsampling_x, subsampling_y, dest, dest_stride);
     }
   }
-  return true;
+  return ok;
 }
 
 void Tile::ObmcBlockPrediction(const MotionVector& mv, const Plane plane,
@@ -860,41 +889,37 @@
                                const int height, const int x, const int y,
                                const int candidate_row,
                                const int candidate_column,
-                               const uint8_t* const mask,
-                               const int blending_direction,
+                               const ObmcDirection blending_direction,
                                const uint8_t* const round_bits) {
-  uint16_t
-      obmc_prediction[kMaxSuperBlockSizeInPixels *
-                      (kMaxSuperBlockSizeInPixels + 2 * kRestorationBorder)];
-  const int obmc_prediction_stride = width + 2 * kRestorationBorder;
   const int bitdepth = sequence_header_.color_config.bitdepth;
   // Obmc's prediction needs to be clipped before blending with above/left
   // prediction blocks.
-  uint8_t obmc_clipped_prediction[2 * kMaxSuperBlockSizeInPixels *
-                                  (kMaxSuperBlockSizeInPixels +
-                                   2 * kRestorationBorder)];
+  uint8_t obmc_clipped_prediction[kObmcBufferSize
+#if LIBGAV1_MAX_BITDEPTH >= 10
+                                  * 2
+#endif
+  ];
   const ptrdiff_t obmc_clipped_prediction_stride =
-      (bitdepth == 8) ? obmc_prediction_stride
-                      : obmc_prediction_stride * sizeof(uint16_t);
+      (bitdepth == 8) ? width : width * sizeof(uint16_t);
   BlockInterPrediction(plane, reference_frame_index, mv, x, y, width, height,
-                       candidate_row, candidate_column, obmc_prediction,
-                       obmc_prediction_stride, round_bits, false, false,
-                       obmc_clipped_prediction, obmc_clipped_prediction_stride);
+                       candidate_row, candidate_column, nullptr, width,
+                       round_bits, false, false, obmc_clipped_prediction,
+                       obmc_clipped_prediction_stride);
 
   uint8_t* const prediction = GetStartPoint(buffer_, plane, x, y, bitdepth);
   const ptrdiff_t prediction_stride = buffer_[plane].columns();
-  dsp_.obmc_blend(prediction, prediction_stride, width, height,
-                  blending_direction, mask, obmc_clipped_prediction,
-                  obmc_clipped_prediction_stride);
+  dsp_.obmc_blend[blending_direction](prediction, prediction_stride, width,
+                                      height, obmc_clipped_prediction,
+                                      obmc_clipped_prediction_stride);
 }
 
 void Tile::ObmcPrediction(const Block& block, const Plane plane,
                           const int width, const int height,
                           const uint8_t* const round_bits) {
-  const int subsampling_x = SubsamplingX(plane);
-  const int subsampling_y = SubsamplingY(plane);
+  const int subsampling_x = subsampling_x_[plane];
+  const int subsampling_y = subsampling_y_[plane];
   const BlockSize plane_block_size =
-      kPlaneResidualSize[block.size][SubsamplingX(plane)][SubsamplingY(plane)];
+      kPlaneResidualSize[block.size][subsampling_x][subsampling_y];
   assert(plane_block_size != kBlockInvalid);
   const int num4x4_wide = kNum4x4BlocksWide[block.size];
   const int num4x4_high = kNum4x4BlocksHigh[block.size];
@@ -906,6 +931,7 @@
     const int candidate_row = block.row4x4 - 1;
     const int block_start_y = MultiplyBy4(block.row4x4) >> subsampling_y;
     int column4x4 = block.column4x4;
+    const int prediction_height = std::min(height >> 1, 32 >> subsampling_y);
     for (int i = 0, step; i < num_limit && column4x4 < column4x4_max;
          column4x4 += step) {
       const int candidate_column = column4x4 | 1;
@@ -920,14 +946,12 @@
                                                 kReferenceFrameLast];
         const int prediction_width =
             std::min(width, MultiplyBy4(step) >> subsampling_x);
-        const int prediction_height =
-            std::min(height >> 1, 32 >> subsampling_y);
-        const uint8_t* mask = GetObmcMask(prediction_height);
-        ObmcBlockPrediction(
-            bp_top.mv[0], plane, candidate_reference_frame_index,
-            prediction_width, prediction_height,
-            MultiplyBy4(column4x4) >> subsampling_x, block_start_y,
-            candidate_row, candidate_column, mask, kBlendFromAbove, round_bits);
+        ObmcBlockPrediction(bp_top.mv[0], plane,
+                            candidate_reference_frame_index, prediction_width,
+                            prediction_height,
+                            MultiplyBy4(column4x4) >> subsampling_x,
+                            block_start_y, candidate_row, candidate_column,
+                            kObmcDirectionVertical, round_bits);
       }
     }
   }
@@ -939,6 +963,7 @@
     const int candidate_column = block.column4x4 - 1;
     int row4x4 = block.row4x4;
     const int block_start_x = MultiplyBy4(block.column4x4) >> subsampling_x;
+    const int prediction_width = std::min(width >> 1, 32 >> subsampling_x);
     for (int i = 0, step; i < num_limit && row4x4 < row4x4_max;
          row4x4 += step) {
       const int candidate_row = row4x4 | 1;
@@ -951,15 +976,13 @@
         const int candidate_reference_frame_index =
             frame_header_.reference_frame_index[bp_left.reference_frame[0] -
                                                 kReferenceFrameLast];
-        const int prediction_width = std::min(width >> 1, 32 >> subsampling_x);
         const int prediction_height =
             std::min(height, MultiplyBy4(step) >> subsampling_y);
-        const uint8_t* mask = GetObmcMask(prediction_width);
-        ObmcBlockPrediction(bp_left.mv[0], plane,
-                            candidate_reference_frame_index, prediction_width,
-                            prediction_height, block_start_x,
-                            MultiplyBy4(row4x4) >> subsampling_y, candidate_row,
-                            candidate_column, mask, kBlendFromLeft, round_bits);
+        ObmcBlockPrediction(
+            bp_left.mv[0], plane, candidate_reference_frame_index,
+            prediction_width, prediction_height, block_start_x,
+            MultiplyBy4(row4x4) >> subsampling_y, candidate_row,
+            candidate_column, kObmcDirectionHorizontal, round_bits);
       }
     }
   }
@@ -969,7 +992,7 @@
     uint16_t* prediction_0, ptrdiff_t prediction_stride_0,
     uint16_t* prediction_1, ptrdiff_t prediction_stride_1, const int width,
     const int height, const int candidate_row, const int candidate_column,
-    uint8_t* dest, ptrdiff_t dest_stride, const uint8_t post_round_bits) {
+    uint8_t* dest, ptrdiff_t dest_stride) {
   int distance[2];
   int weight[2];
   for (int reference = 0; reference < 2; ++reference) {
@@ -987,9 +1010,9 @@
   }
   GetDistanceWeights(distance, weight);
 
-  dsp_.distance_weighted_blend(
-      prediction_0, prediction_stride_0, prediction_1, prediction_stride_1,
-      weight[0], weight[1], post_round_bits, width, height, dest, dest_stride);
+  dsp_.distance_weighted_blend(prediction_0, prediction_stride_0, prediction_1,
+                               prediction_stride_1, weight[0], weight[1], width,
+                               height, dest, dest_stride);
 }
 
 bool Tile::GetReferenceBlockPosition(
@@ -1011,12 +1034,11 @@
   *ref_block_end_y =
       GetPixelPositionFromHighScale(start_y, step_y, height - 1) +
       kConvolveBorderRightBottom;
-  int block_height =
-      height + kConvolveBorderLeftTop + kConvolveBorderRightBottom;
   if (is_scaled) {
-    block_height = (((height - 1) * step_y + (1 << kScaleSubPixelBits) - 1) >>
-                    kScaleSubPixelBits) +
-                   kSubPixelTaps;
+    const int block_height =
+        (((height - 1) * step_y + (1 << kScaleSubPixelBits) - 1) >>
+         kScaleSubPixelBits) +
+        kSubPixelTaps;
     *ref_block_end_y = *ref_block_start_y + block_height - 1;
   }
   const bool extend_left = *ref_block_start_x < ref_start_x;
@@ -1116,12 +1138,12 @@
   int step_y;
   ScaleMotionVector(mv, plane, reference_frame_index, x, y, &start_x, &start_y,
                     &step_x, &step_y);
-  // reference_frame_index equal to -1 indicates using current frame as
-  // reference.
   const int horizontal_filter_index = bp.interpolation_filter[1];
   const int vertical_filter_index = bp.interpolation_filter[0];
-  const int subsampling_x = SubsamplingX(plane);
-  const int subsampling_y = SubsamplingY(plane);
+  const int subsampling_x = subsampling_x_[plane];
+  const int subsampling_y = subsampling_y_[plane];
+  // reference_frame_index equal to -1 indicates using current frame as
+  // reference.
   const YuvBuffer* const reference_buffer =
       (reference_frame_index == -1)
           ? current_frame_.buffer()
@@ -1223,6 +1245,7 @@
       (is_compound || is_inter_intra) ? prediction : static_cast<void*>(dest);
   const ptrdiff_t output_stride =
       (is_compound || is_inter_intra) ? prediction_stride : dest_stride;
+  assert(output != nullptr);
   dsp::ConvolveFunc convolve_func =
       is_scaled ? dsp_.convolve_scale[is_compound || is_inter_intra]
                 : dsp_.convolve[reference_frame_index == -1][is_compound]
@@ -1233,17 +1256,19 @@
     convolve_func = dsp_.convolve[0][1][1][1];
   }
   convolve_func(block_start, block_stride, horizontal_filter_index,
-                vertical_filter_index, round_bits, start_x, start_y, step_x,
+                vertical_filter_index, round_bits[1], start_x, start_y, step_x,
                 step_y, width, height, output, output_stride);
 }
 
 bool Tile::BlockWarpProcess(const Block& block, const Plane plane,
-                            const int index, const int width, const int height,
-                            uint16_t* prediction, ptrdiff_t prediction_stride,
-                            GlobalMotion* warp_params,
+                            const int index, const int block_start_x,
+                            const int block_start_y, const int width,
+                            const int height, const ptrdiff_t prediction_stride,
+                            GlobalMotion* const warp_params,
                             const uint8_t* const round_bits,
                             const bool is_compound, const bool is_inter_intra,
                             uint8_t* const dest, const ptrdiff_t dest_stride) {
+  assert(width >= 8 && height >= 8);
   const BlockParameters& bp = *block.bp;
   const int reference_frame_index =
       frame_header_.reference_frame_index[bp.reference_frame[index] -
@@ -1258,16 +1283,10 @@
   const int source_height =
       reference_frames_[reference_frame_index]->buffer()->displayed_height(
           plane);
-  const int block_start_x = MultiplyBy4(block.column4x4) >> SubsamplingX(plane);
-  const int block_start_y = MultiplyBy4(block.row4x4) >> SubsamplingY(plane);
-  const bool warp_valid = SetupShear(warp_params);
-  if (!warp_valid) {
-    LIBGAV1_DLOG(ERROR, "Invalid warp parameters.");
-    return false;
-  }
+  uint16_t* const prediction = block.sb_buffer->prediction_buffer[index].get();
   dsp_.warp(source, source_stride, source_width, source_height,
-            warp_params->params, SubsamplingX(plane), SubsamplingY(plane),
-            round_bits, block_start_x, block_start_y, width, height,
+            warp_params->params, subsampling_x_[plane], subsampling_y_[plane],
+            round_bits[1], block_start_x, block_start_y, width, height,
             warp_params->alpha, warp_params->beta, warp_params->gamma,
             warp_params->delta, prediction, prediction_stride);
   if (!is_compound && !is_inter_intra) {
diff --git a/libgav1/src/tile/tile.cc b/libgav1/src/tile/tile.cc
index 9a5c812..378410d 100644
--- a/libgav1/src/tile/tile.cc
+++ b/libgav1/src/tile/tile.cc
@@ -7,20 +7,23 @@
 #include <cstring>
 #include <memory>
 #include <new>
+#include <numeric>
 #include <type_traits>
 #include <utility>
 
 #include "src/motion_vector.h"
 #include "src/reconstruction.h"
+#include "src/utils/bit_mask_set.h"
 #include "src/utils/logging.h"
-#include "src/utils/scan.h"
 #include "src/utils/segmentation.h"
+#include "src/utils/stack.h"
 
 namespace libgav1 {
 namespace {
 
 // Import all the constants in the anonymous namespace.
 #include "src/quantizer_tables.inc"
+#include "src/scan_tables.inc"
 
 // Precision bits when scaling reference frames.
 constexpr int kReferenceScaleShift = 14;
@@ -32,36 +35,63 @@
     kQuantizerCoefficientBaseRange + kNumQuantizerBaseLevels + 1;
 constexpr int kCoeffBaseRangeMaxIterations =
     kQuantizerCoefficientBaseRange / (kCoeffBaseRangeSymbolCount - 1);
+constexpr int kEntropyContextLeft = 0;
+constexpr int kEntropyContextTop = 1;
 
-const uint8_t kAllZeroContextsByMinMax[5][5] = {{1, 2, 2, 2, 3},
-                                                {1, 4, 4, 4, 5},
-                                                {1, 4, 4, 4, 5},
-                                                {1, 4, 4, 4, 5},
-                                                {1, 4, 4, 4, 6}};
+constexpr uint8_t kAllZeroContextsByTopLeft[5][5] = {{1, 2, 2, 2, 3},
+                                                     {2, 4, 4, 4, 5},
+                                                     {2, 4, 4, 4, 5},
+                                                     {2, 4, 4, 4, 5},
+                                                     {3, 5, 5, 5, 6}};
 
 // The space complexity of DFS is O(branching_factor * max_depth). For the
 // parameter tree, branching_factor = 4 (there could be up to 4 children for
-// every node) and max_depth = 6 (to go from a 128x128 block all the way to a
-// 4x4 block).
-constexpr int kDfsStackSize = 24;
+// every node) and max_depth (excluding the root) = 5 (to go from a 128x128
+// block all the way to a 4x4 block). The worse-case stack size is 16, by
+// counting the number of 'o' nodes in the diagram:
+//
+//   |                    128x128  The highest level (corresponding to the
+//   |                             root of the tree) has no node in the stack.
+//   |-----------------+
+//   |     |     |     |
+//   |     o     o     o  64x64
+//   |
+//   |-----------------+
+//   |     |     |     |
+//   |     o     o     o  32x32    Higher levels have three nodes in the stack,
+//   |                             because we pop one node off the stack before
+//   |-----------------+           pushing its four children onto the stack.
+//   |     |     |     |
+//   |     o     o     o  16x16
+//   |
+//   |-----------------+
+//   |     |     |     |
+//   |     o     o     o  8x8
+//   |
+//   |-----------------+
+//   |     |     |     |
+//   o     o     o     o  4x4      Only the lowest level has four nodes in the
+//                                 stack.
+constexpr int kDfsStackSize = 16;
 
 // Mask indicating whether the transform sets contain a particular transform
 // type. If |tx_type| is present in |tx_set|, then the |tx_type|th LSB is set.
-constexpr uint16_t kTransformTypeInSetMask[kNumTransformSets] = {
-    0x1, 0xE0F, 0x20F, 0xFFFF, 0xFFF, 0x201};
+constexpr BitMaskSet kTransformTypeInSetMask[kNumTransformSets] = {
+    BitMaskSet(0x1),    BitMaskSet(0xE0F), BitMaskSet(0x20F),
+    BitMaskSet(0xFFFF), BitMaskSet(0xFFF), BitMaskSet(0x201)};
 
-const PredictionMode
+constexpr PredictionMode
     kFilterIntraModeToIntraPredictor[kNumFilterIntraPredictors] = {
         kPredictionModeDc, kPredictionModeVertical, kPredictionModeHorizontal,
         kPredictionModeD157, kPredictionModeDc};
 
 // This is computed as:
 // min(transform_width_log2, 5) + min(transform_height_log2, 5) - 4.
-const uint8_t kEobMultiSizeLookup[kNumTransformSizes] = {
+constexpr uint8_t kEobMultiSizeLookup[kNumTransformSizes] = {
     0, 1, 2, 1, 2, 3, 4, 2, 3, 4, 5, 5, 4, 5, 6, 6, 5, 6, 6};
 
 /* clang-format off */
-const uint8_t kCoeffBaseContextOffset[kNumTransformSizes][5][5] = {
+constexpr uint8_t kCoeffBaseContextOffset[kNumTransformSizes][5][5] = {
     {{0, 1, 6, 6, 0}, {1, 6, 6, 21, 0}, {6, 6, 21, 21, 0}, {6, 21, 21, 21, 0},
      {0, 0, 0, 0, 0}},
     {{0, 11, 11, 11, 0}, {11, 11, 11, 11, 0}, {6, 6, 21, 21, 0},
@@ -102,9 +132,9 @@
      {6, 21, 21, 21, 21}, {21, 21, 21, 21, 21}}};
 /* clang-format on */
 
-const uint8_t kCoeffBasePositionContextOffset[3] = {26, 31, 36};
+constexpr uint8_t kCoeffBasePositionContextOffset[3] = {26, 31, 36};
 
-const PredictionMode kInterIntraToIntraMode[kNumInterIntraModes] = {
+constexpr PredictionMode kInterIntraToIntraMode[kNumInterIntraModes] = {
     kPredictionModeDc, kPredictionModeVertical, kPredictionModeHorizontal,
     kPredictionModeSmooth};
 
@@ -127,59 +157,100 @@
     {kNumTransformSizes, kNumTransformSizes, kTransformSize64x16,
      kTransformSize64x32, kTransformSize64x64}};
 
+// Defined in section 9.3 of the spec.
+constexpr TransformType kModeToTransformType[kIntraPredictionModesUV] = {
+    kTransformTypeDctDct,   kTransformTypeDctAdst,  kTransformTypeAdstDct,
+    kTransformTypeDctDct,   kTransformTypeAdstAdst, kTransformTypeDctAdst,
+    kTransformTypeAdstDct,  kTransformTypeAdstDct,  kTransformTypeDctAdst,
+    kTransformTypeAdstAdst, kTransformTypeDctAdst,  kTransformTypeAdstDct,
+    kTransformTypeAdstAdst, kTransformTypeDctDct};
+
+// Defined in section 5.11.47 of the spec. This array does not contain an entry
+// for kTransformSetDctOnly, so the first dimension needs to be
+// |kNumTransformSets| - 1.
+constexpr TransformType kInverseTransformTypeBySet[kNumTransformSets - 1][16] =
+    {{kTransformTypeIdentityIdentity, kTransformTypeDctDct,
+      kTransformTypeIdentityDct, kTransformTypeDctIdentity,
+      kTransformTypeAdstAdst, kTransformTypeDctAdst, kTransformTypeAdstDct},
+     {kTransformTypeIdentityIdentity, kTransformTypeDctDct,
+      kTransformTypeAdstAdst, kTransformTypeDctAdst, kTransformTypeAdstDct},
+     {kTransformTypeIdentityIdentity, kTransformTypeIdentityDct,
+      kTransformTypeDctIdentity, kTransformTypeIdentityAdst,
+      kTransformTypeAdstIdentity, kTransformTypeIdentityFlipadst,
+      kTransformTypeFlipadstIdentity, kTransformTypeDctDct,
+      kTransformTypeDctAdst, kTransformTypeAdstDct, kTransformTypeDctFlipadst,
+      kTransformTypeFlipadstDct, kTransformTypeAdstAdst,
+      kTransformTypeFlipadstFlipadst, kTransformTypeFlipadstAdst,
+      kTransformTypeAdstFlipadst},
+     {kTransformTypeIdentityIdentity, kTransformTypeIdentityDct,
+      kTransformTypeDctIdentity, kTransformTypeDctDct, kTransformTypeDctAdst,
+      kTransformTypeAdstDct, kTransformTypeDctFlipadst,
+      kTransformTypeFlipadstDct, kTransformTypeAdstAdst,
+      kTransformTypeFlipadstFlipadst, kTransformTypeFlipadstAdst,
+      kTransformTypeAdstFlipadst},
+     {kTransformTypeIdentityIdentity, kTransformTypeDctDct}};
+
+// Replaces all occurrences of 64x* and *x64 with 32x* and *x32 respectively.
+constexpr TransformSize kAdjustedTransformSize[kNumTransformSizes] = {
+    kTransformSize4x4,   kTransformSize4x8,   kTransformSize4x16,
+    kTransformSize8x4,   kTransformSize8x8,   kTransformSize8x16,
+    kTransformSize8x32,  kTransformSize16x4,  kTransformSize16x8,
+    kTransformSize16x16, kTransformSize16x32, kTransformSize16x32,
+    kTransformSize32x8,  kTransformSize32x16, kTransformSize32x32,
+    kTransformSize32x32, kTransformSize32x16, kTransformSize32x32,
+    kTransformSize32x32};
+
+// ith entry of this array is computed as:
+// DivideBy2(TransformSizeToSquareTransformIndex(kTransformSizeSquareMin[i]) +
+//           TransformSizeToSquareTransformIndex(kTransformSizeSquareMax[i]) +
+//           1)
+constexpr uint8_t kTransformSizeContext[kNumTransformSizes] = {
+    0, 1, 1, 1, 1, 2, 2, 1, 2, 2, 3, 3, 2, 3, 3, 4, 3, 4, 4};
+
+constexpr int8_t kSgrProjDefaultMultiplier[2] = {-32, 31};
+
+constexpr int8_t kWienerDefaultFilter[3] = {3, -7, 15};
+
 // Maps compound prediction modes into single modes. For e.g.
 // kPredictionModeNearestNewMv will map to kPredictionModeNearestMv for index 0
 // and kPredictionModeNewMv for index 1. It is used to simplify the logic in
 // AssignMv (and avoid duplicate code). This is section 5.11.30. in the spec.
-inline PredictionMode GetSinglePredictionMode(int index,
-                                              PredictionMode y_mode) {
-  if (index == 0) {
-    if (y_mode < kPredictionModeNearestNearestMv) {
-      return y_mode;
-    }
-    if (y_mode == kPredictionModeNewNewMv ||
-        y_mode == kPredictionModeNewNearestMv ||
-        y_mode == kPredictionModeNewNearMv) {
-      return kPredictionModeNewMv;
-    }
-    if (y_mode == kPredictionModeNearestNearestMv ||
-        y_mode == kPredictionModeNearestNewMv) {
-      return kPredictionModeNearestMv;
-    }
-    if (y_mode == kPredictionModeNearNearMv ||
-        y_mode == kPredictionModeNearNewMv) {
-      return kPredictionModeNearMv;
-    }
-    return kPredictionModeGlobalMv;
+constexpr PredictionMode
+    kCompoundToSinglePredictionMode[kNumCompoundInterPredictionModes][2] = {
+        {kPredictionModeNearestMv, kPredictionModeNearestMv},
+        {kPredictionModeNearMv, kPredictionModeNearMv},
+        {kPredictionModeNearestMv, kPredictionModeNewMv},
+        {kPredictionModeNewMv, kPredictionModeNearestMv},
+        {kPredictionModeNearMv, kPredictionModeNewMv},
+        {kPredictionModeNewMv, kPredictionModeNearMv},
+        {kPredictionModeGlobalMv, kPredictionModeGlobalMv},
+        {kPredictionModeNewMv, kPredictionModeNewMv},
+};
+PredictionMode GetSinglePredictionMode(int index, PredictionMode y_mode) {
+  if (y_mode < kPredictionModeNearestNearestMv) {
+    return y_mode;
   }
-  if (y_mode == kPredictionModeNewNewMv ||
-      y_mode == kPredictionModeNearestNewMv ||
-      y_mode == kPredictionModeNearNewMv) {
-    return kPredictionModeNewMv;
-  }
-  if (y_mode == kPredictionModeNearestNearestMv ||
-      y_mode == kPredictionModeNewNearestMv) {
-    return kPredictionModeNearestMv;
-  }
-  if (y_mode == kPredictionModeNearNearMv ||
-      y_mode == kPredictionModeNewNearMv) {
-    return kPredictionModeNearMv;
-  }
-  return kPredictionModeGlobalMv;
+  const int lookup_index = y_mode - kPredictionModeNearestNearestMv;
+  assert(lookup_index >= 0);
+  return kCompoundToSinglePredictionMode[lookup_index][index];
 }
 
 // log2(dqDenom) in section 7.12.3 of the spec. We use the log2 value because
 // dqDenom is always a power of two and hence right shift can be used instead of
 // division.
+constexpr BitMaskSet kQuantizationShift2Mask(kTransformSize32x64,
+                                             kTransformSize64x32,
+                                             kTransformSize64x64);
+constexpr BitMaskSet kQuantizationShift1Mask(kTransformSize16x32,
+                                             kTransformSize16x64,
+                                             kTransformSize32x16,
+                                             kTransformSize32x32,
+                                             kTransformSize64x16);
 int GetQuantizationShift(TransformSize tx_size) {
-  const int tx_width = kTransformWidth[tx_size];
-  const int tx_height = kTransformHeight[tx_size];
-  const int max_tx_dimension = std::max(tx_width, tx_height);
-  const int min_tx_dimension = std::min(tx_width, tx_height);
-  if (max_tx_dimension == 64 && min_tx_dimension >= 32) {
+  if (kQuantizationShift2Mask.Contains(tx_size)) {
     return 2;
   }
-  if (max_tx_dimension >= 32 && min_tx_dimension >= 16) {
+  if (kQuantizationShift1Mask.Contains(tx_size)) {
     return 1;
   }
   return 0;
@@ -193,6 +264,49 @@
   return index + MultiplyBy4(index >> tx_width_log2);
 }
 
+// Returns the minimum of |length| or |max|-|start|. This is used to clamp array
+// indices when accessing arrays whose bound is equal to |max|.
+int GetNumElements(int length, int start, int max) {
+  return std::min(length, max - start);
+}
+
+// This is the same as Max_Tx_Size_Rect array in the spec but with *x64 and 64*x
+// transforms replaced with *x32 and 32x* respectively.
+constexpr TransformSize kUVTransformSize[kMaxBlockSizes] = {
+    kTransformSize4x4,   kTransformSize4x8,   kTransformSize4x16,
+    kTransformSize8x4,   kTransformSize8x8,   kTransformSize8x16,
+    kTransformSize8x32,  kTransformSize16x4,  kTransformSize16x8,
+    kTransformSize16x16, kTransformSize16x32, kTransformSize16x32,
+    kTransformSize32x8,  kTransformSize32x16, kTransformSize32x32,
+    kTransformSize32x32, kTransformSize32x16, kTransformSize32x32,
+    kTransformSize32x32, kTransformSize32x32, kTransformSize32x32,
+    kTransformSize32x32};
+
+// 5.11.37.
+TransformSize GetTransformSize(bool lossless, BlockSize block_size, Plane plane,
+                               TransformSize tx_size, int subsampling_x,
+                               int subsampling_y) {
+  if (plane == kPlaneY) return tx_size;
+  // For Y Plane, |tx_size| is always kTransformSize4x4. So it is sufficient to
+  // have this special case for |lossless| only for U and V planes.
+  if (lossless) return kTransformSize4x4;
+  const BlockSize plane_size =
+      kPlaneResidualSize[block_size][subsampling_x][subsampling_y];
+  assert(plane_size != kBlockInvalid);
+  return kUVTransformSize[plane_size];
+}
+
+void SetTransformType(const Tile::Block& block, int x4, int y4, int w4, int h4,
+                      TransformType tx_type,
+                      TransformType transform_types[32][32]) {
+  const int y_offset = y4 - block.row4x4;
+  const int x_offset = x4 - block.column4x4;
+  static_assert(sizeof(transform_types[0][0]) == 1, "");
+  for (int i = 0; i < h4; ++i) {
+    memset(&transform_types[y_offset + i][x_offset], tx_type, w4);
+  }
+}
+
 }  // namespace
 
 Tile::Tile(
@@ -212,11 +326,16 @@
     Array2D<int16_t>* const cdef_index,
     Array2D<TransformSize>* const inter_transform_sizes,
     const dsp::Dsp* const dsp, ThreadPool* const thread_pool,
-    ResidualBufferPool* const residual_buffer_pool)
+    ResidualBufferPool* const residual_buffer_pool,
+    BlockingCounterWithStatus* const pending_tiles)
     : number_(tile_number),
       data_(data),
       size_(size),
       read_deltas_(false),
+      subsampling_x_{0, sequence_header.color_config.subsampling_x,
+                     sequence_header.color_config.subsampling_x},
+      subsampling_y_{0, sequence_header.color_config.subsampling_y,
+                     sequence_header.color_config.subsampling_y},
       current_quantizer_index_(frame_header.quantizer.base_index),
       sequence_header_(sequence_header),
       frame_header_(frame_header),
@@ -245,21 +364,15 @@
       cdef_index_(*cdef_index),
       inter_transform_sizes_(*inter_transform_sizes),
       thread_pool_(thread_pool),
-      residual_buffer_pool_(residual_buffer_pool) {
+      residual_buffer_pool_(residual_buffer_pool),
+      pending_tiles_(pending_tiles),
+      build_bit_mask_when_parsing_(false) {
   row_ = number_ / frame_header.tile_info.tile_columns;
   column_ = number_ % frame_header.tile_info.tile_columns;
   row4x4_start_ = frame_header.tile_info.tile_row_start[row_];
   row4x4_end_ = frame_header.tile_info.tile_row_start[row_ + 1];
   column4x4_start_ = frame_header.tile_info.tile_column_start[column_];
   column4x4_end_ = frame_header.tile_info.tile_column_start[column_ + 1];
-  for (size_t i = 0; i < entropy_contexts_.size(); ++i) {
-    const int contexts_per_plane = (i == EntropyContext::kLeft)
-                                       ? frame_header_.rows4x4
-                                       : frame_header_.columns4x4;
-    if (!entropy_contexts_[i].Reset(PlaneCount(), contexts_per_plane)) {
-      LIBGAV1_DLOG(ERROR, "entropy_contexts_[%zu].Reset() failed.", i);
-    }
-  }
   const int block_width4x4 = kNum4x4BlocksWide[SuperBlockSize()];
   const int block_width4x4_log2 = k4x4HeightLog2[SuperBlockSize()];
   superblock_rows_ =
@@ -271,17 +384,8 @@
   // superblock columns as |intra_block_copy_lag_|.
   split_parse_and_decode_ =
       thread_pool_ != nullptr && superblock_columns_ > intra_block_copy_lag_;
-  if (split_parse_and_decode_) {
-    assert(residual_buffer_pool != nullptr);
-    if (!residual_buffer_threaded_.Reset(superblock_rows_, superblock_columns_,
-                                         /*zero_initialize=*/false)) {
-      LIBGAV1_DLOG(ERROR, "residual_buffer_threaded_.Reset() failed.");
-    }
-  } else {
-    residual_buffer_ = MakeAlignedUniquePtr<uint8_t>(32, 4096 * residual_size_);
-    prediction_parameters_.reset(new (std::nothrow) PredictionParameters());
-  }
   memset(delta_lf_, 0, sizeof(delta_lf_));
+  delta_lf_all_zero_ = true;
   YuvBuffer* const buffer = current_frame->buffer();
   for (int plane = 0; plane < PlaneCount(); ++plane) {
     buffer_[plane].Reset(buffer->height(plane) + buffer->bottom_border(plane),
@@ -289,13 +393,58 @@
   }
 }
 
-bool Tile::Decode() {
+bool Tile::Init() {
+  assert(coefficient_levels_.size() == dc_categories_.size());
+  for (size_t i = 0; i < coefficient_levels_.size(); ++i) {
+    const int contexts_per_plane = (i == kEntropyContextLeft)
+                                       ? frame_header_.rows4x4
+                                       : frame_header_.columns4x4;
+    if (!coefficient_levels_[i].Reset(PlaneCount(), contexts_per_plane)) {
+      LIBGAV1_DLOG(ERROR, "coefficient_levels_[%zu].Reset() failed.", i);
+      return false;
+    }
+    if (!dc_categories_[i].Reset(PlaneCount(), contexts_per_plane)) {
+      LIBGAV1_DLOG(ERROR, "dc_categories_[%zu].Reset() failed.", i);
+      return false;
+    }
+  }
+  if (split_parse_and_decode_) {
+    assert(residual_buffer_pool_ != nullptr);
+    if (!residual_buffer_threaded_.Reset(superblock_rows_, superblock_columns_,
+                                         /*zero_initialize=*/false)) {
+      LIBGAV1_DLOG(ERROR, "residual_buffer_threaded_.Reset() failed.");
+      return false;
+    }
+  } else {
+    residual_buffer_ = MakeAlignedUniquePtr<uint8_t>(32, 4096 * residual_size_);
+    if (residual_buffer_ == nullptr) {
+      LIBGAV1_DLOG(ERROR, "Allocation of residual_buffer_ failed.");
+      return false;
+    }
+    prediction_parameters_.reset(new (std::nothrow) PredictionParameters());
+    if (prediction_parameters_ == nullptr) {
+      LIBGAV1_DLOG(ERROR, "Allocation of prediction_parameters_ failed.");
+      return false;
+    }
+  }
+  return true;
+}
+
+bool Tile::Decode(bool is_main_thread) {
+  if (!Init()) {
+    pending_tiles_->Decrement(false);
+    return false;
+  }
   if (frame_header_.use_ref_frame_mvs) {
     SetupMotionField(sequence_header_, frame_header_, current_frame_,
                      reference_frames_, motion_field_mv_, row4x4_start_,
                      row4x4_end_, column4x4_start_, column4x4_end_);
   }
   ResetLoopRestorationParams();
+  // If this is the main thread, we build the loop filter bit masks when parsing
+  // so that it happens in the current thread. This ensures that the main thread
+  // does as much work as possible.
+  build_bit_mask_when_parsing_ = is_main_thread;
   if (split_parse_and_decode_) {
     if (!ThreadedDecode()) return false;
   } else {
@@ -307,6 +456,7 @@
            column4x4 += block_width4x4) {
         if (!ProcessSuperBlock(row4x4, column4x4, block_width4x4, &sb_buffer,
                                kProcessingModeParseAndDecode)) {
+          pending_tiles_->Decrement(false);
           LIBGAV1_DLOG(ERROR, "Error decoding super block row: %d column: %d",
                        row4x4, column4x4);
           return false;
@@ -318,17 +468,22 @@
       number_ == frame_header_.tile_info.context_update_id) {
     *saved_symbol_decoder_context_ = symbol_decoder_context_;
   }
+  if (!split_parse_and_decode_) {
+    pending_tiles_->Decrement(true);
+  }
   return true;
 }
 
 bool Tile::ThreadedDecode() {
-  ThreadingParameters threading;
   {
-    std::lock_guard<std::mutex> lock(threading.mutex);
-    if (!threading.sb_state.Reset(superblock_rows_, superblock_columns_)) {
+    std::lock_guard<std::mutex> lock(threading_.mutex);
+    if (!threading_.sb_state.Reset(superblock_rows_, superblock_columns_)) {
+      pending_tiles_->Decrement(false);
       LIBGAV1_DLOG(ERROR, "threading.sb_state.Reset() failed.");
       return false;
     }
+    // Account for the parsing job.
+    ++threading_.pending_jobs;
   }
 
   const int block_width4x4 = kNum4x4BlocksWide[SuperBlockSize()];
@@ -342,47 +497,58 @@
          column4x4 += block_width4x4, ++column_index) {
       if (!ProcessSuperBlock(row4x4, column4x4, block_width4x4, &sb_buffer,
                              kProcessingModeParseOnly)) {
-        std::lock_guard<std::mutex> lock(threading.mutex);
-        threading.abort = true;
+        std::lock_guard<std::mutex> lock(threading_.mutex);
+        threading_.abort = true;
         break;
       }
-      std::lock_guard<std::mutex> lock(threading.mutex);
-      if (threading.abort) break;
-      threading.sb_state[row_index][column_index] = kSuperBlockStateParsed;
+      std::unique_lock<std::mutex> lock(threading_.mutex);
+      if (threading_.abort) break;
+      threading_.sb_state[row_index][column_index] = kSuperBlockStateParsed;
       // Schedule the decoding of this superblock if it is allowed.
-      if (CanDecode(row_index, column_index, threading.sb_state)) {
-        ++threading.pending_jobs;
-        threading.sb_state[row_index][column_index] = kSuperBlockStateScheduled;
-        thread_pool_->Schedule([this, row_index, column_index, block_width4x4,
-                                &threading]() {
-          DecodeSuperBlock(row_index, column_index, block_width4x4, &threading);
-        });
+      if (CanDecode(row_index, column_index)) {
+        ++threading_.pending_jobs;
+        threading_.sb_state[row_index][column_index] =
+            kSuperBlockStateScheduled;
+        lock.unlock();
+        thread_pool_->Schedule(
+            [this, row_index, column_index, block_width4x4]() {
+              DecodeSuperBlock(row_index, column_index, block_width4x4);
+            });
       }
     }
-    std::lock_guard<std::mutex> lock(threading.mutex);
-    if (threading.abort) break;
+    std::lock_guard<std::mutex> lock(threading_.mutex);
+    if (threading_.abort) break;
   }
 
-  // Wait for the decode jobs to finish.
-  std::unique_lock<std::mutex> lock(threading.mutex);
-  while (threading.pending_jobs != 0) {
-    threading.pending_jobs_zero_condvar.wait(lock);
+  // We are done parsing. We can return here since the calling thread will make
+  // sure that it waits for all the superblocks to be decoded.
+  //
+  // Finish using |threading_| before |pending_tiles_->Decrement()| because the
+  // Tile object could go out of scope as soon as |pending_tiles_->Decrement()|
+  // is called.
+  threading_.mutex.lock();
+  const bool no_pending_jobs = (--threading_.pending_jobs == 0);
+  const bool job_succeeded = !threading_.abort;
+  threading_.mutex.unlock();
+  if (no_pending_jobs) {
+    // We are done parsing and decoding this tile.
+    pending_tiles_->Decrement(job_succeeded);
   }
-
-  return !threading.abort;
+  return job_succeeded;
 }
 
-bool Tile::CanDecode(int row_index, int column_index,
-                     const Array2D<SuperBlockState>& sb_state) {
-  // If |sb_state| is not equal to kSuperBlockStateParsed, then return false.
-  // This is ok because if |sb_state| is equal to:
+bool Tile::CanDecode(int row_index, int column_index) const {
+  assert(row_index >= 0);
+  assert(column_index >= 0);
+  // If |threading_.sb_state[row_index][column_index]| is not equal to
+  // kSuperBlockStateParsed, then return false. This is ok because if
+  // |threading_.sb_state[row_index][column_index]| is equal to:
   //   kSuperBlockStateNone - then the superblock is not yet parsed.
   //   kSuperBlockStateScheduled - then the superblock is already scheduled for
   //                               decode.
   //   kSuperBlockStateDecoded - then the superblock has already been decoded.
-  if (row_index < 0 || column_index < 0 || row_index >= superblock_rows_ ||
-      column_index >= superblock_columns_ ||
-      sb_state[row_index][column_index] != kSuperBlockStateParsed) {
+  if (row_index >= superblock_rows_ || column_index >= superblock_columns_ ||
+      threading_.sb_state[row_index][column_index] != kSuperBlockStateParsed) {
     return false;
   }
   // First superblock has no dependencies.
@@ -392,29 +558,30 @@
   // Superblocks in the first row only depend on the superblock to the left of
   // it.
   if (row_index == 0) {
-    return sb_state[0][column_index - 1] == kSuperBlockStateDecoded;
+    return threading_.sb_state[0][column_index - 1] == kSuperBlockStateDecoded;
   }
   // All other superblocks depend on superblock to the left of it (if one
   // exists) and superblock to the top right with a lag of
   // |intra_block_copy_lag_| (if one exists).
   const int top_right_column_index =
       std::min(column_index + intra_block_copy_lag_, superblock_columns_ - 1);
-  return sb_state[row_index - 1][top_right_column_index] ==
+  return threading_.sb_state[row_index - 1][top_right_column_index] ==
              kSuperBlockStateDecoded &&
          (column_index == 0 ||
-          sb_state[row_index][column_index - 1] == kSuperBlockStateDecoded);
+          threading_.sb_state[row_index][column_index - 1] ==
+              kSuperBlockStateDecoded);
 }
 
-void Tile::DecodeSuperBlock(int row_index, int column_index, int block_width4x4,
-                            ThreadingParameters* const threading) {
+void Tile::DecodeSuperBlock(int row_index, int column_index,
+                            int block_width4x4) {
   SuperBlockBuffer sb_buffer;
   const int row4x4 = row4x4_start_ + (row_index * block_width4x4);
   const int column4x4 = column4x4_start_ + (column_index * block_width4x4);
   const bool ok = ProcessSuperBlock(row4x4, column4x4, block_width4x4,
                                     &sb_buffer, kProcessingModeDecodeOnly);
-  std::lock_guard<std::mutex> lock(threading->mutex);
+  std::unique_lock<std::mutex> lock(threading_.mutex);
   if (ok) {
-    threading->sb_state[row_index][column_index] = kSuperBlockStateDecoded;
+    threading_.sb_state[row_index][column_index] = kSuperBlockStateDecoded;
     // Candidate rows and columns that we could potentially begin the decoding
     // (if it is allowed to do so). The candidates are:
     //   1) The superblock to the bottom-left of the current superblock with a
@@ -429,26 +596,29 @@
          ++i) {
       const int candidate_row_index = candidate_row_indices[i];
       const int candidate_column_index = candidate_column_indices[i];
-      if (!CanDecode(candidate_row_index, candidate_column_index,
-                     threading->sb_state)) {
+      if (!CanDecode(candidate_row_index, candidate_column_index)) {
         continue;
       }
-      ++threading->pending_jobs;
-      threading->sb_state[candidate_row_index][candidate_column_index] =
+      ++threading_.pending_jobs;
+      threading_.sb_state[candidate_row_index][candidate_column_index] =
           kSuperBlockStateScheduled;
+      lock.unlock();
       thread_pool_->Schedule([this, candidate_row_index, candidate_column_index,
-                              block_width4x4, threading]() {
+                              block_width4x4]() {
         DecodeSuperBlock(candidate_row_index, candidate_column_index,
-                         block_width4x4, threading);
+                         block_width4x4);
       });
+      lock.lock();
     }
   } else {
-    threading->abort = true;
+    threading_.abort = true;
   }
-  if (--threading->pending_jobs == 0) {
-    // TODO(jzern): the mutex doesn't need to be locked to signal the
-    // condition.
-    threading->pending_jobs_zero_condvar.notify_one();
+  if (--threading_.pending_jobs == 0) {
+    // The lock needs to be unlocked here in this case because the Tile object
+    // could go out of scope as soon as |pending_tiles_->Decrement()| is called.
+    lock.unlock();
+    // We are done parsing and decoding this tile.
+    pending_tiles_->Decrement(!threading_.abort);
   }
 }
 
@@ -460,54 +630,55 @@
 int Tile::GetTransformAllZeroContext(const Block& block, Plane plane,
                                      TransformSize tx_size, int x4, int y4,
                                      int w4, int h4) {
-  const int max_x4x4 = frame_header_.columns4x4 >> SubsamplingX(plane);
-  const int max_y4x4 = frame_header_.rows4x4 >> SubsamplingY(plane);
+  const int max_x4x4 = frame_header_.columns4x4 >> subsampling_x_[plane];
+  const int max_y4x4 = frame_header_.rows4x4 >> subsampling_y_[plane];
 
   const int tx_width = kTransformWidth[tx_size];
   const int tx_height = kTransformHeight[tx_size];
   const BlockSize plane_block_size =
-      kPlaneResidualSize[block.size][SubsamplingX(plane)][SubsamplingY(plane)];
+      kPlaneResidualSize[block.size][subsampling_x_[plane]]
+                        [subsampling_y_[plane]];
   assert(plane_block_size != kBlockInvalid);
   const int block_width = kBlockWidthPixels[plane_block_size];
   const int block_height = kBlockHeightPixels[plane_block_size];
 
   int top = 0;
   int left = 0;
+  const int num_top_elements = GetNumElements(w4, x4, max_x4x4);
+  const int num_left_elements = GetNumElements(h4, y4, max_y4x4);
   if (plane == kPlaneY) {
     if (block_width == tx_width && block_height == tx_height) return 0;
-    for (int i = 0; i < w4 && (i + x4 < max_x4x4); ++i) {
-      top = std::max(top,
-                     static_cast<int>(
-                         entropy_contexts_[EntropyContext::kTop][plane][x4 + i]
-                             .coefficient_level));
+    const uint8_t* coefficient_levels =
+        &coefficient_levels_[kEntropyContextTop][plane][x4];
+    for (int i = 0; i < num_top_elements; ++i) {
+      top = std::max(top, static_cast<int>(coefficient_levels[i]));
     }
-    for (int i = 0; i < h4 && (i + y4 < max_y4x4); ++i) {
-      left = std::max(
-          left, static_cast<int>(
-                    entropy_contexts_[EntropyContext::kLeft][plane][y4 + i]
-                        .coefficient_level));
+    coefficient_levels = &coefficient_levels_[kEntropyContextLeft][plane][y4];
+    for (int i = 0; i < num_left_elements; ++i) {
+      left = std::max(left, static_cast<int>(coefficient_levels[i]));
     }
-    top = std::min(top, 255);
-    left = std::min(left, 255);
-    const int min = std::min({top, left, 4});
-    const int max = std::min(std::max(top, left), 4);
-    // kAllZeroContextsByMinMax is pre-computed based on the logic in the spec
+    assert(top <= 4);
+    assert(left <= 4);
+    // kAllZeroContextsByTopLeft is pre-computed based on the logic in the spec
     // for top and left.
-    return kAllZeroContextsByMinMax[min][max];
+    return kAllZeroContextsByTopLeft[top][left];
   }
-  for (int i = 0; i < w4 && (i + x4 < max_x4x4); ++i) {
-    top |= entropy_contexts_[EntropyContext::kTop][plane][x4 + i]
-               .coefficient_level;
-    top |= entropy_contexts_[EntropyContext::kTop][plane][x4 + i].dc_category;
+  const uint8_t* coefficient_levels =
+      &coefficient_levels_[kEntropyContextTop][plane][x4];
+  const int8_t* dc_categories = &dc_categories_[kEntropyContextTop][plane][x4];
+  for (int i = 0; i < num_top_elements; ++i) {
+    top |= coefficient_levels[i];
+    top |= dc_categories[i];
   }
-  for (int i = 0; i < h4 && (i + y4 < max_y4x4); ++i) {
-    left |= entropy_contexts_[EntropyContext::kLeft][plane][y4 + i]
-                .coefficient_level;
-    left |= entropy_contexts_[EntropyContext::kLeft][plane][y4 + i].dc_category;
+  coefficient_levels = &coefficient_levels_[kEntropyContextLeft][plane][y4];
+  dc_categories = &dc_categories_[kEntropyContextLeft][plane][y4];
+  for (int i = 0; i < num_left_elements; ++i) {
+    left |= coefficient_levels[i];
+    left |= dc_categories[i];
   }
-  int context = static_cast<int>(top != 0) + static_cast<int>(left != 0) + 7;
-  if (block_width * block_height > tx_width * tx_height) context += 3;
-  return context;
+  return static_cast<int>(top != 0) + static_cast<int>(left != 0) + 7 +
+         3 * static_cast<int>(block_width * block_height >
+                              tx_width * tx_height);
 }
 
 TransformSet Tile::GetTransformSet(TransformSize tx_size, bool is_inter) const {
@@ -546,15 +717,13 @@
   TransformType tx_type;
   if (bp.is_inter) {
     const int x4 =
-        std::max(block.column4x4,
-                 block_x << sequence_header_.color_config.subsampling_x);
-    const int y4 = std::max(
-        block.row4x4, block_y << sequence_header_.color_config.subsampling_y);
+        std::max(block.column4x4, block_x << subsampling_x_[kPlaneU]);
+    const int y4 = std::max(block.row4x4, block_y << subsampling_y_[kPlaneU]);
     tx_type = transform_types_[y4 - block.row4x4][x4 - block.column4x4];
   } else {
     tx_type = kModeToTransformType[bp.uv_mode];
   }
-  return static_cast<bool>((kTransformTypeInSetMask[tx_set] >> tx_type) & 1)
+  return kTransformTypeInSetMask[tx_set].Contains(tx_type)
              ? tx_type
              : kTransformTypeDctDct;
 }
@@ -590,13 +759,8 @@
     // first dimension needs to be offset by 1.
     tx_type = kInverseTransformTypeBySet[tx_set - 1][tx_type];
   }
-  transform_types_[y4 - block.row4x4][x4 - block.column4x4] = tx_type;
-  for (int i = 0; i < DivideBy4(kTransformWidth[tx_size]); ++i) {
-    for (int j = 0; j < DivideBy4(kTransformHeight[tx_size]); ++j) {
-      transform_types_[y4 + j - block.row4x4][x4 + i - block.column4x4] =
-          tx_type;
-    }
-  }
+  SetTransformType(block, x4, y4, kTransformWidth4x4[tx_size],
+                   kTransformHeight4x4[tx_size], tx_type, transform_types_);
 }
 
 // Section 8.3.2 in the spec, under coeff_base_eob.
@@ -734,48 +898,36 @@
 }
 
 int Tile::GetDcSignContext(int x4, int y4, int w4, int h4, Plane plane) {
-  const int max_x4x4 = frame_header_.columns4x4 >> SubsamplingX(plane);
-  const int max_y4x4 = frame_header_.rows4x4 >> SubsamplingY(plane);
-  int dc_sign = 0;
-  for (int i = 0; i < w4 && (i + x4 < max_x4x4); ++i) {
-    const int sign =
-        entropy_contexts_[EntropyContext::kTop][plane][x4 + i].dc_category;
-    if (sign == 1) {
-      dc_sign--;
-    } else if (sign == 2) {
-      dc_sign++;
-    }
-  }
-  for (int i = 0; i < h4 && (i + y4 < max_y4x4); ++i) {
-    const int sign =
-        entropy_contexts_[EntropyContext::kLeft][plane][y4 + i].dc_category;
-    if (sign == 1) {
-      dc_sign--;
-    } else if (sign == 2) {
-      dc_sign++;
-    }
-  }
-  if (dc_sign < 0) return 1;
-  if (dc_sign > 0) return 2;
-  return 0;
+  const int max_x4x4 = frame_header_.columns4x4 >> subsampling_x_[plane];
+  const int8_t* dc_categories = &dc_categories_[kEntropyContextTop][plane][x4];
+  int dc_sign = std::accumulate(
+      dc_categories, dc_categories + GetNumElements(w4, x4, max_x4x4), 0);
+  const int max_y4x4 = frame_header_.rows4x4 >> subsampling_y_[plane];
+  dc_categories = &dc_categories_[kEntropyContextLeft][plane][y4];
+  dc_sign = std::accumulate(
+      dc_categories, dc_categories + GetNumElements(h4, y4, max_y4x4), dc_sign);
+  // This return statement is equivalent to:
+  //   if (dc_sign < 0) return 1;
+  //   if (dc_sign > 0) return 2;
+  //   return 0;
+  return static_cast<int>(dc_sign < 0) +
+         MultiplyBy2(static_cast<int>(dc_sign > 0));
 }
 
 void Tile::SetEntropyContexts(int x4, int y4, int w4, int h4, Plane plane,
-                              uint8_t coefficient_level, uint8_t dc_category) {
-  const int max_x4x4 = frame_header_.columns4x4 >> SubsamplingX(plane);
-  const int max_y4x4 = frame_header_.rows4x4 >> SubsamplingY(plane);
-  for (int i = 0; i < w4 && (i + x4 < max_x4x4); ++i) {
-    entropy_contexts_[EntropyContext::kTop][plane][x4 + i].coefficient_level =
-        coefficient_level;
-    entropy_contexts_[EntropyContext::kTop][plane][x4 + i].dc_category =
-        dc_category;
-  }
-  for (int i = 0; i < h4 && (i + y4 < max_y4x4); ++i) {
-    entropy_contexts_[EntropyContext::kLeft][plane][y4 + i].coefficient_level =
-        coefficient_level;
-    entropy_contexts_[EntropyContext::kLeft][plane][y4 + i].dc_category =
-        dc_category;
-  }
+                              uint8_t coefficient_level, int8_t dc_category) {
+  const int max_x4x4 = frame_header_.columns4x4 >> subsampling_x_[plane];
+  const int num_top_elements = GetNumElements(w4, x4, max_x4x4);
+  memset(&coefficient_levels_[kEntropyContextTop][plane][x4], coefficient_level,
+         num_top_elements);
+  memset(&dc_categories_[kEntropyContextTop][plane][x4], dc_category,
+         num_top_elements);
+  const int max_y4x4 = frame_header_.rows4x4 >> subsampling_y_[plane];
+  const int num_left_elements = GetNumElements(h4, y4, max_y4x4);
+  memset(&coefficient_levels_[kEntropyContextLeft][plane][y4],
+         coefficient_level, num_left_elements);
+  memset(&dc_categories_[kEntropyContextLeft][plane][y4], dc_category,
+         num_left_elements);
 }
 
 void Tile::ScaleMotionVector(const MotionVector& mv, const Plane plane,
@@ -798,8 +950,8 @@
   const bool is_scaled_x = reference_upscaled_width != frame_header_.width;
   const bool is_scaled_y = reference_height != frame_header_.height;
   const int half_sample = 1 << (kSubPixelBits - 1);
-  int orig_x = (x << kSubPixelBits) + ((2 * mv.mv[1]) >> SubsamplingX(plane));
-  int orig_y = (y << kSubPixelBits) + ((2 * mv.mv[0]) >> SubsamplingY(plane));
+  int orig_x = (x << kSubPixelBits) + ((2 * mv.mv[1]) >> subsampling_x_[plane]);
+  int orig_y = (y << kSubPixelBits) + ((2 * mv.mv[0]) >> subsampling_y_[plane]);
   const int rounding_offset =
       DivideBy2(1 << (kScaleSubPixelBits - kSubPixelBits));
   if (is_scaled_x) {
@@ -840,33 +992,112 @@
   }
 }
 
+template <bool is_dc_coefficient>
+bool Tile::ReadSignAndApplyDequantization(
+    const Block& block, const uint16_t* const scan, int i,
+    int adjusted_tx_width_log2, int tx_width, int q_value,
+    const uint8_t* const quantizer_matrix, int shift, int min_value,
+    int max_value, uint16_t* const dc_sign_cdf, int8_t* const dc_category,
+    int* const coefficient_level) {
+  int pos = is_dc_coefficient ? 0 : scan[i];
+  const int pos_index =
+      is_dc_coefficient ? 0 : PaddedIndex(pos, adjusted_tx_width_log2);
+  const bool sign = quantized_[pos_index] != 0 &&
+                    (is_dc_coefficient ? reader_.ReadSymbol(dc_sign_cdf)
+                                       : static_cast<bool>(reader_.ReadBit()));
+  if (quantized_[pos_index] >
+      kNumQuantizerBaseLevels + kQuantizerCoefficientBaseRange) {
+    int length = 0;
+    bool golomb_length_bit = false;
+    do {
+      golomb_length_bit = static_cast<bool>(reader_.ReadBit());
+      ++length;
+      if (length > 20) {
+        LIBGAV1_DLOG(ERROR, "Invalid golomb_length %d", length);
+        return false;
+      }
+    } while (!golomb_length_bit);
+    int x = 1;
+    for (int i = length - 2; i >= 0; --i) {
+      x = (x << 1) | reader_.ReadBit();
+    }
+    quantized_[pos_index] += x - 1;
+  }
+  if (is_dc_coefficient && quantized_[0] > 0) {
+    *dc_category = sign ? -1 : 1;
+  }
+  quantized_[pos_index] &= 0xfffff;
+  *coefficient_level += quantized_[pos_index];
+  // Apply dequantization. Step 1 of section 7.12.3 in the spec.
+  int q = q_value;
+  if (quantizer_matrix != nullptr) {
+    q = RightShiftWithRounding(q * quantizer_matrix[pos], 5);
+  }
+  // The intermediate multiplication can exceed 32 bits, so it has to be
+  // performed by promoting one of the values to int64_t.
+  int32_t dequantized_value =
+      (static_cast<int64_t>(q) * quantized_[pos_index]) & 0xffffff;
+  dequantized_value >>= shift;
+  if (sign) {
+    dequantized_value = -dequantized_value;
+  }
+  // Inverse transform process assumes that the quantized coefficients are
+  // stored as a virtual 2d array of size |tx_width| x |tx_height|. If
+  // transform width is 64, then this assumption is broken because the scan
+  // order used for populating the coefficients for such transforms is the
+  // same as the one used for corresponding transform with width 32 (e.g. the
+  // scan order used for 64x16 is the same as the one used for 32x16). So we
+  // have to recompute the value of pos so that it reflects the index of the
+  // 2d array of size 64 x |tx_height|.
+  if (!is_dc_coefficient && tx_width == 64) {
+    const int row_index = DivideBy32(pos);
+    const int column_index = Mod32(pos);
+    pos = MultiplyBy64(row_index) + column_index;
+  }
+  if (sequence_header_.color_config.bitdepth == 8) {
+    auto* const residual_buffer =
+        reinterpret_cast<int16_t*>(block.sb_buffer->residual);
+    residual_buffer[pos] = Clip3(dequantized_value, min_value, max_value);
+#if LIBGAV1_MAX_BITDEPTH >= 10
+  } else {
+    auto* const residual_buffer =
+        reinterpret_cast<int32_t*>(block.sb_buffer->residual);
+    residual_buffer[pos] = Clip3(dequantized_value, min_value, max_value);
+#endif
+  }
+  return true;
+}
+
+int Tile::ReadCoeffBaseRange(int clamped_tx_size_context, int cdf_context,
+                             int plane_type) {
+  int level = 0;
+  for (int j = 0; j < kCoeffBaseRangeMaxIterations; ++j) {
+    const int coeff_base_range = reader_.ReadSymbol<kCoeffBaseRangeSymbolCount>(
+        symbol_decoder_context_.coeff_base_range_cdf[clamped_tx_size_context]
+                                                    [plane_type][cdf_context]);
+    level += coeff_base_range;
+    if (coeff_base_range < (kCoeffBaseRangeSymbolCount - 1)) break;
+  }
+  return level;
+}
+
 int16_t Tile::ReadTransformCoefficients(const Block& block, Plane plane,
                                         int start_x, int start_y,
                                         TransformSize tx_size,
                                         TransformType* const tx_type) {
   const int x4 = DivideBy4(start_x);
   const int y4 = DivideBy4(start_y);
-  const int w4 = DivideBy4(kTransformWidth[tx_size]);
-  const int h4 = DivideBy4(kTransformHeight[tx_size]);
-
-  const int tx_size_square_min =
-      TransformSizeToSquareTransformIndex(kTransformSizeSquareMin[tx_size]);
-  const int tx_size_square_max =
-      TransformSizeToSquareTransformIndex(kTransformSizeSquareMax[tx_size]);
-  const int tx_size_context =
-      DivideBy2(tx_size_square_min + tx_size_square_max + 1);
+  const int w4 = kTransformWidth4x4[tx_size];
+  const int h4 = kTransformHeight4x4[tx_size];
+  const int tx_size_context = kTransformSizeContext[tx_size];
   int context =
       GetTransformAllZeroContext(block, plane, tx_size, x4, y4, w4, h4);
   const bool all_zero = reader_.ReadSymbol(
       symbol_decoder_context_.all_zero_cdf[tx_size_context][context]);
   if (all_zero) {
     if (plane == kPlaneY) {
-      for (int i = 0; i < w4; ++i) {
-        for (int j = 0; j < h4; ++j) {
-          transform_types_[y4 + j - block.row4x4][x4 + i - block.column4x4] =
-              kTransformTypeDctDct;
-        }
-      }
+      SetTransformType(block, x4, y4, w4, h4, kTransformTypeDctDct,
+                       transform_types_);
     }
     SetEntropyContexts(x4, y4, w4, h4, plane, 0, 0);
     // This is not used in this case, so it can be set to any value.
@@ -921,10 +1152,8 @@
       break;
   }
   const int16_t eob_pt =
-      1 + reader_.ReadSymbol(cdf, kEobPtSymbolCount[eob_multi_size]);
+      1 + reader_.ReadSymbol(cdf, kEobPt16SymbolCount + eob_multi_size);
   int16_t eob = (eob_pt < 2) ? eob_pt : ((1 << (eob_pt - 2)) + 1);
-  int coefficient_level = 0;
-  uint8_t dc_category = 0;
   if (eob_pt >= 3) {
     context = eob_pt - 3;
     const bool eob_extra = reader_.ReadSymbol(
@@ -933,16 +1162,16 @@
     if (eob_extra) eob += 1 << (eob_pt - 3);
     for (int i = 1; i < eob_pt - 2; ++i) {
       assert(eob_pt - i >= 3);
-      assert(eob_pt <= kEobPtSymbolCount[6]);
+      assert(eob_pt <= kEobPt1024SymbolCount);
       if (static_cast<bool>(reader_.ReadBit())) {
         eob += 1 << (eob_pt - i - 3);
       }
     }
   }
-  const uint16_t* scan = GetScan(tx_size, *tx_type);
+  const TransformClass tx_class = GetTransformClass(*tx_type);
+  const uint16_t* scan = kScan[tx_class][tx_size];
   const TransformSize adjusted_tx_size = kAdjustedTransformSize[tx_size];
   const int adjusted_tx_width_log2 = kTransformWidthLog2[adjusted_tx_size];
-  const TransformClass tx_class = GetTransformClass(*tx_type);
   // Lookup used to call the right variant of GetCoeffBaseContext*() based on
   // the transform class.
   static constexpr int (Tile::*kGetCoeffBaseContextFunc[])(
@@ -958,36 +1187,37 @@
       &Tile::GetCoeffBaseRangeContextVertical};
   auto get_coeff_base_range_context_func =
       kGetCoeffBaseRangeContextFunc[tx_class];
-  for (int i = eob - 1; i >= 0; --i) {
-    const uint16_t pos = scan[i];
-    int level;
-    int symbol_count;
-    if (i == eob - 1) {
-      level = 1;
-      context = GetCoeffBaseContextEob(tx_size, i);
-      cdf = symbol_decoder_context_
-                .coeff_base_eob_cdf[tx_size_context][plane_type][context];
-      symbol_count = kCoeffBaseEobSymbolCount;
-    } else {
-      level = 0;
-      context = (this->*get_coeff_base_context_func)(
-          tx_size, adjusted_tx_width_log2, pos);
-      cdf = symbol_decoder_context_
-                .coeff_base_cdf[tx_size_context][plane_type][context];
-      symbol_count = kCoeffBaseSymbolCount;
-    }
-    level += reader_.ReadSymbol(cdf, symbol_count);
+  const int clamped_tx_size_context = std::min(tx_size_context, 3);
+  // Read the last coefficient.
+  {
+    context = GetCoeffBaseContextEob(tx_size, eob - 1);
+    const uint16_t pos = scan[eob - 1];
+    int level =
+        1 + reader_.ReadSymbol(
+                symbol_decoder_context_
+                    .coeff_base_eob_cdf[tx_size_context][plane_type][context],
+                kCoeffBaseEobSymbolCount);
     if (level > kNumQuantizerBaseLevels) {
-      context = (this->*get_coeff_base_range_context_func)(
-          adjusted_tx_width_log2, pos);
-      for (int j = 0; j < kCoeffBaseRangeMaxIterations; ++j) {
-        const int coeff_base_range = reader_.ReadSymbol(
-            symbol_decoder_context_.coeff_base_range_cdf[std::min(
-                tx_size_context, 3)][plane_type][context],
-            kCoeffBaseRangeSymbolCount);
-        level += coeff_base_range;
-        if (coeff_base_range < (kCoeffBaseRangeSymbolCount - 1)) break;
-      }
+      level += ReadCoeffBaseRange(clamped_tx_size_context,
+                                  (this->*get_coeff_base_range_context_func)(
+                                      adjusted_tx_width_log2, pos),
+                                  plane_type);
+    }
+    quantized_[PaddedIndex(pos, adjusted_tx_width_log2)] = level;
+  }
+  // Read all the other coefficients.
+  for (int i = eob - 2; i >= 0; --i) {
+    const uint16_t pos = scan[i];
+    context = (this->*get_coeff_base_context_func)(tx_size,
+                                                   adjusted_tx_width_log2, pos);
+    int level = reader_.ReadSymbol<kCoeffBaseSymbolCount>(
+        symbol_decoder_context_
+            .coeff_base_cdf[tx_size_context][plane_type][context]);
+    if (level > kNumQuantizerBaseLevels) {
+      level += ReadCoeffBaseRange(clamped_tx_size_context,
+                                  (this->*get_coeff_base_range_context_func)(
+                                      adjusted_tx_width_log2, pos),
+                                  plane_type);
     }
     quantized_[PaddedIndex(pos, adjusted_tx_width_log2)] = level;
   }
@@ -998,86 +1228,37 @@
   const int dc_q_value = quantizer_.GetDcValue(plane, current_quantizer_index);
   const int ac_q_value = quantizer_.GetAcValue(plane, current_quantizer_index);
   const int shift = GetQuantizationShift(tx_size);
-  for (int i = 0; i < eob; ++i) {
-    int pos = scan[i];
-    const int pos_index = PaddedIndex(pos, adjusted_tx_width_log2);
-    bool sign = false;
-    if (quantized_[pos_index] != 0) {
-      if (i == 0) {
-        context = GetDcSignContext(x4, y4, w4, h4, plane);
-        sign = reader_.ReadSymbol(
-            symbol_decoder_context_.dc_sign_cdf[plane_type][context]);
-      } else {
-        sign = static_cast<bool>(reader_.ReadBit());
-      }
-    }
-    if (quantized_[pos_index] >
-        kNumQuantizerBaseLevels + kQuantizerCoefficientBaseRange) {
-      int length = 0;
-      bool golomb_length_bit = false;
-      do {
-        golomb_length_bit = static_cast<bool>(reader_.ReadBit());
-        ++length;
-        if (length > 20) {
-          LIBGAV1_DLOG(ERROR, "Invalid golomb_length %d", length);
-          return -1;
-        }
-      } while (!golomb_length_bit);
-      int x = 1;
-      for (int i = length - 2; i >= 0; --i) {
-        x = (x << 1) | reader_.ReadBit();
-      }
-      quantized_[pos_index] += x - 1;
-    }
-    if (pos == 0 && quantized_[pos_index] > 0) {
-      dc_category = sign ? 1 : 2;
-    }
-    quantized_[pos_index] &= 0xfffff;
-    coefficient_level += quantized_[pos_index];
-    // Apply dequantization. Step 1 of section 7.12.3 in the spec.
-    int q = (pos == 0) ? dc_q_value : ac_q_value;
-    if (frame_header_.quantizer.use_matrix &&
-        *tx_type < kTransformTypeIdentityIdentity &&
-        !frame_header_.segmentation.lossless[bp.segment_id] &&
-        frame_header_.quantizer.matrix_level[plane] < 15) {
-      q *= kQuantizerMatrix[frame_header_.quantizer.matrix_level[plane]]
-                           [plane_type][kQuantizerMatrixOffset[tx_size] + pos];
-      q = RightShiftWithRounding(q, 5);
-    }
-    // The intermediate multiplication can exceed 32 bits, so it has to be
-    // performed by promoting one of the values to int64_t.
-    int32_t dequantized_value =
-        (static_cast<int64_t>(q) * quantized_[pos_index]) & 0xffffff;
-    dequantized_value >>= shift;
-    if (sign) {
-      dequantized_value = -dequantized_value;
-    }
-    // Inverse transform process assumes that the quantized coefficients are
-    // stored as a virtual 2d array of size |tx_width| x |tx_height|. If
-    // transform width is 64, then this assumption is broken because the scan
-    // order used for populating the coefficients for such transforms is the
-    // same as the one used for corresponding transform with width 32 (e.g. the
-    // scan order used for 64x16 is the same as the one used for 32x16). So we
-    // have to recompute the value of pos so that it reflects the index of the
-    // 2d array of size 64 x |tx_height|.
-    if (tx_width == 64) {
-      const int row_index = DivideBy32(pos);
-      const int column_index = Mod32(pos);
-      pos = MultiplyBy64(row_index) + column_index;
-    }
-    if (sequence_header_.color_config.bitdepth == 8) {
-      auto* const residual_buffer =
-          reinterpret_cast<int16_t*>(block.sb_buffer->residual);
-      residual_buffer[pos] = Clip3(dequantized_value, min_value, max_value);
-#if LIBGAV1_MAX_BITDEPTH >= 10
-    } else {
-      auto* const residual_buffer =
-          reinterpret_cast<int32_t*>(block.sb_buffer->residual);
-      residual_buffer[pos] = Clip3(dequantized_value, min_value, max_value);
-#endif
+  const uint8_t* const quantizer_matrix =
+      (frame_header_.quantizer.use_matrix &&
+       *tx_type < kTransformTypeIdentityIdentity &&
+       !frame_header_.segmentation.lossless[bp.segment_id] &&
+       frame_header_.quantizer.matrix_level[plane] < 15)
+          ? &kQuantizerMatrix[frame_header_.quantizer.matrix_level[plane]]
+                             [plane_type][kQuantizerMatrixOffset[tx_size]]
+          : nullptr;
+  int coefficient_level = 0;
+  int8_t dc_category = 0;
+  uint16_t* const dc_sign_cdf =
+      (quantized_[0] != 0)
+          ? symbol_decoder_context_.dc_sign_cdf[plane_type][GetDcSignContext(
+                x4, y4, w4, h4, plane)]
+          : nullptr;
+  assert(scan[0] == 0);
+  if (!ReadSignAndApplyDequantization</*is_dc_coefficient=*/true>(
+          block, scan, 0, adjusted_tx_width_log2, tx_width, dc_q_value,
+          quantizer_matrix, shift, min_value, max_value, dc_sign_cdf,
+          &dc_category, &coefficient_level)) {
+    return -1;
+  }
+  for (int i = 1; i < eob; ++i) {
+    if (!ReadSignAndApplyDequantization</*is_dc_coefficient=*/false>(
+            block, scan, i, adjusted_tx_width_log2, tx_width, ac_q_value,
+            quantizer_matrix, shift, min_value, max_value, nullptr, nullptr,
+            &coefficient_level)) {
+      return -1;
     }
   }
-  SetEntropyContexts(x4, y4, w4, h4, plane, std::min(63, coefficient_level),
+  SetEntropyContexts(x4, y4, w4, h4, plane, std::min(4, coefficient_level),
                      dc_category);
   if (split_parse_and_decode_) {
     block.sb_buffer->residual += tx_width * tx_height * residual_size_;
@@ -1089,8 +1270,8 @@
                           int base_y, TransformSize tx_size, int x, int y,
                           ProcessingMode mode) {
   BlockParameters& bp = *block.bp;
-  const int subsampling_x = SubsamplingX(plane);
-  const int subsampling_y = SubsamplingY(plane);
+  const int subsampling_x = subsampling_x_[plane];
+  const int subsampling_y = subsampling_y_[plane];
   const int start_x = base_x + MultiplyBy4(x);
   const int start_y = base_y + MultiplyBy4(y);
   const int max_x = MultiplyBy4(frame_header_.columns4x4) >> subsampling_x;
@@ -1101,8 +1282,8 @@
   const int mask = sequence_header_.use_128x128_superblock ? 31 : 15;
   const int sub_block_row4x4 = row & mask;
   const int sub_block_column4x4 = column & mask;
-  const int step_x = DivideBy4(kTransformWidth[tx_size]);
-  const int step_y = DivideBy4(kTransformHeight[tx_size]);
+  const int step_x = kTransformWidth4x4[tx_size];
+  const int step_y = kTransformHeight4x4[tx_size];
   const bool do_decode = mode == kProcessingModeDecodeOnly ||
                          mode == kProcessingModeParseAndDecode;
   if (do_decode && !bp.is_inter) {
@@ -1122,10 +1303,11 @@
               ? bp.y_mode
               : (bp.uv_mode == kPredictionModeChromaFromLuma ? kPredictionModeDc
                                                              : bp.uv_mode);
-      const int tr_row4x4 = (sub_block_row4x4 >> subsampling_y) - 1;
-      const int tr_column4x4 = (sub_block_column4x4 >> subsampling_x) + step_x;
-      const int bl_row4x4 = (sub_block_row4x4 >> subsampling_y) + step_y;
-      const int bl_column4x4 = (sub_block_column4x4 >> subsampling_x) - 1;
+      const int tr_row4x4 = (sub_block_row4x4 >> subsampling_y);
+      const int tr_column4x4 =
+          (sub_block_column4x4 >> subsampling_x) + step_x + 1;
+      const int bl_row4x4 = (sub_block_row4x4 >> subsampling_y) + step_y + 1;
+      const int bl_column4x4 = (sub_block_column4x4 >> subsampling_x);
       const bool has_left =
           x > 0 || (plane == kPlaneY ? block.left_available
                                      : block.LeftAvailableChroma());
@@ -1135,16 +1317,16 @@
       if (sequence_header_.color_config.bitdepth == 8) {
         IntraPrediction<uint8_t>(
             block, plane, start_x, start_y, has_left, has_top,
-            BlockDecoded(block, plane, tr_row4x4, tr_column4x4, has_top),
-            BlockDecoded(block, plane, bl_row4x4, bl_column4x4, has_left), mode,
-            tx_size);
+            block.sb_buffer->block_decoded[plane][tr_row4x4][tr_column4x4],
+            block.sb_buffer->block_decoded[plane][bl_row4x4][bl_column4x4],
+            mode, tx_size);
 #if LIBGAV1_MAX_BITDEPTH >= 10
       } else {
         IntraPrediction<uint16_t>(
             block, plane, start_x, start_y, has_left, has_top,
-            BlockDecoded(block, plane, tr_row4x4, tr_column4x4, has_top),
-            BlockDecoded(block, plane, bl_row4x4, bl_column4x4, has_left), mode,
-            tx_size);
+            block.sb_buffer->block_decoded[plane][tr_row4x4][tr_column4x4],
+            block.sb_buffer->block_decoded[plane][bl_row4x4][bl_column4x4],
+            mode, tx_size);
 #endif
       }
       if (plane != kPlaneY && bp.uv_mode == kPredictionModeChromaFromLuma) {
@@ -1198,58 +1380,74 @@
     }
   }
   if (do_decode) {
+    bool* block_decoded =
+        &block.sb_buffer
+             ->block_decoded[plane][(sub_block_row4x4 >> subsampling_y) + 1]
+                            [(sub_block_column4x4 >> subsampling_x) + 1];
     for (int i = 0; i < step_y; ++i) {
-      for (int j = 0; j < step_x; ++j) {
-        block.sb_buffer
-            ->block_decoded[plane][(sub_block_row4x4 >> subsampling_y) + i]
-                           [(sub_block_column4x4 >> subsampling_x) + j] = true;
-      }
+      static_assert(sizeof(bool) == 1, "");
+      memset(block_decoded, 1, step_x);
+      block_decoded += kBlockDecodedStride;
     }
   }
   return true;
 }
 
 bool Tile::TransformTree(const Block& block, int start_x, int start_y,
-                         int width, int height, ProcessingMode mode) {
-  const int row = DivideBy4(start_y);
-  const int column = DivideBy4(start_x);
-  if (row >= frame_header_.rows4x4 || column >= frame_header_.columns4x4) {
-    return true;
-  }
-  const TransformSize inter_tx_size = inter_transform_sizes_[row][column];
-  if (width <= kTransformWidth[inter_tx_size] &&
-      height <= kTransformHeight[inter_tx_size]) {
-    TransformSize tx_size = kNumTransformSizes;
-    for (int i = 0; i < kNumTransformSizes; ++i) {
-      if (kTransformWidth[i] == width && kTransformHeight[i] == height) {
-        tx_size = static_cast<TransformSize>(i);
-        break;
-      }
+                         BlockSize plane_size, ProcessingMode mode) {
+  assert(plane_size <= kBlock64x64);
+  // Branching factor is 4; Maximum Depth is 4; So the maximum stack size
+  // required is (4 - 1) * 4 + 1 = 13.
+  Stack<TransformTreeNode, 13> stack;
+  // It is okay to cast BlockSize to TransformSize here since the enum are
+  // equivalent for all BlockSize values <= kBlock64x64.
+  stack.Push(TransformTreeNode(start_x, start_y,
+                               static_cast<TransformSize>(plane_size)));
+
+  while (!stack.Empty()) {
+    TransformTreeNode node = stack.Pop();
+    const int row = DivideBy4(node.y);
+    const int column = DivideBy4(node.x);
+    if (row >= frame_header_.rows4x4 || column >= frame_header_.columns4x4) {
+      continue;
     }
-    assert(tx_size < kNumTransformSizes);
-    return TransformBlock(block, kPlaneY, start_x, start_y, tx_size, 0, 0,
-                          mode);
+    const TransformSize inter_tx_size = inter_transform_sizes_[row][column];
+    const int width = kTransformWidth[node.tx_size];
+    const int height = kTransformHeight[node.tx_size];
+    if (width <= kTransformWidth[inter_tx_size] &&
+        height <= kTransformHeight[inter_tx_size]) {
+      if (!TransformBlock(block, kPlaneY, node.x, node.y, node.tx_size, 0, 0,
+                          mode)) {
+        return false;
+      }
+      continue;
+    }
+    // The split transform size look up gives the right transform size that we
+    // should push in the stack.
+    //   if (width > height) => transform size whose width is half.
+    //   if (width < height) => transform size whose height is half.
+    //   if (width == height) => transform size whose width and height are half.
+    const TransformSize split_tx_size = kSplitTransformSize[node.tx_size];
+    const int half_width = DivideBy2(width);
+    if (width > height) {
+      stack.Push(TransformTreeNode(node.x + half_width, node.y, split_tx_size));
+      stack.Push(TransformTreeNode(node.x, node.y, split_tx_size));
+      continue;
+    }
+    const int half_height = DivideBy2(height);
+    if (width < height) {
+      stack.Push(
+          TransformTreeNode(node.x, node.y + half_height, split_tx_size));
+      stack.Push(TransformTreeNode(node.x, node.y, split_tx_size));
+      continue;
+    }
+    stack.Push(TransformTreeNode(node.x + half_width, node.y + half_height,
+                                 split_tx_size));
+    stack.Push(TransformTreeNode(node.x, node.y + half_height, split_tx_size));
+    stack.Push(TransformTreeNode(node.x + half_width, node.y, split_tx_size));
+    stack.Push(TransformTreeNode(node.x, node.y, split_tx_size));
   }
-  const int half_width = DivideBy2(width);
-  const int half_height = DivideBy2(height);
-  if (width > height) {
-    return TransformTree(block, start_x, start_y, half_width, height, mode) &&
-           TransformTree(block, start_x + half_width, start_y, half_width,
-                         height, mode);
-  }
-  if (width < height) {
-    return TransformTree(block, start_x, start_y, width, half_height, mode) &&
-           TransformTree(block, start_x, start_y + half_height, width,
-                         half_height, mode);
-  }
-  return TransformTree(block, start_x, start_y, half_width, half_height,
-                       mode) &&
-         TransformTree(block, start_x + half_width, start_y, half_width,
-                       half_height, mode) &&
-         TransformTree(block, start_x, start_y + half_height, half_width,
-                       half_height, mode) &&
-         TransformTree(block, start_x + half_width, start_y + half_height,
-                       half_width, half_height, mode);
+  return true;
 }
 
 void Tile::ReconstructBlock(const Block& block, Plane plane, int start_x,
@@ -1260,7 +1458,7 @@
   if (non_zero_coeff_count == 0) return;
   // Reconstruction process. Steps 2 and 3 of Section 7.12.3 in the spec.
   if (sequence_header_.color_config.bitdepth == 8) {
-    Reconstruct(dsp_, tx_type, tx_size, sequence_header_.color_config.bitdepth,
+    Reconstruct(dsp_, tx_type, tx_size,
                 frame_header_.segmentation.lossless[block.bp->segment_id],
                 reinterpret_cast<int16_t*>(block.sb_buffer->residual), start_x,
                 start_y, &buffer_[plane], non_zero_coeff_count);
@@ -1269,7 +1467,7 @@
     Array2DView<uint16_t> buffer(
         buffer_[plane].rows(), buffer_[plane].columns() / sizeof(uint16_t),
         reinterpret_cast<uint16_t*>(&buffer_[plane][0][0]));
-    Reconstruct(dsp_, tx_type, tx_size, sequence_header_.color_config.bitdepth,
+    Reconstruct(dsp_, tx_type, tx_size,
                 frame_header_.segmentation.lossless[block.bp->segment_id],
                 reinterpret_cast<int32_t*>(block.sb_buffer->residual), start_x,
                 start_y, &buffer, non_zero_coeff_count);
@@ -1287,17 +1485,17 @@
   const BlockSize size_chunk4x4 =
       (width_chunks > 1 || height_chunks > 1) ? kBlock64x64 : block.size;
   const BlockParameters& bp = *block.bp;
-
+  const TransformSize y_tx_size =
+      GetTransformSize(frame_header_.segmentation.lossless[bp.segment_id],
+                       block.size, kPlaneY, bp.transform_size, 0, 0);
   for (int chunk_y = 0; chunk_y < height_chunks; ++chunk_y) {
     for (int chunk_x = 0; chunk_x < width_chunks; ++chunk_x) {
       for (int plane = 0; plane < (block.HasChroma() ? PlaneCount() : 1);
            ++plane) {
-        const int subsampling_x = SubsamplingX(static_cast<Plane>(plane));
-        const int subsampling_y = SubsamplingY(static_cast<Plane>(plane));
+        const int subsampling_x = subsampling_x_[plane];
+        const int subsampling_y = subsampling_y_[plane];
         const TransformSize tx_size =
-            GetTransformSize(frame_header_.segmentation.lossless[bp.segment_id],
-                             block.size, static_cast<Plane>(plane),
-                             bp.transform_size, subsampling_x, subsampling_y);
+            (plane == kPlaneY) ? y_tx_size : bp.uv_transform_size;
         const BlockSize plane_size =
             kPlaneResidualSize[size_chunk4x4][subsampling_x][subsampling_y];
         assert(plane_size != kBlockInvalid);
@@ -1308,16 +1506,14 @@
           const int column_chunk4x4 = block.column4x4 + MultiplyBy16(chunk_x);
           const int base_x = MultiplyBy4(column_chunk4x4 >> subsampling_x);
           const int base_y = MultiplyBy4(row_chunk4x4 >> subsampling_y);
-          if (!TransformTree(block, base_x, base_y,
-                             kBlockWidthPixels[plane_size],
-                             kBlockHeightPixels[plane_size], mode)) {
+          if (!TransformTree(block, base_x, base_y, plane_size, mode)) {
             return false;
           }
         } else {
           const int base_x = MultiplyBy4(block.column4x4 >> subsampling_x);
           const int base_y = MultiplyBy4(block.row4x4 >> subsampling_y);
-          const int step_x = DivideBy4(kTransformWidth[tx_size]);
-          const int step_y = DivideBy4(kTransformHeight[tx_size]);
+          const int step_x = kTransformWidth4x4[tx_size];
+          const int step_y = kTransformHeight4x4[tx_size];
           const int num4x4_wide = kNum4x4BlocksWide[plane_size];
           const int num4x4_high = kNum4x4BlocksHigh[plane_size];
           for (int y = 0; y < num4x4_high; y += step_y) {
@@ -1366,10 +1562,10 @@
   const int src_bottom_edge = src_top_edge + block_height;
   const int src_right_edge = src_left_edge + block_width;
   if (block.HasChroma()) {
-    if (block_width < 8 && sequence_header_.color_config.subsampling_x != 0) {
+    if (block_width < 8 && subsampling_x_[kPlaneU] != 0) {
       src_left_edge -= 4;
     }
-    if (block_height < 8 && sequence_header_.color_config.subsampling_y != 0) {
+    if (block_height < 8 && subsampling_y_[kPlaneU] != 0) {
       src_top_edge -= 4;
     }
   }
@@ -1455,22 +1651,24 @@
   const int block_width4x4 = kNum4x4BlocksWide[block.size];
   const int block_height4x4 = kNum4x4BlocksHigh[block.size];
   for (int plane = 0; plane < (block.HasChroma() ? PlaneCount() : 1); ++plane) {
-    const int subsampling_x = SubsamplingX(static_cast<Plane>(plane));
+    const int subsampling_x = subsampling_x_[plane];
     const int start_x = block.column4x4 >> subsampling_x;
     const int end_x =
         std::min((block.column4x4 + block_width4x4) >> subsampling_x,
                  frame_header_.columns4x4);
-    for (int x = start_x; x < end_x; ++x) {
-      entropy_contexts_[EntropyContext::kTop][plane][x] = {};
-    }
-    const int subsampling_y = SubsamplingY(static_cast<Plane>(plane));
+    memset(&coefficient_levels_[kEntropyContextTop][plane][start_x], 0,
+           end_x - start_x);
+    memset(&dc_categories_[kEntropyContextTop][plane][start_x], 0,
+           end_x - start_x);
+    const int subsampling_y = subsampling_y_[plane];
     const int start_y = block.row4x4 >> subsampling_y;
     const int end_y =
         std::min((block.row4x4 + block_height4x4) >> subsampling_y,
                  frame_header_.rows4x4);
-    for (int y = start_y; y < end_y; ++y) {
-      entropy_contexts_[EntropyContext::kLeft][plane][y] = {};
-    }
+    memset(&coefficient_levels_[kEntropyContextLeft][plane][start_y], 0,
+           end_y - start_y);
+    memset(&dc_categories_[kEntropyContextLeft][plane][start_y], 0,
+           end_y - start_y);
   }
 }
 
@@ -1488,25 +1686,25 @@
   // Local warping parameters, similar usage as is_local_valid.
   GlobalMotion local_warp_params;
   for (int plane = 0; plane < (block.HasChroma() ? PlaneCount() : 1); ++plane) {
-    const int8_t subsampling_x = SubsamplingX(static_cast<Plane>(plane));
-    const int8_t subsampling_y = SubsamplingY(static_cast<Plane>(plane));
+    const int8_t subsampling_x = subsampling_x_[plane];
+    const int8_t subsampling_y = subsampling_y_[plane];
     const BlockSize plane_size =
         kPlaneResidualSize[block.size][subsampling_x][subsampling_y];
     assert(plane_size != kBlockInvalid);
     const int block_width4x4 = kNum4x4BlocksWide[plane_size];
     const int block_height4x4 = kNum4x4BlocksHigh[plane_size];
-    const int block_width = kBlockWidthPixels[plane_size];
-    const int block_height = kBlockHeightPixels[plane_size];
+    const int block_width = MultiplyBy4(block_width4x4);
+    const int block_height = MultiplyBy4(block_height4x4);
     const int base_x = MultiplyBy4(block.column4x4 >> subsampling_x);
     const int base_y = MultiplyBy4(block.row4x4 >> subsampling_y);
     const BlockParameters& bp = *block.bp;
     if (bp.is_inter && bp.reference_frame[1] == kReferenceFrameIntra) {
-      const int tr_row4x4 = (sub_block_row4x4 >> subsampling_y) - 1;
+      const int tr_row4x4 = (sub_block_row4x4 >> subsampling_y);
       const int tr_column4x4 =
-          (sub_block_column4x4 >> subsampling_x) + block_width4x4;
+          (sub_block_column4x4 >> subsampling_x) + block_width4x4 + 1;
       const int bl_row4x4 =
           (sub_block_row4x4 >> subsampling_y) + block_height4x4;
-      const int bl_column4x4 = (sub_block_column4x4 >> subsampling_x) - 1;
+      const int bl_column4x4 = (sub_block_column4x4 >> subsampling_x) + 1;
       const TransformSize tx_size =
           k4x4SizeToTransformSize[k4x4WidthLog2[plane_size]]
                                  [k4x4HeightLog2[plane_size]];
@@ -1517,10 +1715,8 @@
       if (sequence_header_.color_config.bitdepth == 8) {
         IntraPrediction<uint8_t>(
             block, static_cast<Plane>(plane), base_x, base_y, has_left, has_top,
-            BlockDecoded(block, static_cast<Plane>(plane), tr_row4x4,
-                         tr_column4x4, has_top),
-            BlockDecoded(block, static_cast<Plane>(plane), bl_row4x4,
-                         bl_column4x4, has_left),
+            block.sb_buffer->block_decoded[plane][tr_row4x4][tr_column4x4],
+            block.sb_buffer->block_decoded[plane][bl_row4x4][bl_column4x4],
             kInterIntraToIntraMode[block.bp->prediction_parameters
                                        ->inter_intra_mode],
             tx_size);
@@ -1528,10 +1724,8 @@
       } else {
         IntraPrediction<uint16_t>(
             block, static_cast<Plane>(plane), base_x, base_y, has_left, has_top,
-            BlockDecoded(block, static_cast<Plane>(plane), tr_row4x4,
-                         tr_column4x4, has_top),
-            BlockDecoded(block, static_cast<Plane>(plane), bl_row4x4,
-                         bl_column4x4, has_left),
+            block.sb_buffer->block_decoded[plane][tr_row4x4][tr_column4x4],
+            block.sb_buffer->block_decoded[plane][bl_row4x4][bl_column4x4],
             kInterIntraToIntraMode[block.bp->prediction_parameters
                                        ->inter_intra_mode],
             tx_size);
@@ -1580,12 +1774,18 @@
   return true;
 }
 
-void Tile::ComputeDeblockFilterLevel(const Block& block) {
+void Tile::PopulateDeblockFilterLevel(const Block& block) {
+  if (!post_filter_.DoDeblock()) return;
   BlockParameters& bp = *block.bp;
-  for (int plane = kPlaneY; plane < PlaneCount(); ++plane) {
-    for (int i = kLoopFilterTypeVertical; i < kNumLoopFilterTypes; ++i) {
-      bp.deblock_filter_level[plane][i] = LoopFilterMask::GetDeblockFilterLevel(
-          frame_header_, bp, static_cast<Plane>(plane), i, delta_lf_);
+  for (int i = 0; i < kFrameLfCount; ++i) {
+    if (delta_lf_all_zero_) {
+      bp.deblock_filter_level[i] = post_filter_.GetZeroDeltaDeblockFilterLevel(
+          bp.segment_id, i, bp.reference_frame[0],
+          LoopFilterMask::GetModeId(bp.y_mode));
+    } else {
+      bp.deblock_filter_level[i] =
+          deblock_filter_levels_[bp.segment_id][i][bp.reference_frame[0]]
+                                [LoopFilterMask::GetModeId(bp.y_mode)];
     }
   }
 }
@@ -1612,10 +1812,13 @@
                               : std::move(prediction_parameters_);
   if (block.bp->prediction_parameters == nullptr) return false;
   if (!DecodeModeInfo(block)) return false;
-  ComputeDeblockFilterLevel(block);
-  ReadPaletteTokens(block);
+  PopulateDeblockFilterLevel(block);
+  if (!ReadPaletteTokens(block)) return false;
   DecodeTransformSize(block);
-  const BlockParameters& bp = *block.bp;
+  BlockParameters& bp = *block.bp;
+  bp.uv_transform_size = GetTransformSize(
+      frame_header_.segmentation.lossless[bp.segment_id], block.size, kPlaneU,
+      bp.transform_size, subsampling_x_[kPlaneU], subsampling_y_[kPlaneU]);
   if (bp.skip) ResetEntropyContext(block);
   const int block_width4x4 = kNum4x4BlocksWide[block_size];
   const int block_height4x4 = kNum4x4BlocksHigh[block_size];
@@ -1644,8 +1847,10 @@
     current_frame_.segmentation_map()->FillBlock(row4x4, column4x4, x_limit,
                                                  y_limit, bp.segment_id);
   }
-  if (!split_parse_and_decode_) {
+  if (build_bit_mask_when_parsing_ || !split_parse_and_decode_) {
     BuildBitMask(row4x4, column4x4, block_size);
+  }
+  if (!split_parse_and_decode_) {
     StoreMotionFieldMvsIntoCurrentFrame(block);
     prediction_parameters_ = std::move(block.bp->prediction_parameters);
   }
@@ -1667,7 +1872,9 @@
       !Residual(block, kProcessingModeDecodeOnly)) {
     return false;
   }
-  BuildBitMask(row4x4, column4x4, block_size);
+  if (!build_bit_mask_when_parsing_) {
+    BuildBitMask(row4x4, column4x4, block_size);
+  }
   StoreMotionFieldMvsIntoCurrentFrame(block);
   block.bp->prediction_parameters.reset(nullptr);
   return true;
@@ -1676,8 +1883,7 @@
 bool Tile::ProcessPartition(int row4x4_start, int column4x4_start,
                             ParameterTree* const root,
                             SuperBlockBuffer* const sb_buffer) {
-  std::vector<ParameterTree*> stack;
-  stack.reserve(kDfsStackSize);
+  Stack<ParameterTree*, kDfsStackSize> stack;
 
   // Set up the first iteration.
   ParameterTree* node = root;
@@ -1688,10 +1894,9 @@
   // DFS loop. If it sees a terminal node (leaf node), ProcessBlock is invoked.
   // Otherwise, the children are pushed into the stack for future processing.
   do {
-    if (!stack.empty()) {
+    if (!stack.Empty()) {
       // Set up subsequent iterations.
-      node = stack.back();
-      stack.pop_back();
+      node = stack.Pop();
       row4x4 = node->row4x4();
       column4x4 = node->column4x4();
       block_size = node->block_size();
@@ -1731,7 +1936,10 @@
           sequence_header_.color_config.subsampling_y);
       return false;
     }
-    node->SetPartitionType(partition);
+    if (!node->SetPartitionType(partition)) {
+      LIBGAV1_DLOG(ERROR, "node->SetPartitionType() failed.");
+      return false;
+    }
     switch (partition) {
       case kPartitionNone:
         if (!ProcessBlock(row4x4, column4x4, sub_size, node, sb_buffer)) {
@@ -1744,7 +1952,7 @@
         for (int i = 3; i >= 0; --i) {
           ParameterTree* const child = node->children(i);
           assert(child != nullptr);
-          stack.push_back(child);
+          stack.Push(child);
         }
         break;
       case kPartitionHorizontal:
@@ -1767,7 +1975,7 @@
         }
         break;
     }
-  } while (!stack.empty());
+  } while (!stack.Empty());
   return true;
 }
 
@@ -1799,6 +2007,40 @@
   }
 }
 
+void Tile::ClearBlockDecoded(SuperBlockBuffer* const sb_buffer, int row4x4,
+                             int column4x4) {
+  // Set everything to false.
+  memset(sb_buffer->block_decoded, 0, sizeof(sb_buffer->block_decoded));
+  // Set specific edge cases to true.
+  const int sb_size4 = sequence_header_.use_128x128_superblock ? 32 : 16;
+  for (int plane = 0; plane < PlaneCount(); ++plane) {
+    const int subsampling_x = subsampling_x_[plane];
+    const int subsampling_y = subsampling_y_[plane];
+    const int sb_width4 = (column4x4_end_ - column4x4) >> subsampling_x;
+    const int sb_height4 = (row4x4_end_ - row4x4) >> subsampling_y;
+    // The memset is equivalent to the following lines in the spec:
+    // for ( x = -1; x <= ( sbSize4 >> subX ); x++ ) {
+    //   if ( y < 0 && x < sbWidth4 ) {
+    //     BlockDecoded[plane][y][x] = 1
+    //   }
+    // }
+    const int num_elements =
+        std::min((sb_size4 >> subsampling_x_[plane]) + 1, sb_width4) + 1;
+    memset(&sb_buffer->block_decoded[plane][0][0], 1, num_elements);
+    // The for loop is equivalent to the following lines in the spec:
+    // for ( y = -1; y <= ( sbSize4 >> subY ); y++ )
+    //   if ( x < 0 && y < sbHeight4 )
+    //     BlockDecoded[plane][y][x] = 1
+    //   }
+    // }
+    // BlockDecoded[plane][sbSize4 >> subY][-1] = 0
+    for (int y = -1; y < std::min((sb_size4 >> subsampling_y), sb_height4);
+         ++y) {
+      sb_buffer->block_decoded[plane][y + 1][0] = true;
+    }
+  }
+}
+
 bool Tile::ProcessSuperBlock(int row4x4, int column4x4, int block_width4x4,
                              SuperBlockBuffer* const sb_buffer,
                              ProcessingMode mode) {
@@ -1811,10 +2053,7 @@
     ResetCdef(row4x4, column4x4);
   }
   if (decoding) {
-    memset(sb_buffer->block_decoded, 0,
-           sizeof(sb_buffer->block_decoded));  // Section 5.11.3.
-    sb_buffer->block_decoded_width_threshold = column4x4_end_ - column4x4;
-    sb_buffer->block_decoded_height_threshold = row4x4_end_ - row4x4;
+    ClearBlockDecoded(sb_buffer, row4x4, column4x4);
   }
   const BlockSize block_size = SuperBlockSize();
   if (parsing) {
@@ -1843,10 +2082,10 @@
       return false;
     }
     sb_buffer->residual =
-        residual_buffer_threaded_[sb_row_index][sb_column_index]->buffer.get();
+        residual_buffer_threaded_[sb_row_index][sb_column_index]->buffer();
     sb_buffer->transform_parameters =
-        &residual_buffer_threaded_[sb_row_index][sb_column_index]
-             ->transform_parameters;
+        residual_buffer_threaded_[sb_row_index][sb_column_index]
+            ->transform_parameters();
     if (!ProcessPartition(row4x4, column4x4,
                           block_parameters_holder_.Tree(row, column),
                           sb_buffer)) {
@@ -1856,10 +2095,10 @@
     }
   } else {
     sb_buffer->residual =
-        residual_buffer_threaded_[sb_row_index][sb_column_index]->buffer.get();
+        residual_buffer_threaded_[sb_row_index][sb_column_index]->buffer();
     sb_buffer->transform_parameters =
-        &residual_buffer_threaded_[sb_row_index][sb_column_index]
-             ->transform_parameters;
+        residual_buffer_threaded_[sb_row_index][sb_column_index]
+            ->transform_parameters();
     if (!DecodeSuperBlock(block_parameters_holder_.Tree(row, column),
                           sb_buffer)) {
       LIBGAV1_DLOG(ERROR, "Error decoding superblock row: %d column: %d",
@@ -1874,16 +2113,14 @@
 
 bool Tile::DecodeSuperBlock(ParameterTree* const tree,
                             SuperBlockBuffer* const sb_buffer) {
-  std::vector<ParameterTree*> stack;
-  stack.reserve(kDfsStackSize);
-  stack.push_back(tree);
-  while (!stack.empty()) {
-    ParameterTree* const node = stack.back();
-    stack.pop_back();
+  Stack<ParameterTree*, kDfsStackSize> stack;
+  stack.Push(tree);
+  while (!stack.Empty()) {
+    ParameterTree* const node = stack.Pop();
     if (node->partition() != kPartitionNone) {
       for (int i = 3; i >= 0; --i) {
         if (node->children(i) == nullptr) continue;
-        stack.push_back(node->children(i));
+        stack.Push(node->children(i));
       }
       continue;
     }
@@ -1926,14 +2163,13 @@
 
 void Tile::BuildBitMask(int row4x4, int column4x4, BlockSize block_size) {
   if (!post_filter_.DoDeblock()) return;
-  const int block_width4x4 = kNum4x4BlocksWide[block_size];
-  const int block_height4x4 = kNum4x4BlocksHigh[block_size];
-  if (block_width4x4 <= kNum4x4BlocksWide[kBlock64x64] &&
-      block_height4x4 <= kNum4x4BlocksHigh[kBlock64x64]) {
+  if (block_size <= kBlock64x64) {
     BuildBitMaskHelper(row4x4, column4x4, block_size, true, true);
   } else {
-    for (int y = 0; y < block_height4x4; y += kNum4x4BlocksHigh[kBlock64x64]) {
-      for (int x = 0; x < block_width4x4; x += kNum4x4BlocksWide[kBlock64x64]) {
+    const int block_width4x4 = kNum4x4BlocksWide[block_size];
+    const int block_height4x4 = kNum4x4BlocksHigh[block_size];
+    for (int y = 0; y < block_height4x4; y += 16) {
+      for (int x = 0; x < block_width4x4; x += 16) {
         BuildBitMaskHelper(row4x4 + y, column4x4 + x, kBlock64x64, x == 0,
                            y == 0);
       }
@@ -1951,12 +2187,6 @@
   LoopFilterMask* const masks = post_filter_.masks();
   const int unit_id = DivideBy16(row4x4) * masks->num_64x64_blocks_per_row() +
                       DivideBy16(column4x4);
-  const int row_limit = row4x4 + block_height4x4;
-  const int column_limit = column4x4 + block_width4x4;
-  const TransformSize current_block_uv_tx_size = GetTransformSize(
-      frame_header_.segmentation.lossless[bp.segment_id], block_size, kPlaneU,
-      kNumTransformSizes,  // This parameter is unused when plane != Y.
-      SubsamplingX(kPlaneU), SubsamplingY(kPlaneU));
 
   for (int plane = kPlaneY; plane < PlaneCount(); ++plane) {
     // For U and V planes, do not build bit masks if level == 0.
@@ -1964,38 +2194,39 @@
       continue;
     }
     // Build bit mask for vertical edges.
-    const int subsampling_x = SubsamplingX(static_cast<Plane>(plane));
-    const int subsampling_y = SubsamplingY(static_cast<Plane>(plane));
+    const int subsampling_x = subsampling_x_[plane];
+    const int subsampling_y = subsampling_y_[plane];
     const int plane_width =
         RightShiftWithRounding(frame_header_.width, subsampling_x);
+    const int column_limit =
+        std::min({column4x4 + block_width4x4, frame_header_.columns4x4,
+                  DivideBy4(plane_width + 3) << subsampling_x});
     const int plane_height =
         RightShiftWithRounding(frame_header_.height, subsampling_y);
-    const int vertical_step = 1 << subsampling_y;
-    const int horizontal_step = 1 << subsampling_x;
+    const int row_limit =
+        std::min({row4x4 + block_height4x4, frame_header_.rows4x4,
+                  DivideBy4(plane_height + 3) << subsampling_y});
     const int row_start = GetDeblockPosition(row4x4, subsampling_y);
     const int column_start = GetDeblockPosition(column4x4, subsampling_x);
-    if (row_start >= row4x4 + block_height4x4 ||
-        MultiplyBy4(row_start >> subsampling_y) >= plane_height ||
-        column_start >= column4x4 + block_width4x4 ||
-        MultiplyBy4(column_start >> subsampling_x) >= plane_width) {
+    if (row_start >= row_limit || column_start >= column_limit) {
       continue;
     }
+    const int vertical_step = 1 << subsampling_y;
+    const int horizontal_step = 1 << subsampling_x;
     const BlockParameters& bp =
         *block_parameters_holder_.Find(row_start, column_start);
+    const int horizontal_level_index =
+        kDeblockFilterLevelIndex[plane][kLoopFilterTypeHorizontal];
+    const int vertical_level_index =
+        kDeblockFilterLevelIndex[plane][kLoopFilterTypeVertical];
     const uint8_t vertical_level =
-        bp.deblock_filter_level[plane][kLoopFilterTypeVertical];
+        bp.deblock_filter_level[vertical_level_index];
 
-    for (int row = row_start;
-         row < row_limit && MultiplyBy4(row >> subsampling_y) < plane_height &&
-         row < frame_header_.rows4x4;
-         row += vertical_step) {
-      for (int column = column_start;
-           column < column_limit &&
-           MultiplyBy4(column >> subsampling_x) < plane_width &&
-           column < frame_header_.columns4x4;) {
+    for (int row = row_start; row < row_limit; row += vertical_step) {
+      for (int column = column_start; column < column_limit;) {
         const TransformSize tx_size = (plane == kPlaneY)
                                           ? inter_transform_sizes_[row][column]
-                                          : current_block_uv_tx_size;
+                                          : bp.uv_transform_size;
         // (1). Don't filter frame boundary.
         // (2). For tile boundary, we don't know whether the previous tile is
         // available or not, thus we handle it after all tiles are decoded.
@@ -2013,22 +2244,19 @@
             *block_parameters_holder_.Find(row, column - horizontal_step);
         const uint8_t left_level =
             is_vertical_border
-                ? bp_left.deblock_filter_level[plane][kLoopFilterTypeVertical]
+                ? bp_left.deblock_filter_level[vertical_level_index]
                 : vertical_level;
         // We don't have to check if the left block is skipped or not,
         // because if the current transform block is on the edge of the coding
         // block, is_vertical_border is true; if it's not on the edge,
         // left skip is equal to skip.
         if (vertical_level != 0 || left_level != 0) {
-          const TransformSize left_tx_size = GetTransformSize(
-              frame_header_.segmentation.lossless[bp_left.segment_id],
-              bp_left.size, static_cast<Plane>(plane),
-              inter_transform_sizes_[row][column - horizontal_step],
-              subsampling_x, subsampling_y);
-          // 0: 4x4, 1: 8x8, 2: 16x16.
-          const int transform_size_id =
-              std::min({kTransformWidthLog2[tx_size] - 2,
-                        kTransformWidthLog2[left_tx_size] - 2, 2});
+          const TransformSize left_tx_size =
+              (plane == kPlaneY)
+                  ? inter_transform_sizes_[row][column - horizontal_step]
+                  : bp_left.uv_transform_size;
+          const LoopFilterTransformSizeId transform_size_id =
+              GetTransformSizeIdWidth(tx_size, left_tx_size);
           const int r = row & (kNum4x4InLoopFilterMaskUnit - 1);
           const int c = column & (kNum4x4InLoopFilterMaskUnit - 1);
           const int shift = LoopFilterMask::GetShift(r, c);
@@ -2047,19 +2275,13 @@
 
     // Build bit mask for horizontal edges.
     const uint8_t horizontal_level =
-        bp.deblock_filter_level[plane][kLoopFilterTypeHorizontal];
-    for (int column = column_start;
-         column < column_limit &&
-         MultiplyBy4(column >> subsampling_x) < plane_width &&
-         column < frame_header_.columns4x4;
+        bp.deblock_filter_level[horizontal_level_index];
+    for (int column = column_start; column < column_limit;
          column += horizontal_step) {
-      for (int row = row_start;
-           row < row_limit &&
-           MultiplyBy4(row >> subsampling_y) < plane_height &&
-           row < frame_header_.rows4x4;) {
+      for (int row = row_start; row < row_limit;) {
         const TransformSize tx_size = (plane == kPlaneY)
                                           ? inter_transform_sizes_[row][column]
-                                          : current_block_uv_tx_size;
+                                          : bp.uv_transform_size;
 
         // (1). Don't filter frame boundary.
         // (2). For tile boundary, we don't know whether the previous tile is
@@ -2078,22 +2300,21 @@
             *block_parameters_holder_.Find(row - vertical_step, column);
         const uint8_t top_level =
             is_horizontal_border
-                ? bp_top.deblock_filter_level[plane][kLoopFilterTypeHorizontal]
+                ? bp_top.deblock_filter_level[horizontal_level_index]
                 : horizontal_level;
         // We don't have to check it the top block is skippped or not,
         // because if the current transform block is on the edge of the coding
         // block, is_horizontal_border is true; if it's not on the edge,
         // top skip is equal to skip.
         if (horizontal_level != 0 || top_level != 0) {
-          const TransformSize top_tx_size = GetTransformSize(
-              frame_header_.segmentation.lossless[bp_top.segment_id],
-              bp_top.size, static_cast<Plane>(plane),
-              inter_transform_sizes_[row - vertical_step][column],
-              subsampling_x, subsampling_y);
-          // 0: 4x4, 1: 8x8, 2: 16x16.
-          const int transform_size_id =
-              std::min({kTransformHeightLog2[tx_size] - 2,
-                        kTransformHeightLog2[top_tx_size] - 2, 2});
+          const TransformSize top_tx_size =
+              (plane == kPlaneY)
+                  ? inter_transform_sizes_[row - vertical_step][column]
+                  : bp_top.uv_transform_size;
+          const LoopFilterTransformSizeId transform_size_id =
+              static_cast<LoopFilterTransformSizeId>(
+                  std::min({kTransformHeightLog2[tx_size] - 2,
+                            kTransformHeightLog2[top_tx_size] - 2, 2}));
           const int r = row & (kNum4x4InLoopFilterMaskUnit - 1);
           const int c = column & (kNum4x4InLoopFilterMaskUnit - 1);
           const int shift = LoopFilterMask::GetShift(r, c);
diff --git a/libgav1/src/utils/allocator.cc b/libgav1/src/utils/allocator.cc
deleted file mode 100644
index 309aabc..0000000
--- a/libgav1/src/utils/allocator.cc
+++ /dev/null
@@ -1,25 +0,0 @@
-#include "src/utils/allocator.h"
-
-#include <cstdlib>
-
-namespace libgav1 {
-
-AllocatorBase::AllocatorBase(size_t element_size)
-    : element_size_(element_size), ok_(true) {}
-
-void* AllocatorBase::Allocate(size_t s) {
-  if (!ok_) return nullptr;
-  void* const ptr = malloc(s);
-  ok_ = !(s > 0 && ptr == nullptr);
-  return ptr;
-}
-
-bool AllocatorBase::Deallocate(void* p, size_t /*nb_elements*/) {
-  if (!ok_) return false;
-  free(p);
-  return true;
-}
-
-void AllocatorBase::Init(size_t /*element_size*/) {}
-
-}  // namespace libgav1
diff --git a/libgav1/src/utils/allocator.h b/libgav1/src/utils/allocator.h
deleted file mode 100644
index 79e5355..0000000
--- a/libgav1/src/utils/allocator.h
+++ /dev/null
@@ -1,94 +0,0 @@
-#ifndef LIBGAV1_SRC_UTILS_ALLOCATOR_H_
-#define LIBGAV1_SRC_UTILS_ALLOCATOR_H_
-
-#include <cstddef>
-#include <utility>
-
-namespace libgav1 {
-
-class AllocatorBase {
- public:
-  bool ok() const { return ok_; }
-
- protected:
-  explicit AllocatorBase(size_t element_size);
-
-  void* Allocate(size_t s);
-  bool Deallocate(void* p, size_t nb_elements);
-  void Init(size_t element_size);
-
-  size_t element_size_;
-  bool ok_;
-};
-
-// This allocator will NOT call the constructor, since the construct()
-// method has been voided.
-template <typename T>
-class AllocatorNoCtor : protected AllocatorBase {
- public:
-  using value_type = T;
-  AllocatorNoCtor() : AllocatorBase(sizeof(T)) {}
-  template <typename U>
-  explicit AllocatorNoCtor(const AllocatorNoCtor<U>& /*other*/)
-      : AllocatorBase(sizeof(T)) {}
-
-  T* allocate(size_t nb_elements) {
-    return static_cast<T*>(AllocatorBase::Allocate(nb_elements * sizeof(T)));
-  }
-  void deallocate(T* p, size_t nb_elements) {
-    ok_ = ok_ && AllocatorBase::Deallocate(p, nb_elements);
-  }
-  bool ok() const { return AllocatorBase::ok(); }
-
-  // The allocator disables any construction...
-  template <typename U, typename... Args>
-  void construct(U*, Args&&...) noexcept {}
-
-  // ...but copy and move constructions, which are called by the vector
-  // implementation itself.
-  void construct(T* p, const T& v) noexcept {
-    static_assert(noexcept(new ((void*)p) T(v)),
-                  "needs a noexcept copy constructor");
-    if (ok_) new ((void*)p) T(v);
-  }
-  void construct(T* p, T&& v) noexcept {
-    static_assert(noexcept(new ((void*)p) T(std::move(v))),
-                  "needs a noexcept move constructor");
-    if (ok_) new ((void*)p) T(std::move(v));
-  }
-
-  template <typename U>
-  void destroy(U* p) noexcept {
-    if (ok_) p->~U();
-  }
-};
-
-// This allocator calls the constructors
-template <typename T>
-class Allocator : public AllocatorNoCtor<T> {
- public:
-  using value_type = T;
-  Allocator() : AllocatorNoCtor<T>() {}
-  template <typename U>
-  explicit Allocator(const Allocator<U>& other) : AllocatorNoCtor<T>(other) {}
-  // Enable the constructor.
-  template <typename U, typename... Args>
-  void construct(U* p, Args&&... args) noexcept {
-    static_assert(noexcept(new ((void*)p) U(std::forward<Args>(args)...)),
-                  "needs a noexcept constructor");
-    if (AllocatorBase::ok_) new ((void*)p) U(std::forward<Args>(args)...);
-  }
-};
-
-template <typename U, typename V>
-bool operator==(const AllocatorNoCtor<U>&, const AllocatorNoCtor<V>&) {
-  return true;
-}
-template <typename U, typename V>
-bool operator!=(const AllocatorNoCtor<U>&, const AllocatorNoCtor<V>&) {
-  return false;
-}
-
-}  // namespace libgav1
-
-#endif  // LIBGAV1_SRC_UTILS_ALLOCATOR_H_
diff --git a/libgav1/src/utils/array_2d.h b/libgav1/src/utils/array_2d.h
index 0b093ff..c054178 100644
--- a/libgav1/src/utils/array_2d.h
+++ b/libgav1/src/utils/array_2d.h
@@ -63,8 +63,11 @@
     const size_t size = rows * columns;
     if (size_ < size) {
       // Note: This invokes the global operator new if T is a non-class type,
-      // such as integer or enum types, or a class that we don't own, such as
-      // std::unique_ptr.
+      // such as integer or enum types, or a class type that is not derived
+      // from libgav1::Allocable, such as std::unique_ptr. If we enforce a
+      // maximum allocation size or keep track of our own heap memory
+      // consumption, we will need to handle the allocations here that use the
+      // global operator new.
       if (zero_initialize) {
         data_.reset(new (std::nothrow) T[size]());
       } else {
diff --git a/libgav1/src/utils/bit_mask_set.h b/libgav1/src/utils/bit_mask_set.h
new file mode 100644
index 0000000..6e10315
--- /dev/null
+++ b/libgav1/src/utils/bit_mask_set.h
@@ -0,0 +1,63 @@
+#ifndef LIBGAV1_SRC_UTILS_BIT_MASK_SET_H_
+#define LIBGAV1_SRC_UTILS_BIT_MASK_SET_H_
+
+#include <cstdint>
+
+namespace libgav1 {
+
+// This class is used to check if a given value is equal to one of the several
+// predetermined values using a bit mask instead of a chain of comparisons and
+// ||s. This usually results in fewer instructions.
+//
+// Usage:
+//   constexpr BitMaskSet set(value1, value2);
+//   set.Contains(value1) => returns true.
+//   set.Contains(value3) => returns false.
+class BitMaskSet {
+ public:
+  explicit constexpr BitMaskSet(uint32_t mask) : mask_(mask) {}
+
+  constexpr BitMaskSet(int v1, int v2) : mask_((1U << v1) | (1U << v2)) {}
+
+  constexpr BitMaskSet(int v1, int v2, int v3)
+      : mask_((1U << v1) | (1U << v2) | (1U << v3)) {}
+
+  constexpr BitMaskSet(int v1, int v2, int v3, int v4)
+      : mask_((1U << v1) | (1U << v2) | (1U << v3) | (1U << v4)) {}
+
+  constexpr BitMaskSet(int v1, int v2, int v3, int v4, int v5)
+      : mask_((1U << v1) | (1U << v2) | (1U << v3) | (1U << v4) | (1U << v5)) {}
+
+  constexpr BitMaskSet(int v1, int v2, int v3, int v4, int v5, int v6)
+      : mask_((1U << v1) | (1U << v2) | (1U << v3) | (1U << v4) | (1U << v5) |
+              (1U << v6)) {}
+
+  constexpr BitMaskSet(int v1, int v2, int v3, int v4, int v5, int v6, int v7)
+      : mask_((1U << v1) | (1U << v2) | (1U << v3) | (1U << v4) | (1U << v5) |
+              (1U << v6) | (1U << v7)) {}
+
+  constexpr BitMaskSet(int v1, int v2, int v3, int v4, int v5, int v6, int v7,
+                       int v8, int v9)
+      : mask_((1U << v1) | (1U << v2) | (1U << v3) | (1U << v4) | (1U << v5) |
+              (1U << v6) | (1U << v7) | (1U << v8) | (1U << v9)) {}
+
+  constexpr BitMaskSet(int v1, int v2, int v3, int v4, int v5, int v6, int v7,
+                       int v8, int v9, int v10)
+      : mask_((1U << v1) | (1U << v2) | (1U << v3) | (1U << v4) | (1U << v5) |
+              (1U << v6) | (1U << v7) | (1U << v8) | (1U << v9) | (1U << v10)) {
+  }
+
+  constexpr bool Contains(uint8_t value) const {
+    return MaskContainsValue(mask_, value);
+  }
+
+  static constexpr bool MaskContainsValue(uint32_t mask, uint8_t value) {
+    return ((mask >> value) & 1) != 0;
+  }
+
+ private:
+  const uint32_t mask_;
+};
+
+}  // namespace libgav1
+#endif  // LIBGAV1_SRC_UTILS_BIT_MASK_SET_H_
diff --git a/libgav1/src/utils/bit_reader.cc b/libgav1/src/utils/bit_reader.cc
index 1f58fed..8975965 100644
--- a/libgav1/src/utils/bit_reader.cc
+++ b/libgav1/src/utils/bit_reader.cc
@@ -8,13 +8,13 @@
 namespace libgav1 {
 namespace {
 
-inline bool Assign(int* const value, int assignment, bool return_value) {
+bool Assign(int* const value, int assignment, bool return_value) {
   *value = assignment;
   return return_value;
 }
 
 // 5.9.29.
-inline int InverseRecenter(int r, int v) {
+int InverseRecenter(int r, int v) {
   if (v > (r << 1)) {
     return v;
   }
diff --git a/libgav1/src/utils/block_parameters_holder.cc b/libgav1/src/utils/block_parameters_holder.cc
index a7ee0c8..a207aa9 100644
--- a/libgav1/src/utils/block_parameters_holder.cc
+++ b/libgav1/src/utils/block_parameters_holder.cc
@@ -23,26 +23,37 @@
 
 BlockParametersHolder::BlockParametersHolder(int rows4x4, int columns4x4,
                                              bool use_128x128_superblock)
-    : rows4x4_(rows4x4), columns4x4_(columns4x4) {
-  if (!block_parameters_cache_.Reset(rows4x4, columns4x4)) {
+    : rows4x4_(rows4x4),
+      columns4x4_(columns4x4),
+      use_128x128_superblock_(use_128x128_superblock) {}
+
+bool BlockParametersHolder::Init() {
+  if (!block_parameters_cache_.Reset(rows4x4_, columns4x4_)) {
     LIBGAV1_DLOG(ERROR, "block_parameters_cache_.Reset() failed.");
+    return false;
   }
   const int rows =
-      RowsOrColumns4x4ToSuperBlocks(rows4x4, use_128x128_superblock);
+      RowsOrColumns4x4ToSuperBlocks(rows4x4_, use_128x128_superblock_);
   const int columns =
-      RowsOrColumns4x4ToSuperBlocks(columns4x4, use_128x128_superblock);
+      RowsOrColumns4x4ToSuperBlocks(columns4x4_, use_128x128_superblock_);
   const BlockSize sb_size =
-      use_128x128_superblock ? kBlock128x128 : kBlock64x64;
+      use_128x128_superblock_ ? kBlock128x128 : kBlock64x64;
   const int multiplier = kNum4x4BlocksWide[sb_size];
   if (!trees_.Reset(rows, columns)) {
     LIBGAV1_DLOG(ERROR, "trees_.Reset() failed.");
+    return false;
   }
   for (int i = 0; i < rows; ++i) {
     for (int j = 0; j < columns; ++j) {
-      trees_[i][j].reset(new (std::nothrow) ParameterTree(
-          i * multiplier, j * multiplier, sb_size));
+      trees_[i][j] =
+          ParameterTree::Create(i * multiplier, j * multiplier, sb_size);
+      if (trees_[i][j] == nullptr) {
+        LIBGAV1_DLOG(ERROR, "Allocation of trees_[%d][%d] failed.", i, j);
+        return false;
+      }
     }
   }
+  return true;
 }
 
 void BlockParametersHolder::FillCache(int row4x4, int column4x4,
diff --git a/libgav1/src/utils/block_parameters_holder.h b/libgav1/src/utils/block_parameters_holder.h
index a55d0d6..d92791b 100644
--- a/libgav1/src/utils/block_parameters_holder.h
+++ b/libgav1/src/utils/block_parameters_holder.h
@@ -4,6 +4,7 @@
 #include <memory>
 
 #include "src/utils/array_2d.h"
+#include "src/utils/compiler_attributes.h"
 #include "src/utils/constants.h"
 #include "src/utils/parameter_tree.h"
 #include "src/utils/types.h"
@@ -23,10 +24,13 @@
   BlockParametersHolder(const BlockParametersHolder&) = delete;
   BlockParametersHolder& operator=(const BlockParametersHolder&) = delete;
 
+  // Must be called first.
+  LIBGAV1_MUST_USE_RESULT bool Init();
+
   // Finds the BlockParameters corresponding to |row4x4| and |column4x4|. This
   // is done as a simple look up of the |block_parameters_cache_| matrix.
   // Returns nullptr if the BlockParameters cannot be found.
-  BlockParameters* Find(int row4x4, int column4x4) {
+  BlockParameters* Find(int row4x4, int column4x4) const {
     return block_parameters_cache_[row4x4][column4x4];
   }
 
@@ -40,8 +44,9 @@
                  BlockParameters* bp);
 
  private:
-  int rows4x4_;
-  int columns4x4_;
+  const int rows4x4_;
+  const int columns4x4_;
+  const bool use_128x128_superblock_;
   Array2D<std::unique_ptr<ParameterTree>> trees_;
 
   // This is a 2d array of size |rows4x4_| * |columns4x4_|.This is filled in by
diff --git a/libgav1/src/utils/blocking_counter.h b/libgav1/src/utils/blocking_counter.h
new file mode 100644
index 0000000..7fe3c97
--- /dev/null
+++ b/libgav1/src/utils/blocking_counter.h
@@ -0,0 +1,81 @@
+#ifndef LIBGAV1_SRC_UTILS_BLOCKING_COUNTER_H_
+#define LIBGAV1_SRC_UTILS_BLOCKING_COUNTER_H_
+
+#include <cassert>
+#include <condition_variable>  // NOLINT (unapproved c++11 header)
+#include <mutex>               // NOLINT (unapproved c++11 header)
+
+#include "src/utils/compiler_attributes.h"
+
+namespace libgav1 {
+
+// Implementation of a Blocking Counter that is used for the "fork-join"
+// use case. Typical usage would be as follows:
+//   BlockingCounter counter(num_jobs);
+//     - spawn the jobs.
+//     - call counter.Wait() on the master thread.
+//     - worker threads will call counter.Decrement().
+//     - master thread will return from counter.Wait() when all workers are
+//     complete.
+template <bool has_failure_status>
+class BlockingCounterImpl {
+ public:
+  explicit BlockingCounterImpl(int initial_count)
+      : count_(initial_count), job_failed_(false) {}
+
+  // Increment the counter by |count|. This must be called before Wait() is
+  // called. This must be called from the same thread that will call Wait().
+  void IncrementBy(int count) {
+    assert(count >= 0);
+    std::unique_lock<std::mutex> lock(mutex_);
+    count_ += count;
+  }
+
+  // Decrement the counter by 1. This function can be called only when
+  // |has_failure_status| is false (i.e.) when this class is being used with the
+  // |BlockingCounter| alias.
+  void Decrement() {
+    static_assert(!has_failure_status, "");
+    std::unique_lock<std::mutex> lock(mutex_);
+    if (--count_ == 0) {
+      condition_.notify_one();
+    }
+  }
+
+  // Decrement the counter by 1. This function can be called only when
+  // |has_failure_status| is true (i.e.) when this class is being used with the
+  // |BlockingCounterWithStatus| alias. |job_succeeded| is used to update the
+  // state of |job_failed_|.
+  void Decrement(bool job_succeeded) {
+    static_assert(has_failure_status, "");
+    std::unique_lock<std::mutex> lock(mutex_);
+    job_failed_ |= !job_succeeded;
+    if (--count_ == 0) {
+      condition_.notify_one();
+    }
+  }
+
+  // Block until the counter becomes 0. This function can be called only once
+  // per object. If |has_failure_status| is true, true is returned if all the
+  // jobs succeeded and false is returned if any of the jobs failed. If
+  // |has_failure_status| is false, this function always returns true.
+  bool Wait() {
+    std::unique_lock<std::mutex> lock(mutex_);
+    condition_.wait(lock, [this]() { return count_ == 0; });
+    // If |has_failure_status| is false, we simply return true.
+    return has_failure_status ? !job_failed_ : true;
+  }
+
+ private:
+  std::mutex mutex_;
+  std::condition_variable condition_;
+  int count_ LIBGAV1_GUARDED_BY(mutex_);
+  bool job_failed_ LIBGAV1_GUARDED_BY(mutex_);
+};
+
+using BlockingCounterWithStatus = BlockingCounterImpl<true>;
+using BlockingCounter = BlockingCounterImpl<false>;
+
+}  // namespace libgav1
+
+#endif  // LIBGAV1_SRC_UTILS_BLOCKING_COUNTER_H_
diff --git a/libgav1/src/utils/common.h b/libgav1/src/utils/common.h
index bfbdb8c..9803364 100644
--- a/libgav1/src/utils/common.h
+++ b/libgav1/src/utils/common.h
@@ -14,6 +14,7 @@
 #include <cstddef>
 #include <cstdint>
 
+#include "src/utils/bit_mask_set.h"
 #include "src/utils/constants.h"
 
 namespace libgav1 {
@@ -114,7 +115,11 @@
 }
 
 inline int CeilLog2(unsigned int n) {
-  return (n < 2) ? 0 : FloorLog2(n) + static_cast<int>((n & (n - 1)) != 0);
+  // The expression FloorLog2(n - 1) + 1 is undefined not only for n == 0 but
+  // also for n == 1, so this expression must be guarded by the n < 2 test. An
+  // alternative implementation is:
+  // return (n == 0) ? 0 : FloorLog2(n) + static_cast<int>((n & (n - 1)) != 0);
+  return (n < 2) ? 0 : FloorLog2(n - 1) + 1;
 }
 
 inline int32_t RightShiftWithRounding(int32_t value, int bits) {
@@ -146,13 +151,13 @@
                       : -RightShiftWithRounding(-value, bits);
 }
 
-inline int DivideBy2(int n) { return n >> 1; }
-inline int DivideBy4(int n) { return n >> 2; }
-inline int DivideBy8(int n) { return n >> 3; }
-inline int DivideBy16(int n) { return n >> 4; }
-inline int DivideBy32(int n) { return n >> 5; }
-inline int DivideBy64(int n) { return n >> 6; }
-inline int DivideBy128(int n) { return n >> 7; }
+constexpr int DivideBy2(int n) { return n >> 1; }
+constexpr int DivideBy4(int n) { return n >> 2; }
+constexpr int DivideBy8(int n) { return n >> 3; }
+constexpr int DivideBy16(int n) { return n >> 4; }
+constexpr int DivideBy32(int n) { return n >> 5; }
+constexpr int DivideBy64(int n) { return n >> 6; }
+constexpr int DivideBy128(int n) { return n >> 7; }
 
 // Convert |value| to unsigned before shifting to avoid undefined behavior with
 // negative values.
@@ -169,25 +174,27 @@
 inline int MultiplyBy32(int n) { return LeftShift(n, 5); }
 inline int MultiplyBy64(int n) { return LeftShift(n, 6); }
 
-inline int Mod32(int n) { return n & 0x1f; }
-inline int Mod64(int n) { return n & 0x3f; }
+constexpr int Mod32(int n) { return n & 0x1f; }
+constexpr int Mod64(int n) { return n & 0x3f; }
 
 //------------------------------------------------------------------------------
 // Bitstream functions
 
-inline bool IsIntraFrame(FrameType type) {
+constexpr bool IsIntraFrame(FrameType type) {
   return type == kFrameKey || type == kFrameIntraOnly;
 }
 
 inline TransformClass GetTransformClass(TransformType tx_type) {
-  if (tx_type == kTransformTypeIdentityDct ||
-      tx_type == kTransformTypeIdentityAdst ||
-      tx_type == kTransformTypeIdentityFlipadst) {
+  constexpr BitMaskSet kTransformClassVerticalMask(
+      kTransformTypeIdentityDct, kTransformTypeIdentityAdst,
+      kTransformTypeIdentityFlipadst);
+  if (kTransformClassVerticalMask.Contains(tx_type)) {
     return kTransformClassVertical;
   }
-  if (tx_type == kTransformTypeDctIdentity ||
-      tx_type == kTransformTypeAdstIdentity ||
-      tx_type == kTransformTypeFlipadstIdentity) {
+  constexpr BitMaskSet kTransformClassHorizontalMask(
+      kTransformTypeDctIdentity, kTransformTypeAdstIdentity,
+      kTransformTypeFlipadstIdentity);
+  if (kTransformClassHorizontalMask.Contains(tx_type)) {
     return kTransformClassHorizontal;
   }
   return kTransformClass2D;
@@ -198,12 +205,12 @@
   return MultiplyBy4(row_or_column4x4) >> (plane == kPlaneY ? 0 : subsampling);
 }
 
-inline PlaneType GetPlaneType(Plane plane) {
-  return (plane == kPlaneY) ? kPlaneTypeY : kPlaneTypeUV;
+constexpr PlaneType GetPlaneType(Plane plane) {
+  return static_cast<PlaneType>(plane != kPlaneY);
 }
 
 // 5.11.44.
-inline bool IsDirectionalMode(PredictionMode mode) {
+constexpr bool IsDirectionalMode(PredictionMode mode) {
   return mode >= kPredictionModeVertical && mode <= kPredictionModeD67;
 }
 
@@ -254,14 +261,8 @@
   return (diff & (m - 1)) - (diff & m);
 }
 
-// part of 5.11.23.
-inline bool HasNearMv(PredictionMode mode) {
-  return mode == kPredictionModeNearMv || mode == kPredictionModeNearNearMv ||
-         mode == kPredictionModeNearNewMv || mode == kPredictionModeNewNearMv;
-}
-
 inline bool IsBlockSmallerThan8x8(BlockSize size) {
-  return size == kBlock4x4 || size == kBlock4x8 || size == kBlock8x4;
+  return size < kBlock8x8 && size != kBlock4x16;
 }
 
 // Maps a square transform to an index between [0, 4]. kTransformSize4x4 maps
@@ -279,33 +280,12 @@
   return DivideBy4(tx_size);
 }
 
-// 5.11.37.
-inline TransformSize GetTransformSize(const bool lossless,
-                                      const BlockSize block_size,
-                                      const Plane plane,
-                                      const TransformSize tx_size,
-                                      const int subsampling_x,
-                                      const int subsampling_y) {
-  if (lossless) return kTransformSize4x4;
-  if (plane == kPlaneY) return tx_size;
-  const BlockSize plane_size =
-      kPlaneResidualSize[block_size][subsampling_x][subsampling_y];
-  assert(plane_size != kBlockInvalid);
-  const TransformSize uv_tx_size = kMaxTransformSizeRectangle[plane_size];
-  const uint32_t mask = 1U << uv_tx_size;
-  if ((mask & kTransformSize64Mask) == 0) {
-    return uv_tx_size;
-  }
-  if ((mask & kTransformWidth16Mask) != 0) return kTransformSize16x32;
-  if ((mask & kTransformHeight16Mask) != 0) return kTransformSize32x16;
-  return kTransformSize32x32;
-}
-
 // Gets the corresponding Y/U/V position, to set and get filter masks
 // in deblock filtering.
 // Returns luma_position if it's Y plane, whose subsampling must be 0.
 // Returns the odd position for U/V plane, if there is subsampling.
-inline int GetDeblockPosition(const int luma_position, const int subsampling) {
+constexpr int GetDeblockPosition(const int luma_position,
+                                 const int subsampling) {
   return luma_position | subsampling;
 }
 
@@ -328,6 +308,19 @@
   return (residual_size * rows * columns * subsampling_multiplier_num) >> 1;
 }
 
+// This function is equivalent to:
+// std::min({kTransformWidthLog2[tx_size] - 2,
+//           kTransformWidthLog2[left_tx_size] - 2,
+//           2});
+constexpr LoopFilterTransformSizeId GetTransformSizeIdWidth(
+    TransformSize tx_size, TransformSize left_tx_size) {
+  return static_cast<LoopFilterTransformSizeId>(
+      static_cast<int>(tx_size > kTransformSize4x16 &&
+                       left_tx_size > kTransformSize4x16) +
+      static_cast<int>(tx_size > kTransformSize8x32 &&
+                       left_tx_size > kTransformSize8x32));
+}
+
 }  // namespace libgav1
 
 #endif  // LIBGAV1_SRC_UTILS_COMMON_H_
diff --git a/libgav1/src/utils/constants.cc b/libgav1/src/utils/constants.cc
index 823f591..cc8a33f 100644
--- a/libgav1/src/utils/constants.cc
+++ b/libgav1/src/utils/constants.cc
@@ -117,25 +117,18 @@
 const uint8_t kTransformHeight[kNumTransformSizes] = {
     4, 8, 16, 4, 8, 16, 32, 4, 8, 16, 32, 64, 8, 16, 32, 64, 16, 32, 64};
 
+const uint8_t kTransformWidth4x4[kNumTransformSizes] = {
+    1, 1, 1, 2, 2, 2, 2, 4, 4, 4, 4, 4, 8, 8, 8, 8, 16, 16, 16};
+
+const uint8_t kTransformHeight4x4[kNumTransformSizes] = {
+    1, 2, 4, 1, 2, 4, 8, 1, 2, 4, 8, 16, 2, 4, 8, 16, 4, 8, 16};
+
 const uint8_t kTransformWidthLog2[kNumTransformSizes] = {
     2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6};
 
 const uint8_t kTransformHeightLog2[kNumTransformSizes] = {
     2, 3, 4, 2, 3, 4, 5, 2, 3, 4, 5, 6, 3, 4, 5, 6, 4, 5, 6};
 
-const TransformSize kMaxTransformSizeRectangle[kMaxBlockSizes] = {
-    kTransformSize4x4,   kTransformSize4x8,   kTransformSize4x16,
-    kTransformSize8x4,   kTransformSize8x8,   kTransformSize8x16,
-    kTransformSize8x32,  kTransformSize16x4,  kTransformSize16x8,
-    kTransformSize16x16, kTransformSize16x32, kTransformSize16x64,
-    kTransformSize32x8,  kTransformSize32x16, kTransformSize32x32,
-    kTransformSize32x64, kTransformSize64x16, kTransformSize64x32,
-    kTransformSize64x64, kTransformSize64x64, kTransformSize64x64,
-    kTransformSize64x64};
-
-const int kMaxTransformDepth[kMaxBlockSizes] = {
-    0, 1, 2, 1, 1, 2, 3, 2, 2, 2, 3, 4, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4};
-
 // 9.3 -- Split_Tx_Size[]
 const TransformSize kSplitTransformSize[kNumTransformSizes] = {
     kTransformSize4x4,   kTransformSize4x4,   kTransformSize4x8,
@@ -166,51 +159,8 @@
     kTransformSize64x64, kTransformSize64x64, kTransformSize64x64,
     kTransformSize64x64};
 
-// Defined in section 9.3 of the spec.
-const TransformType kModeToTransformType[kIntraPredictionModesUV] = {
-    kTransformTypeDctDct,   kTransformTypeDctAdst,  kTransformTypeAdstDct,
-    kTransformTypeDctDct,   kTransformTypeAdstAdst, kTransformTypeDctAdst,
-    kTransformTypeAdstDct,  kTransformTypeAdstDct,  kTransformTypeDctAdst,
-    kTransformTypeAdstAdst, kTransformTypeDctAdst,  kTransformTypeAdstDct,
-    kTransformTypeAdstAdst, kTransformTypeDctDct};
-
 const uint8_t kNumTransformTypesInSet[kNumTransformSets] = {1, 7, 5, 16, 12, 2};
 
-// Defined in section 5.11.47 of the spec. This array does not contain an entry
-// for kTransformSetDctOnly, so the first dimension needs to be
-// |kNumTransformSets| - 1.
-const TransformType kInverseTransformTypeBySet[kNumTransformSets - 1][16] = {
-    {kTransformTypeIdentityIdentity, kTransformTypeDctDct,
-     kTransformTypeIdentityDct, kTransformTypeDctIdentity,
-     kTransformTypeAdstAdst, kTransformTypeDctAdst, kTransformTypeAdstDct},
-    {kTransformTypeIdentityIdentity, kTransformTypeDctDct,
-     kTransformTypeAdstAdst, kTransformTypeDctAdst, kTransformTypeAdstDct},
-    {kTransformTypeIdentityIdentity, kTransformTypeIdentityDct,
-     kTransformTypeDctIdentity, kTransformTypeIdentityAdst,
-     kTransformTypeAdstIdentity, kTransformTypeIdentityFlipadst,
-     kTransformTypeFlipadstIdentity, kTransformTypeDctDct,
-     kTransformTypeDctAdst, kTransformTypeAdstDct, kTransformTypeDctFlipadst,
-     kTransformTypeFlipadstDct, kTransformTypeAdstAdst,
-     kTransformTypeFlipadstFlipadst, kTransformTypeFlipadstAdst,
-     kTransformTypeAdstFlipadst},
-    {kTransformTypeIdentityIdentity, kTransformTypeIdentityDct,
-     kTransformTypeDctIdentity, kTransformTypeDctDct, kTransformTypeDctAdst,
-     kTransformTypeAdstDct, kTransformTypeDctFlipadst,
-     kTransformTypeFlipadstDct, kTransformTypeAdstAdst,
-     kTransformTypeFlipadstFlipadst, kTransformTypeFlipadstAdst,
-     kTransformTypeAdstFlipadst},
-    {kTransformTypeIdentityIdentity, kTransformTypeDctDct}};
-
-// Replaces all occurrences of 64x* and *x64 with 32x* and *x32 respectively.
-const TransformSize kAdjustedTransformSize[kNumTransformSizes] = {
-    kTransformSize4x4,   kTransformSize4x8,   kTransformSize4x16,
-    kTransformSize8x4,   kTransformSize8x8,   kTransformSize8x16,
-    kTransformSize8x32,  kTransformSize16x4,  kTransformSize16x8,
-    kTransformSize16x16, kTransformSize16x32, kTransformSize16x32,
-    kTransformSize32x8,  kTransformSize32x16, kTransformSize32x32,
-    kTransformSize32x32, kTransformSize32x16, kTransformSize32x32,
-    kTransformSize32x32};
-
 const uint8_t kSgrProjParams[1 << kSgrProjParamsBits][4] = {
     {2, 12, 1, 4},  {2, 15, 1, 6},  {2, 18, 1, 8},  {2, 21, 1, 9},
     {2, 24, 1, 10}, {2, 29, 1, 11}, {2, 36, 1, 12}, {2, 45, 1, 13},
@@ -221,10 +171,6 @@
 
 const int8_t kSgrProjMultiplierMax[2] = {31, 95};
 
-const int8_t kSgrProjDefaultMultiplier[2] = {-32, 31};
-
-const int8_t kWienerDefaultFilter[3] = {3, -7, 15};
-
 const int8_t kWienerTapsMin[3] = {-5, -23, -17};
 
 const int8_t kWienerTapsMax[3] = {10, 8, 46};
@@ -595,11 +541,314 @@
     3,           // 87, ...
 };
 
-const uint8_t kPredictionModeDeltasLookup[kNumPredictionModes] = {
-    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  // Intra modes.
-    0,                      // ChromaFromLuma mode, not used in y mode.
-    1, 1, 0, 1,             // Inter modes.
-    1, 1, 1, 1, 1, 1, 0, 1  // Compound inter modes.
+const uint8_t kDeblockFilterLevelIndex[kMaxPlanes][kNumLoopFilterTypes] = {
+    {0, 1}, {2, 2}, {3, 3}};
+
+const int8_t kMaskIdLookup[4][kMaxBlockSizes] = {
+    // transform size 4x4.
+    {0,  1,  13, 2, 3,  4,  15, 14, 5,  6,  7,
+     17, 16, 8,  9, 10, 18, 11, 12, -1, -1, -1},
+    // transform size 8x8.
+    {-1, -1, -1, -1, 19, 20, 29, -1, 21, 22, 23,
+     31, 30, 24, 25, 26, 32, 27, 28, -1, -1, -1},
+    // transform size 16x16.
+    {-1, -1, -1, -1, -1, -1, -1, -1, -1, 33, 34,
+     40, -1, 35, 36, 37, 41, 38, 39, -1, -1, -1},
+    // transform size 32x32.
+    {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+     -1, -1, -1, 42, 43, -1, 44, 45, -1, -1, -1},
+};
+
+const int8_t kVerticalBorderMaskIdLookup[kMaxBlockSizes] = {
+    0,  47, 61, 49, 19, 51, 63, 62, 53, 33, 55,
+    65, 64, 57, 42, 59, 66, 60, 46, -1, -1, -1};
+
+const uint64_t kTopMaskLookup[67][4] = {
+    // transform size 4X4
+    {0x0000000000000001ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 4X4, transform size 4X4
+    {0x0000000000010001ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 4X8, transform size 4X4
+    {0x0000000000000003ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 8X4, transform size 4X4
+    {0x0000000000030003ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 8X8, transform size 4X4
+    {0x0003000300030003ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 8X16, transform size 4X4
+    {0x00000000000f000fULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 16X8, transform size 4X4
+    {0x000f000f000f000fULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 16X16, transform size 4X4
+    {0x000f000f000f000fULL, 0x000f000f000f000fULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 16X32, transform size 4X4
+    {0x00ff00ff00ff00ffULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 32X16, transform size 4X4
+    {0x00ff00ff00ff00ffULL, 0x00ff00ff00ff00ffULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 32X32, transform size 4X4
+    {0x00ff00ff00ff00ffULL, 0x00ff00ff00ff00ffULL, 0x00ff00ff00ff00ffULL,
+     0x00ff00ff00ff00ffULL},  // block size 32X64, transform size 4X4
+    {0xffffffffffffffffULL, 0xffffffffffffffffULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 64X32, transform size 4X4
+    {0xffffffffffffffffULL, 0xffffffffffffffffULL, 0xffffffffffffffffULL,
+     0xffffffffffffffffULL},  // block size 64X64, transform size 4x4
+    {0x0001000100010001ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 4X16, transform size 4X4
+    {0x000000000000000fULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 16X4, transform size 4X4
+    {0x0003000300030003ULL, 0x0003000300030003ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 8X32, transform size 4X4
+    {0x0000000000ff00ffULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 32X8, transform size 4X4
+    {0x000f000f000f000fULL, 0x000f000f000f000fULL, 0x000f000f000f000fULL,
+     0x000f000f000f000fULL},  // block size 16X64, transform size 4X4
+    {0xffffffffffffffffULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 64X16, transform size 4X4
+    // transform size 8X8
+    {0x0000000000000003ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 8X8, transform size 8X8
+    {0x0000000300000003ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 8X16, transform size 8X8
+    {0x000000000000000fULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 16X8, transform size 8X8
+    {0x0000000f0000000fULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 16X16, transform size 8X8
+    {0x0000000f0000000fULL, 0x0000000f0000000fULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 16X32, transform size 8X8
+    {0x000000ff000000ffULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 32X16, transform size 8X8
+    {0x000000ff000000ffULL, 0x000000ff000000ffULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 32X32, transform size 8X8
+    {0x000000ff000000ffULL, 0x000000ff000000ffULL, 0x000000ff000000ffULL,
+     0x000000ff000000ffULL},  // block size 32X64, transform size 8X8
+    {0x0000ffff0000ffffULL, 0x0000ffff0000ffffULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 64X32, transform size 8X8
+    {0x0000ffff0000ffffULL, 0x0000ffff0000ffffULL, 0x0000ffff0000ffffULL,
+     0x0000ffff0000ffffULL},  // block size 64X64, transform size 8X8
+    {0x0000000300000003ULL, 0x0000000300000003ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 8X32, transform size 8X8
+    {0x00000000000000ffULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 32X8, transform size 8X8
+    {0x0000000f0000000fULL, 0x0000000f0000000fULL, 0x0000000f0000000fULL,
+     0x0000000f0000000fULL},  // block size 16X64, transform size 8X8
+    {0x0000ffff0000ffffULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 64X16, transform size 8X8
+    // transform size 16X16
+    {0x000000000000000fULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 16X16, transform size 16X16
+    {0x000000000000000fULL, 0x000000000000000fULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 16X32, transform size 16X16
+    {0x00000000000000ffULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 32X16, transform size 16X16
+    {0x00000000000000ffULL, 0x00000000000000ffULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 32X32, transform size 16X16
+    {0x00000000000000ffULL, 0x00000000000000ffULL, 0x00000000000000ffULL,
+     0x00000000000000ffULL},  // block size 32X64, transform size 16X16
+    {0x000000000000ffffULL, 0x000000000000ffffULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 64X32, transform size 16X16
+    {0x000000000000ffffULL, 0x000000000000ffffULL, 0x000000000000ffffULL,
+     0x000000000000ffffULL},  // block size 64X64, transform size 16X16
+    {0x000000000000000fULL, 0x000000000000000fULL, 0x000000000000000fULL,
+     0x000000000000000fULL},  // block size 16X64, transform size 16X16
+    {0x000000000000ffffULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 64X16, transform size 16X16
+    // transform size 32X32
+    {0x00000000000000ffULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 32X32, transform size 32X32
+    {0x00000000000000ffULL, 0x0000000000000000ULL, 0x00000000000000ffULL,
+     0x0000000000000000ULL},  // block size 32X64, transform size 32X32
+    {0x000000000000ffffULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 64X32, transform size 32X32
+    {0x000000000000ffffULL, 0x0000000000000000ULL, 0x000000000000ffffULL,
+     0x0000000000000000ULL},  // block size 64X64, transform size 32X32
+    // transform size 64X64
+    {0x000000000000ffffULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 64X64, transform size 64X64
+    // 2:1, 1:2 transform sizes.
+    {0x0000000000000001ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 4X8, transform size 4X8
+    {0x0000000100000001ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 4X16, transform size 4X8
+    {0x0000000000000003ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 8X4, transform size 8X4
+    {0x000000000000000fULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 16X4, transform size 8X4
+    {0x0000000000000003ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 8X16, transform size 8X16
+    {0x0000000000000003ULL, 0x0000000000000003ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 8X32, transform size 8X16
+    {0x000000000000000fULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 16X8, transform size 16X8
+    {0x00000000000000ffULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 32X8, transform size 16X8
+    {0x000000000000000fULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 16X32, transform size 16X32
+    {0x000000000000000fULL, 0x0000000000000000ULL, 0x000000000000000fULL,
+     0x0000000000000000ULL},  // block size 16X64, transform size 16X32
+    {0x00000000000000ffULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 32X16, transform size 32X16
+    {0x000000000000ffffULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 64X16, transform size 32X16
+    {0x00000000000000ffULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 32X64, transform size 32X64
+    {0x000000000000ffffULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 64X32, transform size 64X32
+    // 4:1, 1:4 transform sizes.
+    {0x0000000000000001ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 4X16, transform size 4X16
+    {0x000000000000000fULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 16X4, transform size 16X4
+    {0x0000000000000003ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 8X32, transform size 8X32
+    {0x00000000000000ffULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 32X8, transform size 32X8
+    {0x000000000000000fULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 16X64, transform size 16X64
+    {0x000000000000ffffULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 64X16, transform size 64X16
+};
+
+const uint64_t kLeftMaskLookup[67][4] = {
+    // transform size 4X4
+    {0x0000000000000001ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 4X4, transform size 4X4
+    {0x0000000000010001ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 4X8, transform size 4X4
+    {0x0000000000000003ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 8X4, transform size 4X4
+    {0x0000000000030003ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 8X8, transform size 4X4
+    {0x0003000300030003ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 8X16, transform size 4X4
+    {0x00000000000f000fULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 16X8, transform size 4X4
+    {0x000f000f000f000fULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 16X16, transform size 4X4
+    {0x000f000f000f000fULL, 0x000f000f000f000fULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 16X32, transform size 4X4
+    {0x00ff00ff00ff00ffULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 32X16, transform size 4X4
+    {0x00ff00ff00ff00ffULL, 0x00ff00ff00ff00ffULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 32X32, transform size 4X4
+    {0x00ff00ff00ff00ffULL, 0x00ff00ff00ff00ffULL, 0x00ff00ff00ff00ffULL,
+     0x00ff00ff00ff00ffULL},  // block size 32X64, transform size 4X4
+    {0xffffffffffffffffULL, 0xffffffffffffffffULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 64X32, transform size 4X4
+    {0xffffffffffffffffULL, 0xffffffffffffffffULL, 0xffffffffffffffffULL,
+     0xffffffffffffffffULL},  // block size 64X64, transform size 4X4
+    {0x0001000100010001ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 4X16, transform size 4X4
+    {0x000000000000000fULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 16X4, transform size 4X4
+    {0x0003000300030003ULL, 0x0003000300030003ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 8X32, transform size 4X4
+    {0x0000000000ff00ffULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 32X8, transform size 4X4
+    {0x000f000f000f000fULL, 0x000f000f000f000fULL, 0x000f000f000f000fULL,
+     0x000f000f000f000fULL},  // block size 16X64, transform size 4X4
+    {0xffffffffffffffffULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 64X16, transform size 4X4
+    // transform size 8X8
+    {0x0000000000010001ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 8X8, transform size 8X8
+    {0x0001000100010001ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 8X16, transform size 8X8
+    {0x0000000000050005ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 16X8, transform size 8X8
+    {0x0005000500050005ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 16X16, transform size 8X8
+    {0x0005000500050005ULL, 0x0005000500050005ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 16X32, transform size 8X8
+    {0x0055005500550055ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 32X16, transform size 8X8
+    {0x0055005500550055ULL, 0x0055005500550055ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 32X32, transform size 8X8
+    {0x0055005500550055ULL, 0x0055005500550055ULL, 0x0055005500550055ULL,
+     0x0055005500550055ULL},  // block size 32X64, transform size 8X8
+    {0x5555555555555555ULL, 0x5555555555555555ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 64X32, transform size 8X8
+    {0x5555555555555555ULL, 0x5555555555555555ULL, 0x5555555555555555ULL,
+     0x5555555555555555ULL},  // block size 64X64, transform size 8X8
+    {0x0001000100010001ULL, 0x0001000100010001ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 8X32, transform size 8X8
+    {0x0000000000550055ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 32X8, transform size 8X8
+    {0x0005000500050005ULL, 0x0005000500050005ULL, 0x0005000500050005ULL,
+     0x0005000500050005ULL},  // block size 16X64, transform size 8X8
+    {0x5555555555555555ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 64X16, transform size 8X8
+    // transform size 16X16
+    {0x0001000100010001ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 16X16, transform size 16X16
+    {0x0001000100010001ULL, 0x0001000100010001ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 16X32, transform size 16X16
+    {0x0011001100110011ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 32X16, transform size 16X16
+    {0x0011001100110011ULL, 0x0011001100110011ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 32X32, transform size 16X16
+    {0x0011001100110011ULL, 0x0011001100110011ULL, 0x0011001100110011ULL,
+     0x0011001100110011ULL},  // block size 32X64, transform size 16X16
+    {0x1111111111111111ULL, 0x1111111111111111ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 64X32, transform size 16X16
+    {0x1111111111111111ULL, 0x1111111111111111ULL, 0x1111111111111111ULL,
+     0x1111111111111111ULL},  // block size 64X64, transform size 16X16
+    {0x0001000100010001ULL, 0x0001000100010001ULL, 0x0001000100010001ULL,
+     0x0001000100010001ULL},  // block size 16X64, transform size 16X16
+    {0x1111111111111111ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 64X16, transform size 16X16
+    // transform size 32X32
+    {0x0001000100010001ULL, 0x0001000100010001ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 32X32, transform size 32X32
+    {0x0101010101010101ULL, 0x0101010101010101ULL, 0x0101010101010101ULL,
+     0x0101010101010101ULL},  // block size 32X64, transform size 32X32
+    {0x0101010101010101ULL, 0x0101010101010101ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 64X32, transform size 32X32
+    {0x0101010101010101ULL, 0x0101010101010101ULL, 0x0101010101010101ULL,
+     0x0101010101010101ULL},  // block size 64X64, transform size 32X32
+    // transform size 64X64
+    {0x0001000100010001ULL, 0x0001000100010001ULL, 0x0001000100010001ULL,
+     0x0001000100010001ULL},  // block size 64X64, transform size 64X64
+    // 2:1, 1:2 transform sizes.
+    {0x0000000000010001ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 4X8, transform size 4X8
+    {0x0001000100010001ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 4X16, transform size 4X8
+    {0x0000000000000001ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 8X4, transform size 8X4
+    {0x0000000000000005ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 16X4, transform size 8X4
+    {0x0001000100010001ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 8X16, transform size 8X16
+    {0x0001000100010001ULL, 0x0001000100010001ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 8X32, transform size 8X16
+    {0x0000000000010001ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 16X8, transform size 16X8
+    {0x0000000000110011ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 32X8, transform size 16X8
+    {0x0001000100010001ULL, 0x0001000100010001ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 16X32, transform size 16X32
+    {0x0001000100010001ULL, 0x0001000100010001ULL, 0x0001000100010001ULL,
+     0x0001000100010001ULL},  // block size 16X64, transform size 16X32
+    {0x0001000100010001ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 32X16, transform size 32X16
+    {0x0101010101010101ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 64X16, transform size 32X16
+    {0x0001000100010001ULL, 0x0001000100010001ULL, 0x0001000100010001ULL,
+     0x0001000100010001ULL},  // block size 32X64, transform size 32X64
+    {0x0001000100010001ULL, 0x0001000100010001ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 64X32, transform size 64X32
+    // 4:1, 1:4 transform sizes.
+    {0x0001000100010001ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 4X16, transform size 4X16
+    {0x0000000000000001ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 16X4, transform size 16X4
+    {0x0001000100010001ULL, 0x0001000100010001ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 8X32, transform size 8X32
+    {0x0000000000010001ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 32X8, transform size 32X8
+    {0x0001000100010001ULL, 0x0001000100010001ULL, 0x0001000100010001ULL,
+     0x0001000100010001ULL},  // block size 16X64, transform size 16X64
+    {0x0001000100010001ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
+     0x0000000000000000ULL},  // block size 64X16, transform size 64X16
 };
 
 }  // namespace libgav1
diff --git a/libgav1/src/utils/constants.h b/libgav1/src/utils/constants.h
index 1f7c23d..5799a91 100644
--- a/libgav1/src/utils/constants.h
+++ b/libgav1/src/utils/constants.h
@@ -2,6 +2,9 @@
 #define LIBGAV1_SRC_UTILS_CONSTANTS_H_
 
 #include <cstdint>
+#include <cstdlib>
+
+#include "src/utils/bit_mask_set.h"
 
 namespace libgav1 {
 
@@ -18,7 +21,6 @@
   kFrameLfCount = 4,
   kMaxLoopFilterValue = 63,
   kNum4x4In64x64 = 256,
-  kNumTransformSizesLoopFilter = 3,  // 0: 4x4, 1: 8x8, 2: 16x16.
   kNumLoopFilterMasks = 4,
   kMaxAngleDelta = 3,
   kDirectionalIntraModes = 8,
@@ -30,9 +32,10 @@
   kRestorationTypeSymbolCount = 3,
   kSgrProjParamsBits = 4,
   kSgrProjPrecisionBits = 7,
-  kRestorationBorder = 3,  // Horizontal and vertical border are both 3.
-  kConvolveBorderLeftTop = 3,
-  kConvolveBorderRightBottom = 4,
+  kRestorationBorder = 3,      // Padding on each side of a restoration block.
+  kCdefBorder = 2,             // Padding on each side of a cdef block.
+  kConvolveBorderLeftTop = 3,  // Left/top padding of a convolve block.
+  kConvolveBorderRightBottom = 4,  // Right/bottom padding of a convolve block.
   kSubPixelTaps = 8,
   kWienerFilterBits = 7,
   kMaxPaletteSize = 8,
@@ -178,6 +181,29 @@
   kBlockInvalid
 };
 
+//  Partition types.  R: Recursive
+//
+//  None          Horizontal    Vertical      Split
+//  +-------+     +-------+     +---+---+     +---+---+
+//  |       |     |       |     |   |   |     | R | R |
+//  |       |     +-------+     |   |   |     +---+---+
+//  |       |     |       |     |   |   |     | R | R |
+//  +-------+     +-------+     +---+---+     +---+---+
+//
+//  Horizontal    Horizontal    Vertical      Vertical
+//  with top      with bottom   with left     with right
+//  split         split         split         split
+//  +---+---+     +-------+     +---+---+     +---+---+
+//  |   |   |     |       |     |   |   |     |   |   |
+//  +---+---+     +---+---+     +---+   |     |   +---+
+//  |       |     |   |   |     |   |   |     |   |   |
+//  +-------+     +---+---+     +---+---+     +---+---+
+//
+//  Horizontal4   Vertical4
+//  +-----+       +-+-+-+
+//  +-----+       | | | |
+//  +-----+       | | | |
+//  +-----+       +-+-+-+
 enum Partition : uint8_t {
   kPartitionNone,
   kPartitionHorizontal,
@@ -276,19 +302,14 @@
   kNumTransformTypes
 };
 
-// Allows checking whether a transform requires rows or columns to be flipped
-// with a single comparison rather than a chain of ||s. This should result in
-// fewer instructions overall.
-enum : uint32_t {
-  kTransformFlipColumnsMask = (1U << kTransformTypeFlipadstDct) |
-                              (1U << kTransformTypeFlipadstAdst) |
-                              (1U << kTransformTypeFlipadstIdentity) |
-                              (1U << kTransformTypeFlipadstFlipadst),
-  kTransformFlipRowsMask = (1U << kTransformTypeDctFlipadst) |
-                           (1U << kTransformTypeAdstFlipadst) |
-                           (1U << kTransformTypeIdentityFlipadst) |
-                           (1U << kTransformTypeFlipadstFlipadst)
-};
+constexpr BitMaskSet kTransformFlipColumnsMask(kTransformTypeFlipadstDct,
+                                               kTransformTypeFlipadstAdst,
+                                               kTransformTypeFlipadstIdentity,
+                                               kTransformTypeFlipadstFlipadst);
+constexpr BitMaskSet kTransformFlipRowsMask(kTransformTypeDctFlipadst,
+                                            kTransformTypeAdstFlipadst,
+                                            kTransformTypeIdentityFlipadst,
+                                            kTransformTypeFlipadstFlipadst);
 
 enum TransformSize : uint8_t {
   kTransformSize4x4,
@@ -313,24 +334,6 @@
   kNumTransformSizes
 };
 
-enum : uint32_t {
-  // Mask of all transform sizes with either dimension equal to 64.
-  kTransformSize64Mask =
-      (1U << kTransformSize64x16) | (1U << kTransformSize64x32) |
-      (1U << kTransformSize64x64) | (1U << kTransformSize16x64) |
-      (1U << kTransformSize32x64),
-  // Mask of all transform sizes with width equal to 16.
-  kTransformWidth16Mask =
-      (1U << kTransformSize16x4) | (1U << kTransformSize16x8) |
-      (1U << kTransformSize16x16) | (1U << kTransformSize16x32) |
-      (1U << kTransformSize16x64),
-  // Mask of all transform sizes with height equal to 16.
-  kTransformHeight16Mask =
-      (1U << kTransformSize4x16) | (1U << kTransformSize8x16) |
-      (1U << kTransformSize16x16) | (1U << kTransformSize32x16) |
-      (1U << kTransformSize64x16)
-};
-
 enum TransformSet : uint8_t {
   // DCT Only (1).
   kTransformSetDctOnly,
@@ -364,6 +367,12 @@
   kNumFilterIntraPredictors
 };
 
+enum ObmcDirection : uint8_t {
+  kObmcDirectionVertical,
+  kObmcDirectionHorizontal,
+  kNumObmcDirections
+};
+
 // In AV1 the name of the filter refers to the direction of filter application.
 // Horizontal refers to the column edge and vertical the row edge.
 enum LoopFilterType : uint8_t {
@@ -372,6 +381,13 @@
   kNumLoopFilterTypes
 };
 
+enum LoopFilterTransformSizeId : uint8_t {
+  kLoopFilterTransformSizeId4x4,
+  kLoopFilterTransformSizeId8x8,
+  kLoopFilterTransformSizeId16x16,
+  kNumLoopFilterTransformSizeIds
+};
+
 enum LoopRestorationType : uint8_t {
   kLoopRestorationTypeNone,
   kLoopRestorationTypeSwitchable,
@@ -433,6 +449,155 @@
   kObuPadding = 15,
 };
 
+//------------------------------------------------------------------------------
+// ToString()
+//
+// These functions are meant to be used only in debug logging and within tests.
+// They are defined inline to avoid including the strings in the release
+// library when logging is disabled; unreferenced functions will not be added to
+// any object file in that case.
+
+inline const char* ToString(const BlockSize size) {
+  switch (size) {
+    case kBlock4x4:
+      return "kBlock4x4";
+    case kBlock4x8:
+      return "kBlock4x8";
+    case kBlock4x16:
+      return "kBlock4x16";
+    case kBlock8x4:
+      return "kBlock8x4";
+    case kBlock8x8:
+      return "kBlock8x8";
+    case kBlock8x16:
+      return "kBlock8x16";
+    case kBlock8x32:
+      return "kBlock8x32";
+    case kBlock16x4:
+      return "kBlock16x4";
+    case kBlock16x8:
+      return "kBlock16x8";
+    case kBlock16x16:
+      return "kBlock16x16";
+    case kBlock16x32:
+      return "kBlock16x32";
+    case kBlock16x64:
+      return "kBlock16x64";
+    case kBlock32x8:
+      return "kBlock32x8";
+    case kBlock32x16:
+      return "kBlock32x16";
+    case kBlock32x32:
+      return "kBlock32x32";
+    case kBlock32x64:
+      return "kBlock32x64";
+    case kBlock64x16:
+      return "kBlock64x16";
+    case kBlock64x32:
+      return "kBlock64x32";
+    case kBlock64x64:
+      return "kBlock64x64";
+    case kBlock64x128:
+      return "kBlock64x128";
+    case kBlock128x64:
+      return "kBlock128x64";
+    case kBlock128x128:
+      return "kBlock128x128";
+    case kMaxBlockSizes:
+      return "kMaxBlockSizes";
+    case kBlockInvalid:
+      return "kBlockInvalid";
+  }
+  abort();
+}
+
+inline const char* ToString(const InterIntraMode mode) {
+  switch (mode) {
+    case kInterIntraModeDc:
+      return "kInterIntraModeDc";
+    case kInterIntraModeVertical:
+      return "kInterIntraModeVertical";
+    case kInterIntraModeHorizontal:
+      return "kInterIntraModeHorizontal";
+    case kInterIntraModeSmooth:
+      return "kInterIntraModeSmooth";
+    case kNumInterIntraModes:
+      return "kNumInterIntraModes";
+  }
+  abort();
+}
+
+inline const char* ToString(const ObmcDirection direction) {
+  switch (direction) {
+    case kObmcDirectionVertical:
+      return "kObmcDirectionVertical";
+    case kObmcDirectionHorizontal:
+      return "kObmcDirectionHorizontal";
+    case kNumObmcDirections:
+      return "kNumObmcDirections";
+  }
+  abort();
+}
+
+inline const char* ToString(const LoopRestorationType type) {
+  switch (type) {
+    case kLoopRestorationTypeNone:
+      return "kLoopRestorationTypeNone";
+    case kLoopRestorationTypeSwitchable:
+      return "kLoopRestorationTypeSwitchable";
+    case kLoopRestorationTypeWiener:
+      return "kLoopRestorationTypeWiener";
+    case kLoopRestorationTypeSgrProj:
+      return "kLoopRestorationTypeSgrProj";
+    case kNumLoopRestorationTypes:
+      return "kNumLoopRestorationTypes";
+  }
+  abort();
+}
+
+inline const char* ToString(const TransformType type) {
+  switch (type) {
+    case kTransformTypeDctDct:
+      return "kTransformTypeDctDct";
+    case kTransformTypeAdstDct:
+      return "kTransformTypeAdstDct";
+    case kTransformTypeDctAdst:
+      return "kTransformTypeDctAdst";
+    case kTransformTypeAdstAdst:
+      return "kTransformTypeAdstAdst";
+    case kTransformTypeFlipadstDct:
+      return "kTransformTypeFlipadstDct";
+    case kTransformTypeDctFlipadst:
+      return "kTransformTypeDctFlipadst";
+    case kTransformTypeFlipadstFlipadst:
+      return "kTransformTypeFlipadstFlipadst";
+    case kTransformTypeAdstFlipadst:
+      return "kTransformTypeAdstFlipadst";
+    case kTransformTypeFlipadstAdst:
+      return "kTransformTypeFlipadstAdst";
+    case kTransformTypeIdentityIdentity:
+      return "kTransformTypeIdentityIdentity";
+    case kTransformTypeIdentityDct:
+      return "kTransformTypeIdentityDct";
+    case kTransformTypeDctIdentity:
+      return "kTransformTypeDctIdentity";
+    case kTransformTypeIdentityAdst:
+      return "kTransformTypeIdentityAdst";
+    case kTransformTypeAdstIdentity:
+      return "kTransformTypeAdstIdentity";
+    case kTransformTypeIdentityFlipadst:
+      return "kTransformTypeIdentityFlipadst";
+    case kTransformTypeFlipadstIdentity:
+      return "kTransformTypeFlipadstIdentity";
+    // case to quiet compiler
+    case kNumTransformTypes:
+      return "kNumTransformTypes";
+  }
+  abort();
+}
+
+//------------------------------------------------------------------------------
+
 extern const uint8_t k4x4WidthLog2[kMaxBlockSizes];
 
 extern const uint8_t k4x4HeightLog2[kMaxBlockSizes];
@@ -453,14 +618,14 @@
 
 extern const uint8_t kTransformHeight[kNumTransformSizes];
 
+extern const uint8_t kTransformWidth4x4[kNumTransformSizes];
+
+extern const uint8_t kTransformHeight4x4[kNumTransformSizes];
+
 extern const uint8_t kTransformWidthLog2[kNumTransformSizes];
 
 extern const uint8_t kTransformHeightLog2[kNumTransformSizes];
 
-extern const TransformSize kMaxTransformSizeRectangle[kMaxBlockSizes];
-
-extern const int kMaxTransformDepth[kMaxBlockSizes];
-
 extern const TransformSize kSplitTransformSize[kNumTransformSizes];
 
 // Square transform of size min(w,h).
@@ -469,26 +634,14 @@
 // Square transform of size max(w,h).
 extern const TransformSize kTransformSizeSquareMax[kNumTransformSizes];
 
-extern const TransformType kModeToTransformType[kIntraPredictionModesUV];
-
 extern const uint8_t kNumTransformTypesInSet[kNumTransformSets];
 
-extern const TransformType kInverseTransformTypeBySet[kNumTransformSets - 1]
-                                                     [16];
-
-// Replaces all occurrences of 64x* and *x64 with 32x* and *x32 respectively.
-extern const TransformSize kAdjustedTransformSize[kNumTransformSizes];
-
 extern const uint8_t kSgrProjParams[1 << kSgrProjParamsBits][4];
 
 extern const int8_t kSgrProjMultiplierMin[2];
 
 extern const int8_t kSgrProjMultiplierMax[2];
 
-extern const int8_t kSgrProjDefaultMultiplier[2];
-
-extern const int8_t kWienerDefaultFilter[3];
-
 extern const int8_t kWienerTapsMin[3];
 
 extern const int8_t kWienerTapsMax[3];
@@ -501,7 +654,15 @@
 
 extern const int16_t kDirectionalIntraPredictorDerivative[44];
 
-extern const uint8_t kPredictionModeDeltasLookup[kNumPredictionModes];
+extern const uint8_t kDeblockFilterLevelIndex[kMaxPlanes][kNumLoopFilterTypes];
+
+extern const int8_t kMaskIdLookup[4][kMaxBlockSizes];
+
+extern const int8_t kVerticalBorderMaskIdLookup[kMaxBlockSizes];
+
+extern const uint64_t kTopMaskLookup[67][4];
+
+extern const uint64_t kLeftMaskLookup[67][4];
 
 }  // namespace libgav1
 
diff --git a/libgav1/src/utils/entropy_decoder.cc b/libgav1/src/utils/entropy_decoder.cc
index a925da4..ff6db39 100644
--- a/libgav1/src/utils/entropy_decoder.cc
+++ b/libgav1/src/utils/entropy_decoder.cc
@@ -7,98 +7,37 @@
 
 namespace {
 
-constexpr uint32_t kWindowSize = static_cast<uint32_t>(sizeof(uint32_t)) * 8;
-constexpr int kCdfPrecision = 6;
-constexpr int kMinimumProbabilityPerSymbol = 4;
 constexpr uint32_t kReadBitMask = ~255;
 // This constant is used to set the value of |bits_| so that bits can be read
 // after end of stream without trying to refill the buffer for a reasonably long
 // time.
 constexpr int kLargeBitCount = 0x4000;
+constexpr int kCdfPrecision = 6;
+constexpr int kMinimumProbabilityPerSymbol = 4;
 
-void UpdateCdf(uint16_t* cdf, int value, int symbol_count) {
-  const uint16_t count = cdf[symbol_count];
-  // rate is computed in the spec as:
-  //  3 + ( cdf[N] > 15 ) + ( cdf[N] > 31 ) + Min(FloorLog2(N), 2)
-  // In this case cdf[N] is |count|.
-  // Min(FloorLog2(N), 2) is 1 for symbol_count == {2, 3} and 2 for all
-  // symbol_count > 3. So the equation becomes:
-  //  4 + (count > 15) + (count > 31) + (symbol_count > 3).
-  // Note that the largest value for count is 32 (it is not incremented beyond
-  // 32). So using that information:
-  //  count >> 4 is 0 for count from 0 to 15.
-  //  count >> 4 is 1 for count from 16 to 31.
-  //  count >> 4 is 2 for count == 31.
-  // Now, the equation becomes:
-  //  4 + (count >> 4) + (symbol_count > 3).
-  // Since (count >> 4) can only be 0 or 1 or 2, the addition can be replaced
-  // with bitwise or. So the final equation is:
-  // (4 | (count >> 4)) + (symbol_count > 3).
-  const int rate = (4 | (count >> 4)) + static_cast<int>(symbol_count > 3);
-  for (int i = 0; i < symbol_count - 1; ++i) {
-    if (i < value) {
-      cdf[i] += (libgav1::kCdfMaxProbability - cdf[i]) >> rate;
-    } else {
-      cdf[i] -= cdf[i] >> rate;
-    }
-  }
-  cdf[symbol_count] += static_cast<uint16_t>(count < 32);
+// This function computes the "cur" variable as specified inside the do-while
+// loop in Section 8.2.6 of the spec. This function is monotonically
+// decreasing as the values of index increases (note that the |cdf| array is
+// sorted in decreasing order).
+uint32_t ScaleCdf(uint16_t values_in_range_shifted, const uint16_t* const cdf,
+                  int index, int symbol_count) {
+  return ((values_in_range_shifted * (cdf[index] >> kCdfPrecision)) >> 1) +
+         (kMinimumProbabilityPerSymbol * (symbol_count - index));
 }
 
 }  // namespace
 
 namespace libgav1 {
 
-#if defined(LIBGAV1_USE_LIBAOM_BIT_READER)
-const int kProbabilityHalf = 128;
+constexpr uint32_t DaalaBitReader::kWindowSize;  // static.
 
-DaalaBitReaderAom::DaalaBitReaderAom(const uint8_t* data, size_t size,
-                                     bool allow_update_cdf)
-    : allow_update_cdf_(allow_update_cdf) {
-  aom_daala_reader_init(&reader_, data, static_cast<int>(size));
-}
-
-int DaalaBitReaderAom::ReadBit() {
-  return aom_daala_read(&reader_, kProbabilityHalf);
-}
-
-int64_t DaalaBitReaderAom::ReadLiteral(int num_bits) {
-  if (num_bits > 32) return -1;
-  uint32_t literal = 0;
-  for (int bit = num_bits - 1; bit >= 0; bit--) {
-    literal |= static_cast<uint32_t>(ReadBit()) << bit;
-  }
-  return literal;
-}
-
-int DaalaBitReaderAom::ReadSymbol(uint16_t* cdf, int symbol_count) {
-  const int symbol = daala_read_symbol(&reader_, cdf, symbol_count);
-  if (allow_update_cdf_) {
-    UpdateCdf(cdf, symbol, symbol_count);
-  }
-  return symbol;
-}
-
-bool DaalaBitReaderAom::ReadSymbol(uint16_t* cdf) {
-  const int symbol = daala_read_symbol(&reader_, cdf, kBooleanSymbolCount);
-  if (allow_update_cdf_) {
-    UpdateCdf(cdf, symbol, kBooleanSymbolCount);
-  }
-  return symbol != 0;
-}
-
-bool DaalaBitReaderAom::ReadSymbolWithoutCdfUpdate(uint16_t* cdf) {
-  return daala_read_symbol(&reader_, cdf, kBooleanSymbolCount) != 0;
-}
-#endif  // defined(LIBGAV1_USE_LIBAOM_BIT_READER)
-
-DaalaBitReaderNative::DaalaBitReaderNative(const uint8_t* data, size_t size,
-                                           bool allow_update_cdf)
+DaalaBitReader::DaalaBitReader(const uint8_t* data, size_t size,
+                               bool allow_update_cdf)
     : data_(data),
       size_(size),
       data_index_(0),
       allow_update_cdf_(allow_update_cdf) {
-  window_diff_ = (uint32_t{1} << (kWindowSize - 1)) - 1;
+  window_diff_ = (WindowSize{1} << (kWindowSize - 1)) - 1;
   values_in_range_ = kCdfMaxProbability;
   bits_ = -15;
   PopulateBits();
@@ -109,10 +48,11 @@
 //   * The probability is fixed at half. So some multiplications can be replaced
 //     with bit operations.
 //   * Symbol count is fixed at 2.
-int DaalaBitReaderNative::ReadBit() {
+int DaalaBitReader::ReadBit() {
   const uint32_t curr =
       ((values_in_range_ & kReadBitMask) >> 1) + kMinimumProbabilityPerSymbol;
-  const uint32_t zero_threshold = curr << (kWindowSize - 16);
+  const WindowSize zero_threshold = static_cast<WindowSize>(curr)
+                                    << (kWindowSize - 16);
   int bit = 1;
   if (window_diff_ >= zero_threshold) {
     values_in_range_ -= curr;
@@ -125,7 +65,7 @@
   return bit;
 }
 
-int64_t DaalaBitReaderNative::ReadLiteral(int num_bits) {
+int64_t DaalaBitReader::ReadLiteral(int num_bits) {
   if (num_bits > 32) return -1;
   uint32_t literal = 0;
   for (int bit = num_bits - 1; bit >= 0; --bit) {
@@ -134,18 +74,16 @@
   return literal;
 }
 
-int DaalaBitReaderNative::ReadSymbol(uint16_t* const cdf, int symbol_count) {
+int DaalaBitReader::ReadSymbol(uint16_t* const cdf, int symbol_count) {
   const int symbol = ReadSymbolImpl(cdf, symbol_count);
   if (allow_update_cdf_) {
-    // TODO(vigneshv): This call can be replaced with the function contents
-    // inline once the DaalaBitReaderAom is removed.
-    UpdateCdf(cdf, symbol, symbol_count);
+    UpdateCdf(cdf, symbol_count, symbol);
   }
   return symbol;
 }
 
-bool DaalaBitReaderNative::ReadSymbol(uint16_t* cdf) {
-  const bool symbol = ReadSymbolImpl(cdf, kBooleanSymbolCount) != 0;
+bool DaalaBitReader::ReadSymbol(uint16_t* cdf) {
+  const bool symbol = ReadSymbolImpl(cdf) != 0;
   if (allow_update_cdf_) {
     const uint16_t count = cdf[2];
     // rate is computed in the spec as:
@@ -173,35 +111,98 @@
   return symbol;
 }
 
-bool DaalaBitReaderNative::ReadSymbolWithoutCdfUpdate(uint16_t* cdf) {
-  return ReadSymbolImpl(cdf, kBooleanSymbolCount) != 0;
+bool DaalaBitReader::ReadSymbolWithoutCdfUpdate(uint16_t* cdf) {
+  return ReadSymbolImpl(cdf) != 0;
 }
 
-int DaalaBitReaderNative::ReadSymbolImpl(const uint16_t* const cdf,
-                                         int symbol_count) {
+int DaalaBitReader::ReadSymbolImpl(const uint16_t* const cdf,
+                                   int symbol_count) {
   assert(cdf[symbol_count - 1] == 0);
   --symbol_count;
   uint32_t curr = values_in_range_;
   int symbol = -1;
   uint32_t prev;
-  uint32_t symbol_value = window_diff_ >> (kWindowSize - 16);
+  const auto symbol_value =
+      static_cast<uint32_t>(window_diff_ >> (kWindowSize - 16));
+  uint32_t delta = kMinimumProbabilityPerSymbol * symbol_count;
+  // Search through the |cdf| array to determine where the scaled cdf value and
+  // |symbol_value| cross over.
   do {
     prev = curr;
-    curr = values_in_range_ >> 8;
-    curr *= cdf[++symbol] >> kCdfPrecision;
-    curr >>= 1;
-    curr += kMinimumProbabilityPerSymbol * (symbol_count - symbol);
+    curr = (((values_in_range_ >> 8) * (cdf[++symbol] >> kCdfPrecision)) >> 1) +
+           delta;
+    delta -= kMinimumProbabilityPerSymbol;
   } while (symbol_value < curr);
   values_in_range_ = prev - curr;
-  window_diff_ -= curr << (kWindowSize - 16);
+  window_diff_ -= static_cast<WindowSize>(curr) << (kWindowSize - 16);
   NormalizeRange();
   return symbol;
 }
 
-void DaalaBitReaderNative::PopulateBits() {
+int DaalaBitReader::ReadSymbolImplBinarySearch(const uint16_t* const cdf,
+                                               int symbol_count) {
+  assert(cdf[symbol_count - 1] == 0);
+  assert(symbol_count > 1 && symbol_count <= 16);
+  --symbol_count;
+  const auto symbol_value =
+      static_cast<uint32_t>(window_diff_ >> (kWindowSize - 16));
+  // Search through the |cdf| array to determine where the scaled cdf value and
+  // |symbol_value| cross over. Since the CDFs are sorted, we can use binary
+  // search to do this. Let |symbol| be the index of the first |cdf| array
+  // entry whose scaled cdf value is less than or equal to |symbol_value|. The
+  // binary search maintains the invariant:
+  //   low <= symbol <= high + 1
+  // and terminates when low == high + 1.
+  int low = 0;
+  int high = symbol_count - 1;
+  // The binary search maintains the invariants that |prev| is the scaled cdf
+  // value for low - 1 and |curr| is the scaled cdf value for high + 1. (By
+  // convention, the scaled cdf value for -1 is values_in_range_.) When the
+  // binary search terminates, |prev| is the scaled cdf value for symbol - 1
+  // and |curr| is the scaled cdf value for |symbol|.
+  uint32_t prev = values_in_range_;
+  uint32_t curr = 0;
+  const uint16_t values_in_range_shifted = values_in_range_ >> 8;
+  do {
+    const int mid = DivideBy2(low + high);
+    const uint32_t scaled_cdf =
+        ScaleCdf(values_in_range_shifted, cdf, mid, symbol_count);
+    if (symbol_value < scaled_cdf) {
+      low = mid + 1;
+      prev = scaled_cdf;
+    } else {
+      high = mid - 1;
+      curr = scaled_cdf;
+    }
+  } while (low <= high);
+  assert(low == high + 1);
+  // At this point, |low| is the symbol that has been decoded.
+  values_in_range_ = prev - curr;
+  window_diff_ -= static_cast<WindowSize>(curr) << (kWindowSize - 16);
+  NormalizeRange();
+  return low;
+}
+
+int DaalaBitReader::ReadSymbolImpl(const uint16_t* const cdf) {
+  assert(cdf[1] == 0);
+  const auto symbol_value =
+      static_cast<uint32_t>(window_diff_ >> (kWindowSize - 16));
+  const uint32_t curr = ScaleCdf(values_in_range_ >> 8, cdf, 0, 1);
+  const int symbol = static_cast<int>(symbol_value < curr);
+  if (symbol == 1) {
+    values_in_range_ = curr;
+  } else {
+    values_in_range_ -= curr;
+    window_diff_ -= static_cast<WindowSize>(curr) << (kWindowSize - 16);
+  }
+  NormalizeRange();
+  return symbol;
+}
+
+void DaalaBitReader::PopulateBits() {
   int shift = kWindowSize - 9 - (bits_ + 15);
   for (; shift >= 0 && data_index_ < size_; shift -= 8) {
-    window_diff_ ^= static_cast<uint32_t>(data_[data_index_++]) << shift;
+    window_diff_ ^= static_cast<WindowSize>(data_[data_index_++]) << shift;
     bits_ += 8;
   }
   if (data_index_ >= size_) {
@@ -209,7 +210,7 @@
   }
 }
 
-void DaalaBitReaderNative::NormalizeRange() {
+void DaalaBitReader::NormalizeRange() {
   const int bits_used = 15 - FloorLog2(values_in_range_);
   bits_ -= bits_used;
   window_diff_ = ((window_diff_ + 1) << bits_used) - 1;
@@ -217,4 +218,55 @@
   if (bits_ < 0) PopulateBits();
 }
 
+void DaalaBitReader::UpdateCdf(uint16_t* const cdf, int symbol_count,
+                               int symbol) {
+  const uint16_t count = cdf[symbol_count];
+  // rate is computed in the spec as:
+  //  3 + ( cdf[N] > 15 ) + ( cdf[N] > 31 ) + Min(FloorLog2(N), 2)
+  // In this case cdf[N] is |count|.
+  // Min(FloorLog2(N), 2) is 1 for symbol_count == {2, 3} and 2 for all
+  // symbol_count > 3. So the equation becomes:
+  //  4 + (count > 15) + (count > 31) + (symbol_count > 3).
+  // Note that the largest value for count is 32 (it is not incremented beyond
+  // 32). So using that information:
+  //  count >> 4 is 0 for count from 0 to 15.
+  //  count >> 4 is 1 for count from 16 to 31.
+  //  count >> 4 is 2 for count == 31.
+  // Now, the equation becomes:
+  //  4 + (count >> 4) + (symbol_count > 3).
+  // Since (count >> 4) can only be 0 or 1 or 2, the addition can be replaced
+  // with bitwise or. So the final equation is:
+  // (4 | (count >> 4)) + (symbol_count > 3).
+  const int rate = (4 | (count >> 4)) + static_cast<int>(symbol_count > 3);
+  // Hints for further optimizations:
+  //
+  // 1. clang can vectorize this for loop with width 4, even though the loop
+  // contains an if-else statement. Therefore, it may be advantageous to use
+  // "i < symbol_count" as the loop condition when symbol_count is 8, 12, or 16
+  // (a multiple of 4 that's not too small).
+  //
+  // 2. The for loop can be rewritten in the following form, which would enable
+  // clang to vectorize the loop with width 8:
+  //
+  //   const int mask = (1 << rate) - 1;
+  //   for (int i = 0; i < symbol_count - 1; ++i) {
+  //     const uint16_t a = (i < symbol) ? kCdfMaxProbability : mask;
+  //     cdf[i] += static_cast<int16_t>(a - cdf[i]) >> rate;
+  //   }
+  //
+  // The subtraction (a - cdf[i]) relies on the overflow semantics of unsigned
+  // integer arithmetic. The result of the unsigned subtraction is cast to a
+  // signed integer and right-shifted. This requires the right shift of a
+  // signed integer be an arithmetic shift, which is true for clang, gcc, and
+  // Visual C++.
+  for (int i = 0; i < symbol_count - 1; ++i) {
+    if (i < symbol) {
+      cdf[i] += (libgav1::kCdfMaxProbability - cdf[i]) >> rate;
+    } else {
+      cdf[i] -= cdf[i] >> rate;
+    }
+  }
+  cdf[symbol_count] += static_cast<uint16_t>(count < 32);
+}
+
 }  // namespace libgav1
diff --git a/libgav1/src/utils/entropy_decoder.h b/libgav1/src/utils/entropy_decoder.h
index f6292bc..b891bae 100644
--- a/libgav1/src/utils/entropy_decoder.h
+++ b/libgav1/src/utils/entropy_decoder.h
@@ -4,53 +4,69 @@
 #include <cstddef>
 #include <cstdint>
 
-#if defined(LIBGAV1_USE_LIBAOM_BIT_READER)
-#include "third_party/libaom/git_root/aom_dsp/daalaboolreader.h"
-#endif
-
 #include "src/utils/bit_reader.h"
 
 namespace libgav1 {
 
-#if defined(LIBGAV1_USE_LIBAOM_BIT_READER)
-class DaalaBitReaderAom : public BitReader {
+class DaalaBitReader : public BitReader {
  public:
-  DaalaBitReaderAom(const uint8_t* data, size_t size, bool allow_update_cdf);
-  ~DaalaBitReaderAom() override = default;
-
-  int ReadBit() override;
-  int64_t ReadLiteral(int num_bits) override;
-  int ReadSymbol(uint16_t* cdf, int symbol_count);
-  bool ReadSymbol(uint16_t* cdf);
-  bool ReadSymbolWithoutCdfUpdate(uint16_t* cdf);
-
- private:
-  bool allow_update_cdf_;
-  daala_reader reader_;
-};
-#endif  // defined(LIBGAV1_USE_LIBAOM_BIT_READER)
-
-class DaalaBitReaderNative : public BitReader {
- public:
-  DaalaBitReaderNative(const uint8_t* data, size_t size, bool allow_update_cdf);
-  ~DaalaBitReaderNative() override = default;
+  DaalaBitReader(const uint8_t* data, size_t size, bool allow_update_cdf);
+  ~DaalaBitReader() override = default;
 
   // Move only.
-  DaalaBitReaderNative(DaalaBitReaderNative&& rhs) noexcept;
-  DaalaBitReaderNative& operator=(DaalaBitReaderNative&& rhs) noexcept;
+  DaalaBitReader(DaalaBitReader&& rhs) noexcept;
+  DaalaBitReader& operator=(DaalaBitReader&& rhs) noexcept;
 
   int ReadBit() override;
   int64_t ReadLiteral(int num_bits) override;
+  // ReadSymbol() calls for which the |symbol_count| is only known at runtime
+  // will use this variant.
   int ReadSymbol(uint16_t* cdf, int symbol_count);
+  // ReadSymbol() calls for which the |symbol_count| is equal to 2 (boolean
+  // symbols) will use this variant.
   bool ReadSymbol(uint16_t* cdf);
   bool ReadSymbolWithoutCdfUpdate(uint16_t* cdf);
+  // Use either linear search or binary search for decoding the symbol depending
+  // on |symbol_count|. ReadSymbol calls for which the |symbol_count| is known
+  // at compile time will use this variant.
+  template <int symbol_count>
+  int ReadSymbol(uint16_t* const cdf) {
+    static_assert(symbol_count >= 3 && symbol_count <= 16, "");
+    const int symbol = (symbol_count <= 13)
+                           ? ReadSymbolImpl(cdf, symbol_count)
+                           : ReadSymbolImplBinarySearch(cdf, symbol_count);
+    if (allow_update_cdf_) {
+      UpdateCdf(cdf, symbol_count, symbol);
+    }
+    return symbol;
+  }
 
  private:
+  using WindowSize = uint32_t;
+  static constexpr uint32_t kWindowSize =
+      static_cast<uint32_t>(sizeof(WindowSize)) * 8;
+
+  // Reads a symbol using the |cdf| table which contains the probabilities of
+  // each symbol. On a high level, this function does the following:
+  //   1) Scale the |cdf| values.
+  //   2) Find the index in the |cdf| array where the scaled CDF value crosses
+  //   the modified |window_diff_| threshold.
+  //   3) That index is the symbol that has been decoded.
+  //   4) Update |window_diff_| and |values_in_range_| based on the symbol that
+  //   has been decoded.
   int ReadSymbolImpl(const uint16_t* cdf, int symbol_count);
+  // Similar to ReadSymbolImpl but it uses binary search to perform step 2 in
+  // the comment above. As of now, this function is called when |symbol_count|
+  // is greater than or equal to 8.
+  int ReadSymbolImplBinarySearch(const uint16_t* cdf, int symbol_count);
+  // Specialized implementation of ReadSymbolImpl based on the fact that
+  // symbol_count == 2.
+  int ReadSymbolImpl(const uint16_t* cdf);
   void PopulateBits();
   // Normalizes the range so that 32768 <= |values_in_range_| < 65536. Also
   // calls PopulateBits() if necessary.
   void NormalizeRange();
+  void UpdateCdf(uint16_t* cdf, int symbol_count, int symbol);
 
   const uint8_t* data_;
   const size_t size_;
@@ -63,15 +79,9 @@
   // The difference between the high end of the current range and the coded
   // value minus 1. The 16 least significant bits of this variable is used to
   // decode the next symbol. It is filled in whenever |bits_| is less than 0.
-  uint32_t window_diff_;
+  WindowSize window_diff_;
 };
 
-#if defined(LIBGAV1_USE_LIBAOM_BIT_READER)
-using DaalaBitReader = DaalaBitReaderAom;
-#else
-using DaalaBitReader = DaalaBitReaderNative;
-#endif
-
 }  // namespace libgav1
 
 #endif  // LIBGAV1_SRC_UTILS_ENTROPY_DECODER_H_
diff --git a/libgav1/src/utils/memory.h b/libgav1/src/utils/memory.h
index 1b73dbb..c5eeb26 100644
--- a/libgav1/src/utils/memory.h
+++ b/libgav1/src/utils/memory.h
@@ -15,6 +15,16 @@
 
 namespace libgav1 {
 
+enum {
+// The byte alignment required for buffers used with SIMD code to be read or
+// written with aligned operations.
+#if defined(__i386__) || defined(_M_IX86)
+  kMaxAlignment = 16,  // extended alignment is safe on x86.
+#else
+  kMaxAlignment = alignof(max_align_t),
+#endif
+};
+
 // AlignedAlloc, AlignedFree
 //
 // void* AlignedAlloc(size_t alignment, size_t size);
@@ -113,28 +123,54 @@
   // Class-specific non-throwing allocation functions
   static void* operator new(size_t size, const std::nothrow_t& tag) noexcept {
     if (size > 0x40000000) return nullptr;
+#ifdef __cpp_aligned_new
+    return ::operator new(size, std::align_val_t(kMaxAlignment), tag);
+#else
     return ::operator new(size, tag);
+#endif
   }
   static void* operator new[](size_t size, const std::nothrow_t& tag) noexcept {
     if (size > 0x40000000) return nullptr;
+#ifdef __cpp_aligned_new
+    return ::operator new[](size, std::align_val_t(kMaxAlignment), tag);
+#else
     return ::operator new[](size, tag);
+#endif
   }
 
   // Class-specific deallocation functions.
-  static void operator delete(void* ptr) noexcept { ::operator delete(ptr); }
+  static void operator delete(void* ptr) noexcept {
+#ifdef __cpp_aligned_new
+    ::operator delete(ptr, std::align_val_t(kMaxAlignment));
+#else
+    ::operator delete(ptr);
+#endif
+  }
   static void operator delete[](void* ptr) noexcept {
+#ifdef __cpp_aligned_new
+    ::operator delete[](ptr, std::align_val_t(kMaxAlignment));
+#else
     ::operator delete[](ptr);
+#endif
   }
 
   // Only called if new (std::nothrow) is used and the constructor throws an
   // exception.
   static void operator delete(void* ptr, const std::nothrow_t& tag) noexcept {
+#ifdef __cpp_aligned_new
+    ::operator delete(ptr, std::align_val_t(kMaxAlignment), tag);
+#else
     ::operator delete(ptr, tag);
+#endif
   }
   // Only called if new[] (std::nothrow) is used and the constructor throws an
   // exception.
   static void operator delete[](void* ptr, const std::nothrow_t& tag) noexcept {
+#ifdef __cpp_aligned_new
+    ::operator delete[](ptr, std::align_val_t(kMaxAlignment), tag);
+#else
     ::operator delete[](ptr, tag);
+#endif
   }
 };
 
diff --git a/libgav1/src/utils/mutex.h b/libgav1/src/utils/mutex.h
deleted file mode 100644
index bd84c1e..0000000
--- a/libgav1/src/utils/mutex.h
+++ /dev/null
@@ -1,4 +0,0 @@
-#ifndef LIBGAV1_SRC_UTILS_MUTEX_H_
-#define LIBGAV1_SRC_UTILS_MUTEX_H_
-
-#endif  // LIBGAV1_SRC_UTILS_MUTEX_H_
diff --git a/libgav1/src/utils/parameter_tree.cc b/libgav1/src/utils/parameter_tree.cc
index a6983e6..cd4b24b 100644
--- a/libgav1/src/utils/parameter_tree.cc
+++ b/libgav1/src/utils/parameter_tree.cc
@@ -1,6 +1,7 @@
 #include "src/utils/parameter_tree.h"
 
 #include <cassert>
+#include <memory>
 #include <new>
 
 #include "src/utils/common.h"
@@ -10,7 +11,19 @@
 
 namespace libgav1 {
 
-void ParameterTree::SetPartitionType(Partition partition) {
+// static
+std::unique_ptr<ParameterTree> ParameterTree::Create(int row4x4, int column4x4,
+                                                     BlockSize block_size,
+                                                     bool is_leaf) {
+  std::unique_ptr<ParameterTree> tree(
+      new (std::nothrow) ParameterTree(row4x4, column4x4, block_size));
+  if (tree != nullptr && is_leaf && !tree->SetPartitionType(kPartitionNone)) {
+    tree = nullptr;
+  }
+  return tree;
+}
+
+bool ParameterTree::SetPartitionType(Partition partition) {
   assert(!partition_type_set_);
   partition_ = partition;
   partition_type_set_ = true;
@@ -23,189 +36,84 @@
   switch (partition) {
     case kPartitionNone:
       parameters_.reset(new (std::nothrow) BlockParameters());
-      return;
+      return parameters_ != nullptr;
     case kPartitionHorizontal:
-      children_[0].reset(new (std::nothrow) ParameterTree(row4x4_, column4x4_,
-                                                          sub_size, true));
-      children_[1].reset(new (std::nothrow) ParameterTree(
-          row4x4_ + half_block4x4, column4x4_, sub_size, true));
-      return;
+      children_[0] = ParameterTree::Create(row4x4_, column4x4_, sub_size, true);
+      children_[1] = ParameterTree::Create(row4x4_ + half_block4x4, column4x4_,
+                                           sub_size, true);
+      return children_[0] != nullptr && children_[1] != nullptr;
     case kPartitionVertical:
-      children_[0].reset(new (std::nothrow) ParameterTree(row4x4_, column4x4_,
-                                                          sub_size, true));
-      children_[1].reset(new (std::nothrow) ParameterTree(
-          row4x4_, column4x4_ + half_block4x4, sub_size, true));
-      return;
+      children_[0] = ParameterTree::Create(row4x4_, column4x4_, sub_size, true);
+      children_[1] = ParameterTree::Create(row4x4_, column4x4_ + half_block4x4,
+                                           sub_size, true);
+      return children_[0] != nullptr && children_[1] != nullptr;
     case kPartitionSplit:
-      children_[0].reset(new (std::nothrow) ParameterTree(row4x4_, column4x4_,
-                                                          sub_size, false));
-      children_[1].reset(new (std::nothrow) ParameterTree(
-          row4x4_, column4x4_ + half_block4x4, sub_size, false));
-      children_[2].reset(new (std::nothrow) ParameterTree(
-          row4x4_ + half_block4x4, column4x4_, sub_size, false));
-      children_[3].reset(new (std::nothrow) ParameterTree(
-          row4x4_ + half_block4x4, column4x4_ + half_block4x4, sub_size,
-          false));
-      return;
+      children_[0] =
+          ParameterTree::Create(row4x4_, column4x4_, sub_size, false);
+      children_[1] = ParameterTree::Create(row4x4_, column4x4_ + half_block4x4,
+                                           sub_size, false);
+      children_[2] = ParameterTree::Create(row4x4_ + half_block4x4, column4x4_,
+                                           sub_size, false);
+      children_[3] = ParameterTree::Create(
+          row4x4_ + half_block4x4, column4x4_ + half_block4x4, sub_size, false);
+      return children_[0] != nullptr && children_[1] != nullptr &&
+             children_[2] != nullptr && children_[3] != nullptr;
     case kPartitionHorizontalWithTopSplit:
       assert(split_size != kBlockInvalid);
-      children_[0].reset(new (std::nothrow) ParameterTree(row4x4_, column4x4_,
-                                                          split_size, true));
-      children_[1].reset(new (std::nothrow) ParameterTree(
-          row4x4_, column4x4_ + half_block4x4, split_size, true));
-      children_[2].reset(new (std::nothrow) ParameterTree(
-          row4x4_ + half_block4x4, column4x4_, sub_size, true));
-      return;
+      children_[0] =
+          ParameterTree::Create(row4x4_, column4x4_, split_size, true);
+      children_[1] = ParameterTree::Create(row4x4_, column4x4_ + half_block4x4,
+                                           split_size, true);
+      children_[2] = ParameterTree::Create(row4x4_ + half_block4x4, column4x4_,
+                                           sub_size, true);
+      return children_[0] != nullptr && children_[1] != nullptr &&
+             children_[2] != nullptr;
     case kPartitionHorizontalWithBottomSplit:
       assert(split_size != kBlockInvalid);
-      children_[0].reset(new (std::nothrow) ParameterTree(row4x4_, column4x4_,
-                                                          sub_size, true));
-      children_[1].reset(new (std::nothrow) ParameterTree(
-          row4x4_ + half_block4x4, column4x4_, split_size, true));
-      children_[2].reset(new (std::nothrow) ParameterTree(
-          row4x4_ + half_block4x4, column4x4_ + half_block4x4, split_size,
-          true));
-      return;
+      children_[0] = ParameterTree::Create(row4x4_, column4x4_, sub_size, true);
+      children_[1] = ParameterTree::Create(row4x4_ + half_block4x4, column4x4_,
+                                           split_size, true);
+      children_[2] =
+          ParameterTree::Create(row4x4_ + half_block4x4,
+                                column4x4_ + half_block4x4, split_size, true);
+      return children_[0] != nullptr && children_[1] != nullptr &&
+             children_[2] != nullptr;
     case kPartitionVerticalWithLeftSplit:
       assert(split_size != kBlockInvalid);
-      children_[0].reset(new (std::nothrow) ParameterTree(row4x4_, column4x4_,
-                                                          split_size, true));
-      children_[1].reset(new (std::nothrow) ParameterTree(
-          row4x4_ + half_block4x4, column4x4_, split_size, true));
-      children_[2].reset(new (std::nothrow) ParameterTree(
-          row4x4_, column4x4_ + half_block4x4, sub_size, true));
-      return;
+      children_[0] =
+          ParameterTree::Create(row4x4_, column4x4_, split_size, true);
+      children_[1] = ParameterTree::Create(row4x4_ + half_block4x4, column4x4_,
+                                           split_size, true);
+      children_[2] = ParameterTree::Create(row4x4_, column4x4_ + half_block4x4,
+                                           sub_size, true);
+      return children_[0] != nullptr && children_[1] != nullptr &&
+             children_[2] != nullptr;
     case kPartitionVerticalWithRightSplit:
       assert(split_size != kBlockInvalid);
-      children_[0].reset(new (std::nothrow) ParameterTree(row4x4_, column4x4_,
-                                                          sub_size, true));
-      children_[1].reset(new (std::nothrow) ParameterTree(
-          row4x4_, column4x4_ + half_block4x4, split_size, true));
-      children_[2].reset(new (std::nothrow) ParameterTree(
-          row4x4_ + half_block4x4, column4x4_ + half_block4x4, split_size,
-          true));
-      return;
+      children_[0] = ParameterTree::Create(row4x4_, column4x4_, sub_size, true);
+      children_[1] = ParameterTree::Create(row4x4_, column4x4_ + half_block4x4,
+                                           split_size, true);
+      children_[2] =
+          ParameterTree::Create(row4x4_ + half_block4x4,
+                                column4x4_ + half_block4x4, split_size, true);
+      return children_[0] != nullptr && children_[1] != nullptr &&
+             children_[2] != nullptr;
     case kPartitionHorizontal4:
       for (int i = 0; i < 4; ++i) {
-        children_[i].reset(new (std::nothrow) ParameterTree(
-            row4x4_ + i * quarter_block4x4, column4x4_, sub_size, true));
+        children_[i] = ParameterTree::Create(row4x4_ + i * quarter_block4x4,
+                                             column4x4_, sub_size, true);
+        if (children_[i] == nullptr) return false;
       }
-      return;
-    case kPartitionVertical4:
+      return true;
+    default:
+      assert(partition == kPartitionVertical4);
       for (int i = 0; i < 4; ++i) {
-        children_[i].reset(new (std::nothrow) ParameterTree(
-            row4x4_, column4x4_ + i * quarter_block4x4, sub_size, true));
+        children_[i] = ParameterTree::Create(
+            row4x4_, column4x4_ + i * quarter_block4x4, sub_size, true);
+        if (children_[i] == nullptr) return false;
       }
-      return;
+      return true;
   }
 }
 
-BlockParameters* ParameterTree::Find(int row4x4, int column4x4) const {
-  if (!partition_type_set_ || row4x4 < row4x4_ || column4x4 < column4x4_ ||
-      row4x4 >= row4x4_ + kNum4x4BlocksHigh[block_size_] ||
-      column4x4 >= column4x4_ + kNum4x4BlocksWide[block_size_]) {
-    // Either partition type is not set or the search range is out of bound.
-    return nullptr;
-  }
-  const ParameterTree* node = this;
-  while (node->partition_ != kPartitionNone) {
-    if (!node->partition_type_set_) {
-      LIBGAV1_DLOG(ERROR,
-                   "Partition type was not set for one of the nodes in the "
-                   "path to row4x4: %d column4x4: %d.",
-                   row4x4, column4x4);
-      return nullptr;
-    }
-    const int block_width4x4 = kNum4x4BlocksWide[node->block_size_];
-    const int half_block4x4 = block_width4x4 >> 1;
-    const int quarter_block4x4 = half_block4x4 >> 1;
-    switch (node->partition_) {
-      case kPartitionNone:
-        assert(false);
-        break;
-      case kPartitionHorizontal:
-        if (row4x4 < node->row4x4_ + half_block4x4) {
-          node = node->children_[0].get();
-        } else {
-          node = node->children_[1].get();
-        }
-        break;
-      case kPartitionVertical:
-        if (column4x4 < node->column4x4_ + half_block4x4) {
-          node = node->children_[0].get();
-        } else {
-          node = node->children_[1].get();
-        }
-        break;
-      case kPartitionSplit:
-        if (row4x4 < node->row4x4_ + half_block4x4 &&
-            column4x4 < node->column4x4_ + half_block4x4) {
-          node = node->children_[0].get();
-        } else if (row4x4 < node->row4x4_ + half_block4x4) {
-          node = node->children_[1].get();
-        } else if (column4x4 < node->column4x4_ + half_block4x4) {
-          node = node->children_[2].get();
-        } else {
-          node = node->children_[3].get();
-        }
-        break;
-      case kPartitionHorizontalWithTopSplit:
-        if (row4x4 < node->row4x4_ + half_block4x4 &&
-            column4x4 < node->column4x4_ + half_block4x4) {
-          node = node->children_[0].get();
-        } else if (row4x4 < node->row4x4_ + half_block4x4) {
-          node = node->children_[1].get();
-        } else {
-          node = node->children_[2].get();
-        }
-        break;
-      case kPartitionHorizontalWithBottomSplit:
-        if (row4x4 < node->row4x4_ + half_block4x4) {
-          node = node->children_[0].get();
-        } else if (column4x4 < node->column4x4_ + half_block4x4) {
-          node = node->children_[1].get();
-        } else {
-          node = node->children_[2].get();
-        }
-        break;
-      case kPartitionVerticalWithLeftSplit:
-        if (row4x4 < node->row4x4_ + half_block4x4 &&
-            column4x4 < node->column4x4_ + half_block4x4) {
-          node = node->children_[0].get();
-        } else if (column4x4 < node->column4x4_ + half_block4x4) {
-          node = node->children_[1].get();
-        } else {
-          node = node->children_[2].get();
-        }
-        break;
-      case kPartitionVerticalWithRightSplit:
-        if (column4x4 < node->column4x4_ + half_block4x4) {
-          node = node->children_[0].get();
-        } else if (row4x4 < node->row4x4_ + half_block4x4) {
-          node = node->children_[1].get();
-        } else {
-          node = node->children_[2].get();
-        }
-        break;
-      case kPartitionHorizontal4:
-        for (int i = 0; i < 4; ++i) {
-          if (row4x4 < node->row4x4_ + quarter_block4x4 * (i + 1)) {
-            node = node->children_[i].get();
-            break;
-          }
-        }
-        break;
-      case kPartitionVertical4:
-        for (int i = 0; i < 4; ++i) {
-          if (column4x4 < node->column4x4_ + quarter_block4x4 * (i + 1)) {
-            node = node->children_[i].get();
-            break;
-          }
-        }
-        break;
-    }
-  }
-  return node->parameters_.get();
-}
-
 }  // namespace libgav1
diff --git a/libgav1/src/utils/parameter_tree.h b/libgav1/src/utils/parameter_tree.h
index 7cd0e0b..fabf51a 100644
--- a/libgav1/src/utils/parameter_tree.h
+++ b/libgav1/src/utils/parameter_tree.h
@@ -5,6 +5,7 @@
 #include <memory>
 
 #include "src/utils/common.h"
+#include "src/utils/compiler_attributes.h"
 #include "src/utils/constants.h"
 #include "src/utils/memory.h"
 #include "src/utils/types.h"
@@ -20,13 +21,9 @@
   // false, |block_size| must be a square block, i.e.,
   // kBlockWidthPixels[block_size] must be equal to
   // kBlockHeightPixels[block_size].
-  ParameterTree(int row4x4, int column4x4, BlockSize block_size,
-                bool is_leaf = false)
-      : row4x4_(row4x4), column4x4_(column4x4), block_size_(block_size) {
-    if (is_leaf) {
-      SetPartitionType(kPartitionNone);
-    }
-  }
+  static std::unique_ptr<ParameterTree> Create(int row4x4, int column4x4,
+                                               BlockSize block_size,
+                                               bool is_leaf = false);
 
   // Move only (not Copyable).
   ParameterTree(ParameterTree&& other) = default;
@@ -46,12 +43,7 @@
   //   will have to set them or their descendants to a terminal type.
   // }
   // This function must be called only once per node.
-  void SetPartitionType(Partition partition);
-
-  // Traverses the tree and searches for the node that contains the
-  // BlockParameters for |row4x4| and |column4x4|. Returns nullptr, if the tree
-  // does not contain the BlockParameters for the given coordinates.
-  BlockParameters* Find(int row4x4, int column4x4) const;
+  LIBGAV1_MUST_USE_RESULT bool SetPartitionType(Partition partition);
 
   // Basic getters.
   int row4x4() const { return row4x4_; }
@@ -68,6 +60,9 @@
   BlockParameters* parameters() const { return parameters_.get(); }
 
  private:
+  ParameterTree(int row4x4, int column4x4, BlockSize block_size)
+      : row4x4_(row4x4), column4x4_(column4x4), block_size_(block_size) {}
+
   Partition partition_ = kPartitionNone;
   std::unique_ptr<BlockParameters> parameters_ = nullptr;
   int row4x4_ = -1;
@@ -93,6 +88,8 @@
   //  * Vertical4: 0 left partition; 1 second left partition; 2 third left
   //    partition; 3 right partition;
   std::unique_ptr<ParameterTree> children_[4] = {};
+
+  friend class ParameterTreeTest;
 };
 
 }  // namespace libgav1
diff --git a/libgav1/src/utils/queue.h b/libgav1/src/utils/queue.h
new file mode 100644
index 0000000..614acef
--- /dev/null
+++ b/libgav1/src/utils/queue.h
@@ -0,0 +1,68 @@
+#ifndef LIBGAV1_SRC_UTILS_QUEUE_H_
+#define LIBGAV1_SRC_UTILS_QUEUE_H_
+
+#include <cassert>
+#include <cstddef>
+#include <memory>
+#include <new>
+
+#include "src/utils/compiler_attributes.h"
+
+namespace libgav1 {
+
+// A FIFO queue of a fixed capacity. The elements are copied, so the element
+// type T should be small.
+//
+// WARNING: No error checking is performed.
+template <typename T>
+class Queue {
+ public:
+  LIBGAV1_MUST_USE_RESULT bool Init(size_t capacity) {
+    elements_.reset(new (std::nothrow) T[capacity]);
+    if (elements_ == nullptr) return false;
+    capacity_ = capacity;
+    return true;
+  }
+
+  // Pushes the element |value| to the end of the queue. It is an error to call
+  // Push() when the queue is full.
+  void Push(T value) {
+    assert(size_ < capacity_);
+    elements_[back_++] = value;
+    if (back_ == capacity_) back_ = 0;
+    ++size_;
+  }
+
+  // Returns the element at the front of the queue and removes it from the
+  // queue. It is an error to call Pop() when the queue is empty.
+  T Pop() {
+    assert(size_ != 0);
+    const T front_element = elements_[front_++];
+    if (front_ == capacity_) front_ = 0;
+    --size_;
+    return front_element;
+  }
+
+  // Returns true if the queue is empty.
+  bool Empty() const { return size_ == 0; }
+
+  // Returns true if the queue is full.
+  bool Full() const { return size_ >= capacity_; }
+
+  // Returns the number of elements in the queue.
+  size_t Size() const { return size_; }
+
+ private:
+  // An array of |capacity| elements. Used as a circular array.
+  std::unique_ptr<T[]> elements_;
+  size_t capacity_ = 0;
+  // The index of the element to be removed by Pop().
+  size_t front_ = 0;
+  // The index where the new element is inserted by Push().
+  size_t back_ = 0;
+  size_t size_ = 0;
+};
+
+}  // namespace libgav1
+
+#endif  // LIBGAV1_SRC_UTILS_QUEUE_H_
diff --git a/libgav1/src/utils/raw_bit_reader.cc b/libgav1/src/utils/raw_bit_reader.cc
index 3c3964d..084535a 100644
--- a/libgav1/src/utils/raw_bit_reader.cc
+++ b/libgav1/src/utils/raw_bit_reader.cc
@@ -16,18 +16,16 @@
 namespace libgav1 {
 namespace {
 
-const int kMaximumLeb128Size = 8;
-const uint8_t kLeb128ValueByteMask = 0x7f;
-const uint8_t kLeb128TerminationByteMask = 0x80;
+constexpr int kMaximumLeb128Size = 8;
+constexpr uint8_t kLeb128ValueByteMask = 0x7f;
+constexpr uint8_t kLeb128TerminationByteMask = 0x80;
 
-inline uint8_t Mod8(size_t n) {
+uint8_t Mod8(size_t n) {
   // Last 3 bits are the value of mod 8.
   return n & 0x07;
 }
 
-inline size_t DivideBy8(size_t n, bool ceil) {
-  return (n + (ceil ? 7 : 0)) >> 3;
-}
+size_t DivideBy8(size_t n, bool ceil) { return (n + (ceil ? 7 : 0)) >> 3; }
 
 }  // namespace
 
diff --git a/libgav1/src/utils/scan.h b/libgav1/src/utils/scan.h
deleted file mode 100644
index c880496..0000000
--- a/libgav1/src/utils/scan.h
+++ /dev/null
@@ -1,15 +0,0 @@
-#ifndef LIBGAV1_SRC_UTILS_SCAN_H_
-#define LIBGAV1_SRC_UTILS_SCAN_H_
-
-#include <cstdint>
-
-#include "src/utils/constants.h"
-
-namespace libgav1 {
-
-const uint16_t* GetScan(TransformSize tx_size,
-                        TransformType tx_type);  // 5.11.41.
-
-}  // namespace libgav1
-
-#endif  // LIBGAV1_SRC_UTILS_SCAN_H_
diff --git a/libgav1/src/utils/segmentation.h b/libgav1/src/utils/segmentation.h
index 9229783..f74bdb8 100644
--- a/libgav1/src/utils/segmentation.h
+++ b/libgav1/src/utils/segmentation.h
@@ -53,7 +53,9 @@
   bool feature_enabled[kMaxSegments][kSegmentFeatureMax];
   int16_t feature_data[kMaxSegments][kSegmentFeatureMax];
   bool lossless[kMaxSegments];
-  int16_t qindex[kMaxSegments];
+  // Cached values of get_qindex(1, segmentId), to be consumed by
+  // Tile::ReadTransformType(). The values are in the range [0, 255].
+  uint8_t qindex[kMaxSegments];
 };
 
 }  // namespace libgav1
diff --git a/libgav1/src/utils/stack.h b/libgav1/src/utils/stack.h
new file mode 100644
index 0000000..59c6061
--- /dev/null
+++ b/libgav1/src/utils/stack.h
@@ -0,0 +1,42 @@
+#ifndef LIBGAV1_SRC_UTILS_STACK_H_
+#define LIBGAV1_SRC_UTILS_STACK_H_
+
+#include <cassert>
+
+namespace libgav1 {
+
+// A LIFO stack of a fixed capacity. The elements are copied, so the element
+// type T should be small.
+//
+// WARNING: No error checking is performed.
+template <typename T, int capacity>
+class Stack {
+ public:
+  // Pushes the element |value| to the top of the stack. It is an error to call
+  // Push() when the stack is full.
+  void Push(T value) {
+    ++top_;
+    assert(top_ < capacity);
+    elements_[top_] = value;
+  }
+
+  // Returns the element at the top of the stack and removes it from the stack.
+  // It is an error to call Pop() when the stack is empty.
+  T Pop() {
+    assert(top_ >= 0);
+    return elements_[top_--];
+  }
+
+  // Returns true if the stack is empty.
+  bool Empty() const { return top_ < 0; }
+
+ private:
+  static_assert(capacity > 0, "");
+  T elements_[capacity];
+  // The array index of the top of the stack. The stack is empty if top_ is -1.
+  int top_ = -1;
+};
+
+}  // namespace libgav1
+
+#endif  // LIBGAV1_SRC_UTILS_STACK_H_
diff --git a/libgav1/src/utils/threadpool.cc b/libgav1/src/utils/threadpool.cc
index dea65f9..681ade6 100644
--- a/libgav1/src/utils/threadpool.cc
+++ b/libgav1/src/utils/threadpool.cc
@@ -83,10 +83,15 @@
 
 void ThreadPool::Schedule(std::function<void()> closure) {
   LockMutex();
-  queue_.push_back(std::move(closure));
-  // TODO(jzern): the mutex doesn't need to be locked to signal the condition.
-  SignalOne();
+  if (!queue_.GrowIfNeeded()) {
+    // queue_ is full and we can't grow it. Run |closure| directly.
+    UnlockMutex();
+    closure();
+    return;
+  }
+  queue_.Push(std::move(closure));
   UnlockMutex();
+  SignalOne();
 }
 
 int ThreadPool::num_threads() const { return num_threads_; }
@@ -178,6 +183,7 @@
     assert(rv >= 0);
     rv = pthread_setname_np(name);
     assert(rv == 0);
+    static_cast<void>(rv);
 #elif defined(__ANDROID__) || defined(__GLIBC__)
     // If the |name| buffer is longer than 16 bytes, pthread_setname_np fails
     // with error 34 (ERANGE) on Android.
@@ -188,6 +194,7 @@
     assert(rv >= 0);
     rv = pthread_setname_np(pthread_self(), name);
     assert(rv == 0);
+    static_cast<void>(rv);
 #endif
   }
 }
@@ -195,13 +202,14 @@
 #endif  // defined(_MSC_VER)
 
 void* ThreadPool::WorkerThread::ThreadBody(void* arg) {
-  auto thread = static_cast<WorkerThread*>(arg);
+  auto* thread = static_cast<WorkerThread*>(arg);
   thread->SetupName();
   thread->pool_->WorkerFunction();
   return nullptr;
 }
 
 bool ThreadPool::StartWorkers() {
+  if (!queue_.Init()) return false;
   for (int i = 0; i < num_threads_; ++i) {
     threads_[i] = new (std::nothrow) WorkerThread(this);
     if (threads_[i] == nullptr) return false;
@@ -217,7 +225,7 @@
 void ThreadPool::WorkerFunction() {
   LockMutex();
   while (true) {
-    if (queue_.empty()) {
+    if (queue_.Empty()) {
       if (exit_threads_) {
         break;  // Queue is empty and exit was requested.
       }
@@ -233,7 +241,7 @@
       const auto wait_start = Clock::now();
       while (Clock::now() - wait_start < kBusyWaitDuration) {
         LockMutex();
-        if (!queue_.empty()) {
+        if (!queue_.Empty()) {
           found_job = true;
           break;
         }
@@ -246,7 +254,7 @@
       // point.
       LockMutex();
       // Ensure that the queue is still empty.
-      if (!queue_.empty()) continue;
+      if (!queue_.Empty()) continue;
       if (exit_threads_) {
         break;  // Queue is empty and exit was requested.
       }
@@ -255,8 +263,8 @@
       Wait();
     } else {
       // Take a job from the queue.
-      std::function<void()> job = std::move(queue_.front());
-      queue_.pop_front();
+      std::function<void()> job = std::move(queue_.Front());
+      queue_.Pop();
 
       UnlockMutex();
       // Note that it is good practice to surround this with a try/catch so
@@ -275,9 +283,8 @@
   // Tell worker threads how to exit.
   LockMutex();
   exit_threads_ = true;
-  // TODO(jzern): the mutex doesn't need to be locked to signal the condition.
-  SignalAll();
   UnlockMutex();
+  SignalAll();
 
   // Join all workers. This will block.
   for (int i = 0; i < num_threads_; ++i) {
diff --git a/libgav1/src/utils/threadpool.h b/libgav1/src/utils/threadpool.h
index 2e710d6..238bc44 100644
--- a/libgav1/src/utils/threadpool.h
+++ b/libgav1/src/utils/threadpool.h
@@ -1,7 +1,6 @@
 #ifndef LIBGAV1_SRC_UTILS_THREADPOOL_H_
 #define LIBGAV1_SRC_UTILS_THREADPOOL_H_
 
-#include <deque>
 #include <functional>
 #include <memory>
 
@@ -20,6 +19,7 @@
 #include "src/utils/compiler_attributes.h"
 #include "src/utils/executor.h"
 #include "src/utils/memory.h"
+#include "src/utils/unbounded_queue.h"
 
 namespace libgav1 {
 
@@ -56,6 +56,13 @@
   // Adds the specified "closure" to the queue for processing. If worker threads
   // are available, "closure" will run immediately. Otherwise "closure" is
   // queued for later execution.
+  //
+  // NOTE: If the internal queue is full and cannot be resized because of an
+  // out-of-memory error, the current thread runs "closure" before returning
+  // from Schedule(). For our use cases, this seems better than the
+  // alternatives:
+  //   1. Return a failure status.
+  //   2. Have the current thread wait until the queue is not full.
   void Schedule(std::function<void()> closure) override;
 
   int num_threads() const;
@@ -113,7 +120,7 @@
 
 #endif  // LIBGAV1_THREADPOOL_USE_STD_MUTEX
 
-  std::deque<std::function<void()>> queue_ LIBGAV1_GUARDED_BY(queue_mutex_);
+  UnboundedQueue<std::function<void()>> queue_ LIBGAV1_GUARDED_BY(queue_mutex_);
   // If not all the worker threads are created, the first entry after the
   // created worker threads is a null pointer.
   const std::unique_ptr<WorkerThread*[]> threads_;
diff --git a/libgav1/src/utils/types.h b/libgav1/src/utils/types.h
index d7093ea..53bfe5c 100644
--- a/libgav1/src/utils/types.h
+++ b/libgav1/src/utils/types.h
@@ -4,6 +4,7 @@
 #include <cstdint>
 #include <memory>
 
+#include "src/utils/array_2d.h"
 #include "src/utils/constants.h"
 #include "src/utils/memory.h"
 
@@ -63,7 +64,7 @@
   int8_t cfl_alpha_v;
   int max_luma_width;
   int max_luma_height;
-  uint8_t color_index_map[kNumPlaneTypes][kMaxPaletteSquare][kMaxPaletteSquare];
+  Array2D<uint8_t> color_index_map[kNumPlaneTypes];
   bool use_intra_block_copy;
   InterIntraMode inter_intra_mode;
   bool is_wedge_inter_intra;
@@ -97,13 +98,19 @@
   PredictionMode y_mode;
   PredictionMode uv_mode;
   TransformSize transform_size;
+  TransformSize uv_transform_size;
   PaletteModeInfo palette_mode_info;
   ReferenceFrameType reference_frame[2];
   MotionVector mv[2];
   bool is_explicit_compound_type;  // comp_group_idx in the spec.
   bool is_compound_type_average;   // compound_idx in the spec.
   InterpolationFilter interpolation_filter[2];
-  uint8_t deblock_filter_level[kMaxPlanes][kNumLoopFilterTypes];
+  // The index of this array is as follows:
+  //  0 - Y plane vertical filtering.
+  //  1 - Y plane horizontal filtering.
+  //  2 - U plane (both directions).
+  //  3 - V plane (both directions).
+  uint8_t deblock_filter_level[kFrameLfCount];
   // When |Tile::split_parse_and_decode_| is true, each block gets its own
   // instance of |prediction_parameters|. When it is false, all the blocks point
   // to |Tile::prediction_parameters_|. This field is valid only as long as the
diff --git a/libgav1/src/utils/unbounded_queue.h b/libgav1/src/utils/unbounded_queue.h
new file mode 100644
index 0000000..611ac50
--- /dev/null
+++ b/libgav1/src/utils/unbounded_queue.h
@@ -0,0 +1,227 @@
+#ifndef LIBGAV1_SRC_UTILS_UNBOUNDED_QUEUE_H_
+#define LIBGAV1_SRC_UTILS_UNBOUNDED_QUEUE_H_
+
+#include <cassert>
+#include <cstddef>
+#include <memory>
+#include <new>
+#include <utility>
+
+#include "src/utils/compiler_attributes.h"
+#include "src/utils/memory.h"
+
+namespace libgav1 {
+
+// A FIFO queue of an unbounded capacity.
+//
+// This implementation uses the general approach used in std::deque
+// implementations. See, for example,
+// https://stackoverflow.com/questions/6292332/what-really-is-a-deque-in-stl
+//
+// It is much simpler because it just needs to support the queue interface.
+// The blocks are chained into a circular list, not managed by a "map". It
+// does not shrink the internal buffer.
+//
+// An alternative implementation approach is a resizable circular array. See,
+// for example, ResizingArrayQueue.java in https://algs4.cs.princeton.edu/code/
+// and base::circular_deque in Chromium's base/containers library.
+template <typename T>
+class UnboundedQueue {
+ public:
+  UnboundedQueue() = default;
+
+  // Move only.
+  UnboundedQueue(UnboundedQueue&& other)
+      : first_block_(other.first_block_),
+        front_(other.front_),
+        last_block_(other.last_block_),
+        back_(other.back_) {
+    other.first_block_ = nullptr;
+    other.front_ = 0;
+    other.last_block_ = nullptr;
+    other.back_ = 0;
+  }
+  UnboundedQueue& operator=(UnboundedQueue&& other) {
+    if (this != &other) {
+      Destroy();
+      first_block_ = other.first_block_;
+      front_ = other.front_;
+      last_block_ = other.last_block_;
+      back_ = other.back_;
+      other.first_block_ = nullptr;
+      other.front_ = 0;
+      other.last_block_ = nullptr;
+      other.back_ = 0;
+    }
+    return *this;
+  }
+
+  ~UnboundedQueue() { Destroy(); }
+
+  // Allocates two Blocks upfront because most access patterns require at
+  // least two Blocks. Returns false if the allocation of the Blocks failed.
+  LIBGAV1_MUST_USE_RESULT bool Init() {
+    std::unique_ptr<Block> new_block0(new (std::nothrow) Block);
+    std::unique_ptr<Block> new_block1(new (std::nothrow) Block);
+    if (new_block0 == nullptr || new_block1 == nullptr) return false;
+    first_block_ = last_block_ = new_block0.release();
+    new_block1->next = first_block_;
+    last_block_->next = new_block1.release();
+    return true;
+  }
+
+  // Checks if the queue has room for a new element. If the queue is full,
+  // tries to grow it. Returns false if the queue is full and the attempt to
+  // grow it failed.
+  //
+  // NOTE: GrowIfNeeded() must be called before each call to Push(). This
+  // inconvenient design is necessary to guarantee a successful Push() call.
+  //
+  // Push(T&& value) is often called with the argument std::move(value). The
+  // moved-from object |value| won't be usable afterwards, so it would be
+  // problematic if Push(T&& value) failed and we lost access to the original
+  // |value| object.
+  LIBGAV1_MUST_USE_RESULT bool GrowIfNeeded() {
+    assert(last_block_ != nullptr);
+    if (back_ == kBlockCapacity) {
+      if (last_block_->next == first_block_) {
+        // All Blocks are in use.
+        std::unique_ptr<Block> new_block(new (std::nothrow) Block);
+        if (new_block == nullptr) return false;
+        new_block->next = first_block_;
+        last_block_->next = new_block.release();
+      }
+      last_block_ = last_block_->next;
+      back_ = 0;
+    }
+    return true;
+  }
+
+  // Pushes the element |value| to the end of the queue. It is an error to call
+  // Push() when the queue is full.
+  void Push(const T& value) {
+    assert(last_block_ != nullptr);
+    assert(back_ < kBlockCapacity);
+    T* elements = reinterpret_cast<T*>(last_block_->buffer);
+    new (&elements[back_++]) T(value);
+  }
+
+  void Push(T&& value) {
+    assert(last_block_ != nullptr);
+    assert(back_ < kBlockCapacity);
+    T* elements = reinterpret_cast<T*>(last_block_->buffer);
+    new (&elements[back_++]) T(std::move(value));
+  }
+
+  // Returns the element at the front of the queue. It is an error to call
+  // Front() when the queue is empty.
+  T& Front() {
+    assert(!Empty());
+    T* elements = reinterpret_cast<T*>(first_block_->buffer);
+    return elements[front_];
+  }
+
+  const T& Front() const {
+    assert(!Empty());
+    T* elements = reinterpret_cast<T*>(first_block_->buffer);
+    return elements[front_];
+  }
+
+  // Removes the element at the front of the queue from the queue. It is an
+  // error to call Pop() when the queue is empty.
+  void Pop() {
+    assert(!Empty());
+    T* elements = reinterpret_cast<T*>(first_block_->buffer);
+    elements[front_++].~T();
+    if (front_ == kBlockCapacity) {
+      // The first block has become empty.
+      front_ = 0;
+      if (first_block_ == last_block_) {
+        // Only one Block is in use. Simply reset back_.
+        back_ = 0;
+      } else {
+        first_block_ = first_block_->next;
+      }
+    }
+  }
+
+  // Returns true if the queue is empty.
+  bool Empty() const { return first_block_ == last_block_ && front_ == back_; }
+
+ private:
+  // kBlockCapacity is the maximum number of elements each Block can hold.
+  // sizeof(void*) is subtracted from 2048 to account for the |next| pointer in
+  // the Block struct.
+  //
+  // In Linux x86_64, sizeof(std::function<void()>) is 32, so each Block can
+  // hold 63 std::function<void()> objects.
+  //
+  // NOTE: The corresponding value in <deque> in libc++ revision
+  // 245b5ba3448b9d3f6de5962066557e253a6bc9a4 is:
+  //   template <class _ValueType, class _DiffType>
+  //   struct __deque_block_size {
+  //     static const _DiffType value =
+  //         sizeof(_ValueType) < 256 ? 4096 / sizeof(_ValueType) : 16;
+  //   };
+  //
+  // Note that 4096 / 256 = 16, so apparently this expression is intended to
+  // ensure the block size is at least 4096 bytes and each block can hold at
+  // least 16 elements.
+  static constexpr size_t kBlockCapacity =
+      (sizeof(T) < 128) ? (2048 - sizeof(void*)) / sizeof(T) : 16;
+
+  struct Block : public Allocable {
+    alignas(T) char buffer[kBlockCapacity * sizeof(T)];
+    Block* next;
+  };
+
+  void Destroy() {
+    if (first_block_ == nullptr) return;  // An uninitialized queue.
+
+    // First free the unused blocks, which are located after last_block and
+    // before first_block_.
+    Block* block = last_block_->next;
+    // Cut the circular list open after last_block_.
+    last_block_->next = nullptr;
+    while (block != first_block_) {
+      Block* next = block->next;
+      delete block;
+      block = next;
+    }
+
+    // Then free the used blocks. Destruct the elements in the used blocks.
+    while (block != nullptr) {
+      const size_t begin = (block == first_block_) ? front_ : 0;
+      const size_t end = (block == last_block_) ? back_ : kBlockCapacity;
+      T* elements = reinterpret_cast<T*>(block->buffer);
+      for (size_t i = begin; i < end; ++i) {
+        elements[i].~T();
+      }
+      Block* next = block->next;
+      delete block;
+      block = next;
+    }
+  }
+
+  // Blocks are chained in a circular singly-linked list. If the list of Blocks
+  // is empty, both first_block_ and last_block_ are null pointers. If the list
+  // is nonempty, first_block_ points to the first used Block and last_block_
+  // points to the last used Block.
+  //
+  // Invariant: If Init() is called and succeeds, the queue is always nonempty.
+  // This allows all methods (except the destructor) to avoid null pointer
+  // checks for first_block_ and last_block_.
+  Block* first_block_ = nullptr;
+  // The index of the element in first_block_ to be removed by Pop().
+  size_t front_ = 0;
+  Block* last_block_ = nullptr;
+  // The index in last_block_ where the new element is inserted by Push().
+  size_t back_ = 0;
+};
+
+template <typename T>
+constexpr size_t UnboundedQueue<T>::kBlockCapacity;
+
+}  // namespace libgav1
+
+#endif  // LIBGAV1_SRC_UTILS_UNBOUNDED_QUEUE_H_
diff --git a/libgav1/src/utils/vector.h b/libgav1/src/utils/vector.h
index ad10a6f..f60a0a0 100644
--- a/libgav1/src/utils/vector.h
+++ b/libgav1/src/utils/vector.h
@@ -1,90 +1,325 @@
+// libgav1::Vector implementation
+
 #ifndef LIBGAV1_SRC_UTILS_VECTOR_H_
 #define LIBGAV1_SRC_UTILS_VECTOR_H_
 
-#include <algorithm>
-#include <initializer_list>
+#include <cassert>
+#include <cstddef>
+#include <cstdlib>
+#include <cstring>
+#include <iterator>
+#include <type_traits>
 #include <utility>
-#include <vector>
 
-#include "src/utils/allocator.h"
 #include "src/utils/compiler_attributes.h"
 
 namespace libgav1 {
+namespace internal {
+
+static constexpr size_t kMinVectorAllocation = 16;
+
+// Returns the smallest power of two greater or equal to 'value'.
+inline size_t NextPow2(size_t value) {
+  if (value == 0) return 0;
+  --value;
+  for (size_t i = 1; i < sizeof(size_t) * 8; i *= 2) value |= value >> i;
+  return value + 1;
+}
+
+// Returns the smallest capacity greater or equal to 'value'.
+inline size_t NextCapacity(size_t value) {
+  if (value == 0) return 0;
+  if (value <= kMinVectorAllocation) return kMinVectorAllocation;
+  return NextPow2(value);
+}
 
 //------------------------------------------------------------------------------
-// Vector class that does *NOT* initialize the content by default, unless an
-// explicit value is passed to the constructor.
-// Should be reserved to POD preferably.
+// Data structure equivalent to std::vector but returning false and to its last
+// valid state on memory allocation failure.
+// std::vector with a custom allocator does not fill this need without
+// exceptions.
 
-// resize(), reserve(), and push_back() are overridden to return bool.
-//
-// New methods: ok() and CopyFrom().
-//
-// DO NOT USE emplace_back(), insert(), and emplace().
-template <typename T, typename super = std::vector<T, AllocatorNoCtor<T>>>
-class VectorNoCtor : public super {
+template <typename T>
+class VectorBase {
  public:
-  using super::super;
-  bool ok() const { return this->get_allocator().ok(); }
-  T* operator*() = delete;
-  LIBGAV1_MUST_USE_RESULT inline bool resize(size_t n) {
-    return ok() && (super::resize(n), ok());
+  using iterator = T*;
+  using const_iterator = const T*;
+
+  VectorBase() noexcept = default;
+  // Move only.
+  VectorBase(const VectorBase&) = delete;
+  VectorBase& operator=(const VectorBase&) = delete;
+  VectorBase(VectorBase&& other) noexcept
+      : items_(other.items_),
+        capacity_(other.capacity_),
+        num_items_(other.num_items_) {
+    other.items_ = nullptr;
+    other.capacity_ = 0;
+    other.num_items_ = 0;
   }
-  LIBGAV1_MUST_USE_RESULT inline bool reserve(size_t n) {
-    return ok() && (super::reserve(n), ok());
+  VectorBase& operator=(VectorBase&& other) noexcept {
+    if (this != &other) {
+      clear();
+      free(items_);
+      items_ = other.items_;
+      capacity_ = other.capacity_;
+      num_items_ = other.num_items_;
+      other.items_ = nullptr;
+      other.capacity_ = 0;
+      other.num_items_ = 0;
+    }
+    return *this;
   }
-  // Release the memory.
-  inline void reset() {
-    VectorNoCtor<T, super> tmp;
-    super::swap(tmp);
+  ~VectorBase() {
+    clear();
+    free(items_);
   }
 
-  // disable resizing ctors
-  VectorNoCtor(size_t size) noexcept = delete;
-  VectorNoCtor(size_t size, const T&) noexcept = delete;
-  VectorNoCtor& operator=(const VectorNoCtor& A) noexcept = delete;
-  VectorNoCtor(const VectorNoCtor& other) noexcept = delete;
-  template <typename InputIt>
-  VectorNoCtor(InputIt first, InputIt last) = delete;
-  VectorNoCtor(std::initializer_list<T> init) = delete;
-
-  // benign ctors
-  VectorNoCtor() noexcept : super() {}
-  VectorNoCtor& operator=(VectorNoCtor&& A) = default;
-  VectorNoCtor(VectorNoCtor&& A) noexcept : super(std::move(A)) {}
-
-  void assign(size_t count, const T& value) = delete;
-  template <typename InputIt>
-  void assign(InputIt first, InputIt last) = delete;
-  void assign(std::initializer_list<T> ilist) = delete;
-
-  // To be used instead of copy-ctor:
-  bool CopyFrom(const VectorNoCtor& A) {
-    if (!resize(A.size())) return false;
-    std::copy(A.begin(), A.end(), super::begin());
+  // Reallocates just enough memory if needed so that 'new_cap' items can fit.
+  LIBGAV1_MUST_USE_RESULT bool reserve(size_t new_cap) {
+    if (capacity_ < new_cap) {
+      T* const new_items = static_cast<T*>(malloc(new_cap * sizeof(T)));
+      if (new_items == nullptr) return false;
+      if (num_items_ > 0) {
+        if (std::is_trivial<T>::value) {
+          memcpy(new_items, items_, num_items_ * sizeof(T));
+        } else {
+          for (size_t i = 0; i < num_items_; ++i) {
+            new (&new_items[i]) T(std::move(items_[i]));
+            items_[i].~T();
+          }
+        }
+      }
+      free(items_);
+      items_ = new_items;
+      capacity_ = new_cap;
+    }
     return true;
   }
-  // Performs a push back *if* the vector was properly allocated.
-  // *NO* re-allocation happens.
-  LIBGAV1_MUST_USE_RESULT inline bool push_back(const T& v) {
-    if (super::size() < super::capacity()) {
-      super::push_back(v);
+
+  // Reallocates less memory so that only the existing items can fit.
+  bool shrink_to_fit() {
+    if (capacity_ == num_items_) return true;
+    if (num_items_ == 0) {
+      free(items_);
+      items_ = nullptr;
+      capacity_ = 0;
       return true;
     }
+    const size_t previous_capacity = capacity_;
+    capacity_ = 0;  // Force reserve() to allocate and copy.
+    if (reserve(num_items_)) return true;
+    capacity_ = previous_capacity;
     return false;
   }
-  LIBGAV1_MUST_USE_RESULT inline bool push_back(T&& v) {
-    if (super::size() < super::capacity()) {
-      super::push_back(v);
-      return true;
+
+  // Constructs a new item by copy constructor. May reallocate if
+  // 'resize_if_needed'.
+  LIBGAV1_MUST_USE_RESULT bool push_back(const T& value,
+                                         bool resize_if_needed = true) {
+    if (num_items_ >= capacity_ &&
+        (!resize_if_needed ||
+         !reserve(internal::NextCapacity(num_items_ + 1)))) {
+      return false;
     }
-    return false;
+    new (&items_[num_items_]) T(value);
+    ++num_items_;
+    return true;
+  }
+
+  // Constructs a new item by copy constructor. reserve() must have been called
+  // with a sufficient capacity.
+  //
+  // WARNING: No error checking is performed.
+  void push_back_unchecked(const T& value) {
+    assert(num_items_ < capacity_);
+    new (&items_[num_items_]) T(value);
+    ++num_items_;
+  }
+
+  // Constructs a new item by move constructor. May reallocate if
+  // 'resize_if_needed'.
+  LIBGAV1_MUST_USE_RESULT bool push_back(T&& value,
+                                         bool resize_if_needed = true) {
+    if (num_items_ >= capacity_ &&
+        (!resize_if_needed ||
+         !reserve(internal::NextCapacity(num_items_ + 1)))) {
+      return false;
+    }
+    new (&items_[num_items_]) T(std::move(value));
+    ++num_items_;
+    return true;
+  }
+
+  // Constructs a new item by move constructor. reserve() must have been called
+  // with a sufficient capacity.
+  //
+  // WARNING: No error checking is performed.
+  void push_back_unchecked(T&& value) {
+    assert(num_items_ < capacity_);
+    new (&items_[num_items_]) T(std::move(value));
+    ++num_items_;
+  }
+
+  // Constructs a new item in place by forwarding the arguments args... to the
+  // constructor. May reallocate.
+  template <typename... Args>
+  LIBGAV1_MUST_USE_RESULT bool emplace_back(Args&&... args) {
+    if (num_items_ >= capacity_ &&
+        !reserve(internal::NextCapacity(num_items_ + 1))) {
+      return false;
+    }
+    new (&items_[num_items_]) T(std::forward<Args>(args)...);
+    ++num_items_;
+    return true;
+  }
+
+  // Destructs the last item.
+  void pop_back() {
+    --num_items_;
+    items_[num_items_].~T();
+  }
+
+  // Destructs the item at 'pos'.
+  void erase(iterator pos) { erase(pos, pos + 1); }
+
+  // Destructs the items in [first,last).
+  void erase(iterator first, iterator last) {
+    for (iterator it = first; it != last; ++it) it->~T();
+    if (last != end()) {
+      if (std::is_trivial<T>::value) {
+        memmove(first, last, (end() - last) * sizeof(T));
+      } else {
+        for (iterator it_src = last, it_dst = first; it_src != end();
+             ++it_src, ++it_dst) {
+          new (it_dst) T(std::move(*it_src));
+          it_src->~T();
+        }
+      }
+    }
+    num_items_ -= std::distance(first, last);
+  }
+
+  // Destructs all the items.
+  void clear() { erase(begin(), end()); }
+
+  // Destroys (including deallocating) all the items.
+  void reset() {
+    clear();
+    if (!shrink_to_fit()) assert(false);
+  }
+
+  // Accessors
+  bool empty() const { return (num_items_ == 0); }
+  size_t size() const { return num_items_; }
+  size_t capacity() const { return capacity_; }
+
+  T* data() { return items_; }
+  T& front() { return items_[0]; }
+  T& back() { return items_[num_items_ - 1]; }
+  T& operator[](size_t i) { return items_[i]; }
+  T& at(size_t i) { return items_[i]; }
+  const T* data() const { return items_; }
+  const T& front() const { return items_[0]; }
+  const T& back() const { return items_[num_items_ - 1]; }
+  const T& operator[](size_t i) const { return items_[i]; }
+  const T& at(size_t i) const { return items_[i]; }
+
+  iterator begin() { return &items_[0]; }
+  const_iterator begin() const { return &items_[0]; }
+  iterator end() { return &items_[num_items_]; }
+  const_iterator end() const { return &items_[num_items_]; }
+
+  void swap(VectorBase& b) {
+    // Although not necessary here, adding "using std::swap;" and then calling
+    // swap() without namespace qualification is recommended. See Effective
+    // C++, Item 25.
+    using std::swap;
+    swap(items_, b.items_);
+    swap(capacity_, b.capacity_);
+    swap(num_items_, b.num_items_);
+  }
+
+ protected:
+  T* items_ = nullptr;
+  size_t capacity_ = 0;
+  size_t num_items_ = 0;
+};
+
+}  // namespace internal
+
+//------------------------------------------------------------------------------
+
+// Vector class that does *NOT* construct the content on resize().
+// Should be reserved to plain old data.
+template <typename T>
+class VectorNoCtor : public internal::VectorBase<T> {
+ public:
+  // Creates or destructs items so that 'new_num_items' exist.
+  // Allocated memory grows every power-of-two items.
+  LIBGAV1_MUST_USE_RESULT bool resize(size_t new_num_items) {
+    using super = internal::VectorBase<T>;
+    if (super::num_items_ < new_num_items) {
+      if (super::capacity_ < new_num_items) {
+        if (!super::reserve(internal::NextCapacity(new_num_items))) {
+          return false;
+        }
+      }
+      super::num_items_ = new_num_items;
+    } else {
+      while (super::num_items_ > new_num_items) {
+        --super::num_items_;
+        super::items_[super::num_items_].~T();
+      }
+    }
+    return true;
   }
 };
 
-// This generic vector class will call the constructors
+// This generic vector class will call the constructors.
 template <typename T>
-using Vector = VectorNoCtor<T, std::vector<T, Allocator<T>>>;
+class Vector : public internal::VectorBase<T> {
+ public:
+  // Constructs or destructs items so that 'new_num_items' exist.
+  // Allocated memory grows every power-of-two items.
+  LIBGAV1_MUST_USE_RESULT bool resize(size_t new_num_items) {
+    using super = internal::VectorBase<T>;
+    if (super::num_items_ < new_num_items) {
+      if (super::capacity_ < new_num_items) {
+        if (!super::reserve(internal::NextCapacity(new_num_items))) {
+          return false;
+        }
+      }
+      while (super::num_items_ < new_num_items) {
+        new (&super::items_[super::num_items_]) T();
+        ++super::num_items_;
+      }
+    } else {
+      while (super::num_items_ > new_num_items) {
+        --super::num_items_;
+        super::items_[super::num_items_].~T();
+      }
+    }
+    return true;
+  }
+};
+
+//------------------------------------------------------------------------------
+
+// Define non-member swap() functions in the namespace in which VectorNoCtor
+// and Vector are implemented. See Effective C++, Item 25.
+
+template <typename T>
+void swap(VectorNoCtor<T>& a, VectorNoCtor<T>& b) {
+  a.swap(b);
+}
+
+template <typename T>
+void swap(Vector<T>& a, Vector<T>& b) {
+  a.swap(b);
+}
+
+//------------------------------------------------------------------------------
 
 }  // namespace libgav1
 
diff --git a/libgav1/src/warp_prediction.cc b/libgav1/src/warp_prediction.cc
index d0d5788..830f951 100644
--- a/libgav1/src/warp_prediction.cc
+++ b/libgav1/src/warp_prediction.cc
@@ -134,16 +134,12 @@
   int bx[2] = {};
   int by[2] = {};
 
-  // TODO(chengchen): for simplicity, the spec always uses absolute coordinates
+  // Note: for simplicity, the spec always uses absolute coordinates
   // in the warp estimation process. subpixel_mid_x, subpixel_mid_y,
   // and candidates are relative to the top left of the frame.
   // In contrast, libaom uses a mixture of coordinate systems.
   // In av1/common/warped_motion.c:find_affine_int(). The coordinate is relative
   // to the top left of the block.
-  // On one hand, we need to make sure libgav1 always keep consistency in
-  // coordinate system.
-  // On the other hand, we might investigate which representation is better for
-  // the sake of efficiency.
   // mid_y/mid_x: the row/column coordinate of the center of the block.
   const int mid_y = MultiplyBy4(row4x4) + MultiplyBy2(block_height4x4) - 1;
   const int mid_x = MultiplyBy4(column4x4) + MultiplyBy2(block_width4x4) - 1;
@@ -167,8 +163,6 @@
     // block, with center of the reference block as origin.
     const int dy = candidates[i][2] - reference_subpixel_mid_y;
     const int dx = candidates[i][3] - reference_subpixel_mid_x;
-    // TODO(chengchen): If this check is done somewhere (in find samples), we
-    // can remove it. If remove, remember to change unit tests input range.
     if (std::abs(sx - dx) < kLargestMotionVectorDiff &&
         std::abs(sy - dy) < kLargestMotionVectorDiff) {
       a[0][0] += LeastSquareProduct(sx, sx) + 8;