Optimizing: Rewrite instruction visiting. Improve performance by rewriting the instruction visiting in a way that clang++ can optimize as an indexed virtual call. This is better than dispatching via `Accept()` as we have less branching and do not need to bring `Accept()` functions scattered across the `libart.so` to instruction cache. Test: m test-art-host-gtest Test: testrunner.py --host --optimizing Bug: 181943478 Bug: 413605257 Flag: EXEMPT refactor Change-Id: I8e3f6fdd880a44679e4b96e35457a18cdf0bdb34

diff --git a/compiler/optimizing/bounds_check_elimination.cc b/compiler/optimizing/bounds_check_elimination.cc
index b44e3b0..6fd1d0e 100644
--- a/compiler/optimizing/bounds_check_elimination.cc
+++ b/compiler/optimizing/bounds_check_elimination.cc

@@ -540,13 +540,13 @@
     for (HInstruction* instruction = block->GetFirstPhi(); instruction != nullptr;) {
       DCHECK(instruction->IsInBlock());
       next_ = instruction->GetNext();
-      instruction->Accept(this);
+      VisitPhi(instruction->AsPhi());
       instruction = next_;
     }
     for (HInstruction* instruction = block->GetFirstInstruction(); instruction != nullptr;) {
       DCHECK(instruction->IsInBlock());
       next_ = instruction->GetNext();
-      instruction->Accept(this);
+      Dispatch(instruction);
       instruction = next_;
     }
     // We should never deoptimize from an osr method, otherwise we might wrongly optimize

diff --git a/compiler/optimizing/cha_guard_optimization.cc b/compiler/optimizing/cha_guard_optimization.cc
index e7e34d8..2f009fa 100644
--- a/compiler/optimizing/cha_guard_optimization.cc
+++ b/compiler/optimizing/cha_guard_optimization.cc

@@ -81,7 +81,7 @@
   instruction_iterator_ = &it;
   for (; !it.Done(); it.Advance()) {
     DCHECK(it.Current()->IsInBlock());
-    it.Current()->Accept(this);
+    Dispatch(it.Current());
   }
 }
 

diff --git a/compiler/optimizing/code_generator.cc b/compiler/optimizing/code_generator.cc
index 935cf72..d013467 100644
--- a/compiler/optimizing/code_generator.cc
+++ b/compiler/optimizing/code_generator.cc

@@ -365,7 +365,7 @@
       }
       DisassemblyScope disassembly_scope(current, *this);
       DCHECK(CheckTypeConsistency(current));
-      current->Accept(instruction_visitor);
+      instruction_visitor->Dispatch(current);
     }
   }
 
@@ -914,7 +914,7 @@
   for (HEnvironment* env = instruction->GetEnvironment(); env != nullptr; env = env->GetParent()) {
     env->AllocateLocations(allocator);
   }
-  instruction->Accept(GetLocationBuilder());
+  GetLocationBuilder()->Dispatch(instruction);
   DCHECK(CheckTypeConsistency(instruction));
   LocationSummary* locations = instruction->GetLocations();
   if (!instruction->IsSuspendCheckEntry()) {

diff --git a/compiler/optimizing/constant_folding.cc b/compiler/optimizing/constant_folding.cc
index 66fb04e..12cfe08 100644
--- a/compiler/optimizing/constant_folding.cc
+++ b/compiler/optimizing/constant_folding.cc

@@ -205,7 +205,7 @@
     // Already replaced inside TryRemoveBinaryOperationViaSelect.
   } else {
     InstructionWithAbsorbingInputSimplifier simplifier(GetGraph());
-    inst->Accept(&simplifier);
+    simplifier.Dispatch(inst);
   }
 }
 

diff --git a/compiler/optimizing/graph_checker.cc b/compiler/optimizing/graph_checker.cc
index 3ad0cd2..0b2e916 100644
--- a/compiler/optimizing/graph_checker.cc
+++ b/compiler/optimizing/graph_checker.cc

@@ -292,7 +292,7 @@
                             current_block_->GetBlockId(),
                             current->GetId()));
     }
-    current->Accept(this);
+    Dispatch(current);
   }
 
   // Visit this block's list of instructions.
@@ -310,7 +310,7 @@
                        current_block_->GetBlockId(),
                        current->GetId()));
     }
-    current->Accept(this);
+    Dispatch(current);
   }
 
   // Ensure that catch blocks are not normal successors, and normal blocks are

diff --git a/compiler/optimizing/graph_visualizer.cc b/compiler/optimizing/graph_visualizer.cc
index 3c3e3a0..3984ea6 100644
--- a/compiler/optimizing/graph_visualizer.cc
+++ b/compiler/optimizing/graph_visualizer.cc

@@ -702,7 +702,7 @@
     HBasicBlock* block = instruction->GetBlock();
     StartAttributeStream("block") << namer_.GetName(block);
 
-    instruction->Accept(this);
+    Dispatch(instruction);
     if (instruction->HasEnvironment()) {
       StringList envs;
       for (HEnvironment* environment = instruction->GetEnvironment();

diff --git a/compiler/optimizing/instruction_simplifier_arm.cc b/compiler/optimizing/instruction_simplifier_arm.cc
index 277d459..25b7405 100644
--- a/compiler/optimizing/instruction_simplifier_arm.cc
+++ b/compiler/optimizing/instruction_simplifier_arm.cc

@@ -64,7 +64,7 @@
     for (HInstructionIteratorPrefetchNext it(block->GetInstructions()); !it.Done(); it.Advance()) {
       HInstruction* instruction = it.Current();
       if (instruction->IsInBlock()) {
-        instruction->Accept(this);
+        Dispatch(instruction);
       }
     }
   }

diff --git a/compiler/optimizing/instruction_simplifier_arm64.cc b/compiler/optimizing/instruction_simplifier_arm64.cc
index 339b006..69b2e95 100644
--- a/compiler/optimizing/instruction_simplifier_arm64.cc
+++ b/compiler/optimizing/instruction_simplifier_arm64.cc

@@ -66,7 +66,7 @@
     for (HInstructionIteratorPrefetchNext it(block->GetInstructions()); !it.Done(); it.Advance()) {
       HInstruction* instruction = it.Current();
       if (instruction->IsInBlock()) {
-        instruction->Accept(this);
+        Dispatch(instruction);
       }
     }
   }

diff --git a/compiler/optimizing/instruction_simplifier_riscv64.cc b/compiler/optimizing/instruction_simplifier_riscv64.cc
index 91b1598..789c431 100644
--- a/compiler/optimizing/instruction_simplifier_riscv64.cc
+++ b/compiler/optimizing/instruction_simplifier_riscv64.cc

@@ -36,7 +36,7 @@
     for (HInstructionIteratorPrefetchNext it(block->GetInstructions()); !it.Done(); it.Advance()) {
       HInstruction* instruction = it.Current();
       if (instruction->IsInBlock()) {
-        instruction->Accept(this);
+        Dispatch(instruction);
       }
     }
   }

diff --git a/compiler/optimizing/instruction_simplifier_x86.cc b/compiler/optimizing/instruction_simplifier_x86.cc
index 7dec4c5..a80dde8 100644
--- a/compiler/optimizing/instruction_simplifier_x86.cc
+++ b/compiler/optimizing/instruction_simplifier_x86.cc

@@ -42,7 +42,7 @@
     for (HInstructionIteratorPrefetchNext it(block->GetInstructions()); !it.Done(); it.Advance()) {
       HInstruction* instruction = it.Current();
       if (instruction->IsInBlock()) {
-        instruction->Accept(this);
+        Dispatch(instruction);
       }
     }
   }

diff --git a/compiler/optimizing/instruction_simplifier_x86_64.cc b/compiler/optimizing/instruction_simplifier_x86_64.cc
index 175369b..eb3227f 100644
--- a/compiler/optimizing/instruction_simplifier_x86_64.cc
+++ b/compiler/optimizing/instruction_simplifier_x86_64.cc

@@ -42,7 +42,7 @@
     for (HInstructionIteratorPrefetchNext it(block->GetInstructions()); !it.Done(); it.Advance()) {
       HInstruction* instruction = it.Current();
       if (instruction->IsInBlock()) {
-        instruction->Accept(this);
+        Dispatch(instruction);
       }
     }
   }

diff --git a/compiler/optimizing/nodes.cc b/compiler/optimizing/nodes.cc
index 184b2a6..1c48834 100644
--- a/compiler/optimizing/nodes.cc
+++ b/compiler/optimizing/nodes.cc

@@ -1718,15 +1718,6 @@
   return nullptr;
 }
 
-#define DEFINE_ACCEPT(name, super)                                             \
-void H##name::Accept(HGraphVisitor* visitor) {                                 \
-  visitor->Visit##name(this);                                                  \
-}
-
-FOR_EACH_CONCRETE_INSTRUCTION(DEFINE_ACCEPT)
-
-#undef DEFINE_ACCEPT
-
 void HGraphVisitor::VisitInsertionOrder() {
   for (HBasicBlock* block : graph_->GetActiveBlocks()) {
     VisitBasicBlock(block);
@@ -1754,14 +1745,14 @@
 void HGraphVisitor::VisitNonPhiInstructions(HBasicBlock* block) {
   for (HInstructionIteratorPrefetchNext it(block->GetInstructions()); !it.Done(); it.Advance()) {
     DCHECK(!it.Current()->IsPhi());
-    it.Current()->Accept(this);
+    Dispatch(it.Current());
   }
 }
 
 void HGraphVisitor::VisitNonPhiInstructionsHandleChanges(HBasicBlock* block) {
   for (HInstructionIterator it(block->GetInstructions()); !it.Done(); it.Advance()) {
     DCHECK(!it.Current()->IsPhi());
-    it.Current()->Accept(this);
+    Dispatch(it.Current());
   }
 }
 

diff --git a/compiler/optimizing/nodes.h b/compiler/optimizing/nodes.h
index 513e37f..7f8f87e 100644
--- a/compiler/optimizing/nodes.h
+++ b/compiler/optimizing/nodes.h

@@ -1519,8 +1519,7 @@
   HInstruction* Clone(ArenaAllocator* arena) const override {             \
     DCHECK(IsClonable());                                                 \
     return new (arena) H##type(*this);                                    \
-  }                                                                       \
-  void Accept(HGraphVisitor* visitor) override
+  }
 
 #define DECLARE_ABSTRACT_INSTRUCTION(type)                              \
   private:                                                              \
@@ -2154,7 +2153,6 @@
     SetRawInputRecordAt(index, HUserRecord<HInstruction*>(input));
   }
 
-  virtual void Accept(HGraphVisitor* visitor) = 0;
   virtual const char* DebugName() const = 0;
 
   DataType::Type GetType() const {
@@ -8417,6 +8415,30 @@
   FOR_EACH_CONCRETE_INSTRUCTION(DECLARE_VISIT_INSTRUCTION)
 #undef DECLARE_VISIT_INSTRUCTION
 
+  ALWAYS_INLINE void Dispatch(HInstruction* insn) {
+    HInstruction::InstructionKind kind;
+    // Use `asm volatile` to prevent clang++ from optimizing the `kind = insn->GetKind()`
+    // together with the `switch`. The simple expression can somehow derail the
+    // `switch` optimization and result in a much worse compiled code. b/413605257
+    asm volatile("" : "=r"(kind) : "0"(insn->GetKind()));
+
+    switch (kind) {
+    #define DEFINE_DISPATCH_CASE(kind, super)                 \
+      case HInstruction::k##kind:                             \
+        Visit##kind(insn->As##kind());                        \
+        break;
+      FOR_EACH_CONCRETE_INSTRUCTION(DEFINE_DISPATCH_CASE)
+    #undef DEFINE_DISPATCH_CASE
+      default:
+        // Note: clang++ can optimize this `switch` to a virtual dispatch with indexed
+        // load from the vtable using an adjusted `invoke->GetKind()` as the index.
+        // However, a non-empty `default` or `case` causes clang++ to produce much
+        // worse code, so we want to limit this check to debug builds only.
+        DCHECK(false) << "UNREACHABLE";
+        UNREACHABLE();
+    }
+  }
+
  protected:
   void VisitPhis(HBasicBlock* block);
   void VisitNonPhiInstructions(HBasicBlock* block);
@@ -8436,7 +8458,7 @@
       : HGraphVisitor(graph, stats) {}
   virtual ~HGraphDelegateVisitor() {}
 
-  // Visit functions that delegate to to super class.
+  // Visit functions that delegate to super class.
 #define DECLARE_VISIT_ABSTRACT_INSTRUCTION(name, super)               \
   virtual void Visit##name(H##name* instr) { Visit##super(instr); }
   FOR_EACH_ABSTRACT_INSTRUCTION(DECLARE_VISIT_ABSTRACT_INSTRUCTION)

diff --git a/compiler/optimizing/prepare_for_register_allocation.cc b/compiler/optimizing/prepare_for_register_allocation.cc
index cc9ba6e..8a9e09a 100644
--- a/compiler/optimizing/prepare_for_register_allocation.cc
+++ b/compiler/optimizing/prepare_for_register_allocation.cc

@@ -63,7 +63,7 @@
     // No need to visit the phis.
     for (HInstructionIterator inst_it(block->GetInstructions()); !inst_it.Done();
          inst_it.Advance()) {
-      inst_it.Current()->Accept(&visitor);
+      visitor.Dispatch(inst_it.Current());
     }
   }
   return true;

diff --git a/compiler/optimizing/reference_type_propagation.cc b/compiler/optimizing/reference_type_propagation.cc
index 5414345..3b3324b 100644
--- a/compiler/optimizing/reference_type_propagation.cc
+++ b/compiler/optimizing/reference_type_propagation.cc

@@ -124,7 +124,7 @@
 
 void ReferenceTypePropagation::Visit(HInstruction* instruction) {
   RTPVisitor visitor(graph_, hint_dex_cache_, is_first_run_);
-  instruction->Accept(&visitor);
+  visitor.Dispatch(instruction);
 }
 
 void ReferenceTypePropagation::Visit(ArrayRef<HInstruction* const> instructions) {
@@ -136,7 +136,7 @@
     }
   }
   for (HInstruction* instruction : instructions) {
-    instruction->Accept(&visitor);
+    visitor.Dispatch(instruction);
     // We don't know if the instruction list is ordered in the same way normal
     // visiting would be so we need to process every instruction manually.
     if (RTPVisitor::IsUpdateable(instruction)) {

diff --git a/compiler/optimizing/scheduler.h b/compiler/optimizing/scheduler.h
index 324e1dd..f781190 100644
--- a/compiler/optimizing/scheduler.h
+++ b/compiler/optimizing/scheduler.h

@@ -414,14 +414,10 @@
     UNREACHABLE();
   }
 
-  void Visit(HInstruction* instruction) {
-    instruction->Accept(this);
-  }
-
   void CalculateLatency(SchedulingNode* node) {
     // By default nodes have no internal latency.
     last_visited_internal_latency_ = 0;
-    Visit(node->GetInstruction());
+    Dispatch(node->GetInstruction());
   }
 
   uint32_t GetLastVisitedLatency() const { return last_visited_latency_; }