[CodeGen] Match LLVM Legalizer ownership

14 files changed, 2094 insertions, 3221 deletions

diff --git a/backend/lib/CodeGen/GenericOpcodes.cpp b/backend/lib/CodeGen/GenericOpcodes.cpp
index f3ff5f1..ef51147 100644
--- a/backend/lib/CodeGen/GenericOpcodes.cpp
+++ b/backend/lib/CodeGen/GenericOpcodes.cpp
@@ -451,7 +451,13 @@ private:
     if (Predicate == nullptr || ((Generic == GO::G_ICMP) != Predicate->isInteger())) {
       return semanticError("generic comparison predicate kind does not match its opcode");
     }
-    return requireI1WithShape(TypeAt(0), TypeAt(2), false);
+    const LowLevelType Result = TypeAt(0);
+    const LowLevelType Compared = TypeAt(2);
+    if (!Compared.isVector())
+      return Result.isCategoryNeutralScalar()
+                 ? Expected<void>{}
+                 : semanticError("scalar generic comparison requires a scalar integer result");
+    return requireI1WithShape(Result, Compared, false);
   }
   case GO::G_SCMP:
   case GO::G_UCMP:
diff --git a/backend/lib/CodeGen/GlobalISel/CMakeLists.txt b/backend/lib/CodeGen/GlobalISel/CMakeLists.txt
index 8a37497..b7064e3 100644
--- a/backend/lib/CodeGen/GlobalISel/CMakeLists.txt
+++ b/backend/lib/CodeGen/GlobalISel/CMakeLists.txt
@@ -2,16 +2,24 @@
 
 set(
   RCCBackendGlobalISelSources
+  GISelChangeObserver.cpp
   InstructionSelect.cpp
   Legalizer.cpp
+  LegalizerHelper.cpp
+  LegalizerInfo.cpp
   RegBankSelect.cpp
 )
 
 set(
   RCCBackendGlobalISelHeaders
+  "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/GlobalISel/GISelChangeObserver.h"
+  "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/GlobalISel/GISelWorkList.h"
+  "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/GlobalISel/GenericMachineInstrs.h"
   "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/GlobalISel/InstructionSelect.h"
   "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/GlobalISel/InstructionSelector.h"
+  "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/GlobalISel/LegalizationArtifactCombiner.h"
   "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/GlobalISel/Legalizer.h"
+  "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/GlobalISel/LegalizerHelper.h"
   "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/GlobalISel/LegalizerInfo.h"
   "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/GlobalISel/RegBankSelect.h"
 )
@@ -19,14 +27,20 @@ set(
 target_sources(RCCBackendCodeGen PRIVATE ${RCCBackendGlobalISelSources})
 
 rcc_backend_register_format_files(
+  "${CMAKE_CURRENT_SOURCE_DIR}/GISelChangeObserver.cpp"
   "${CMAKE_CURRENT_SOURCE_DIR}/InstructionSelect.cpp"
   "${CMAKE_CURRENT_SOURCE_DIR}/Legalizer.cpp"
+  "${CMAKE_CURRENT_SOURCE_DIR}/LegalizerHelper.cpp"
+  "${CMAKE_CURRENT_SOURCE_DIR}/LegalizerInfo.cpp"
   "${CMAKE_CURRENT_SOURCE_DIR}/RegBankSelect.cpp"
   ${RCCBackendGlobalISelHeaders}
 )
 
 rcc_backend_register_tidy_files(
+  "${CMAKE_CURRENT_SOURCE_DIR}/GISelChangeObserver.cpp"
   "${CMAKE_CURRENT_SOURCE_DIR}/InstructionSelect.cpp"
   "${CMAKE_CURRENT_SOURCE_DIR}/Legalizer.cpp"
+  "${CMAKE_CURRENT_SOURCE_DIR}/LegalizerHelper.cpp"
+  "${CMAKE_CURRENT_SOURCE_DIR}/LegalizerInfo.cpp"
   "${CMAKE_CURRENT_SOURCE_DIR}/RegBankSelect.cpp"
 )
diff --git a/backend/lib/CodeGen/GlobalISel/GISelChangeObserver.cpp b/backend/lib/CodeGen/GlobalISel/GISelChangeObserver.cpp
new file mode 100644
index 0000000..4ce1473
--- /dev/null
+++ b/backend/lib/CodeGen/GlobalISel/GISelChangeObserver.cpp
@@ -0,0 +1,61 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "rcc/CodeGen/GlobalISel/GISelChangeObserver.h"
+
+#include <algorithm>
+
+namespace rcc {
+
+void GISelObserverWrapper::addObserver(GISelChangeObserver &Observer) {
+  Observers.push_back(&Observer);
+}
+
+void GISelObserverWrapper::removeObserver(GISelChangeObserver &Observer) {
+  const auto Position = std::ranges::find(Observers, &Observer);
+  if (Position != Observers.end())
+    Observers.erase(Position);
+}
+
+void GISelObserverWrapper::clearObservers() noexcept {
+  Observers.clear();
+}
+
+void GISelObserverWrapper::erasingInstr(MachineInstr &Instruction) {
+  for (GISelChangeObserver *Observer : Observers)
+    Observer->erasingInstr(Instruction);
+}
+
+void GISelObserverWrapper::createdInstr(MachineInstr &Instruction) {
+  for (GISelChangeObserver *Observer : Observers)
+    Observer->createdInstr(Instruction);
+}
+
+void GISelObserverWrapper::changingInstr(MachineInstr &Instruction) {
+  for (GISelChangeObserver *Observer : Observers)
+    Observer->changingInstr(Instruction);
+}
+
+void GISelObserverWrapper::changedInstr(MachineInstr &Instruction) {
+  for (GISelChangeObserver *Observer : Observers)
+    Observer->changedInstr(Instruction);
+}
+
+void GISelObserverWrapper::handleInsertion(MachineInstr &Instruction) {
+  createdInstr(Instruction);
+}
+
+void GISelObserverWrapper::handleRemoval(MachineInstr &Instruction) {
+  erasingInstr(Instruction);
+}
+
+MachineFunctionDelegateInstaller::MachineFunctionDelegateInstaller(
+    MachineFunction &Function, MachineFunction::Delegate &Delegate)
+    : Function(&Function), Delegate(&Delegate) {
+  Function.setDelegate(Delegate);
+}
+
+MachineFunctionDelegateInstaller::~MachineFunctionDelegateInstaller() {
+  Function->resetDelegate(*Delegate);
+}
+
+} // namespace rcc
diff --git a/backend/lib/CodeGen/GlobalISel/Legalizer.cpp b/backend/lib/CodeGen/GlobalISel/Legalizer.cpp
index 83efb6a..d971b42 100644
--- a/backend/lib/CodeGen/GlobalISel/Legalizer.cpp
+++ b/backend/lib/CodeGen/GlobalISel/Legalizer.cpp
@@ -2,6 +2,11 @@
 
 #include "rcc/CodeGen/GlobalISel/Legalizer.h"
 
+#include "rcc/CodeGen/GenericOpcodes.h"
+#include "rcc/CodeGen/GlobalISel/GISelChangeObserver.h"
+#include "rcc/CodeGen/GlobalISel/GISelWorkList.h"
+#include "rcc/CodeGen/GlobalISel/LegalizationArtifactCombiner.h"
+#include "rcc/CodeGen/GlobalISel/LegalizerHelper.h"
 #include "rcc/CodeGen/LiveVariables.h"
 #include "rcc/CodeGen/MachineBasicBlock.h"
 #include "rcc/CodeGen/MachineFunction.h"
@@ -10,8 +15,10 @@
 #include "rcc/CodeGen/MachineRegisterInfo.h"
 #include "rcc/CodeGen/TargetDescription.h"
 
-#include <cstdint>
+#include <algorithm>
+#include <cstddef>
 #include <optional>
+#include <ranges>
 #include <string>
 #include <unordered_set>
 #include <utility>
@@ -34,60 +41,122 @@ namespace {
   return Contract;
 }
 
-struct OpcodeState final {
-  MachineInstrStableID Instruction;
-  Opcode OpcodeValue;
-  std::uint64_t OperandFingerprint = 0;
+[[nodiscard]] std::vector<MachineInstr *> layoutInstructions(MachineFunction &Function) {
+  std::vector<MachineInstr *> Result;
+  for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next())
+    for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr;
+         Instruction = Instruction->next())
+      Result.push_back(Instruction);
+  return Result;
+}
 
-  friend bool operator==(const OpcodeState &Left, const OpcodeState &Right) noexcept = default;
-};
+[[nodiscard]] bool isArtifact(const MachineInstr &Instruction) noexcept {
+  switch (static_cast<GenericOpcode>(Instruction.opcode().value())) {
+  default:
+    return false;
+  case GenericOpcode::G_TRUNC:
+  case GenericOpcode::G_ZEXT:
+  case GenericOpcode::G_ANYEXT:
+  case GenericOpcode::G_SEXT:
+  case GenericOpcode::G_MERGE_VALUES:
+  case GenericOpcode::G_UNMERGE_VALUES:
+  case GenericOpcode::G_CONCAT_VECTORS:
+  case GenericOpcode::G_BUILD_VECTOR:
+  case GenericOpcode::G_EXTRACT:
+  case GenericOpcode::G_INSERT:
+    return true;
+  }
+}
+
+using InstList = GISelWorkList<256>;
+using ArtifactList = GISelWorkList<128>;
+
+class LegalizerWorkListManager final : public GISelChangeObserver {
+public:
+  LegalizerWorkListManager(InstList &Instructions, ArtifactList &Artifacts)
+      : Instructions(Instructions), Artifacts(Artifacts) {}
 
-struct OpcodeStateHash final {
-  [[nodiscard]] std::size_t operator()(const OpcodeState &State) const noexcept {
-    return std::hash<std::uint64_t>{}(State.Instruction.value()) ^
-           (std::hash<std::uint32_t>{}(State.OpcodeValue.value()) << 1U) ^
-           (std::hash<std::uint64_t>{}(State.OperandFingerprint) << 2U);
+  void erasingInstr(MachineInstr &Instruction) override {
+    Instructions.remove(&Instruction);
+    Artifacts.remove(&Instruction);
   }
-};
 
-void combineFingerprint(std::uint64_t &State, std::uint64_t Value) noexcept {
-  State ^= Value + 0x9e3779b97f4a7c15ULL + (State << 6U) + (State >> 2U);
-}
+  void createdInstr(MachineInstr &Instruction) override {
+    createdOrChangedInstr(Instruction);
+  }
+
+  void changingInstr(MachineInstr &) override {}
 
-[[nodiscard]] std::uint64_t operandFingerprint(const MachineInstr &Instruction) noexcept {
-  std::uint64_t Result = 0xcbf29ce484222325ULL;
-  combineFingerprint(Result, Instruction.flags().rawValue());
-  combineFingerprint(Result, Instruction.operandCount());
-  for (const MachineOperand &Operand : Instruction.operands()) {
-    combineFingerprint(Result, static_cast<std::uint64_t>(Operand.kind()));
-    combineFingerprint(Result, Operand.targetFlags());
-    combineFingerprint(Result, Operand.registerFlags().rawValue());
-    combineFingerprint(Result, Operand.subReg().value());
-    if (const Register *Reg = Operand.getIf<Register>())
-      combineFingerprint(Result, Reg->opaqueValue());
-    else if (const std::int64_t *Immediate = Operand.getIf<std::int64_t>())
-      combineFingerprint(Result, static_cast<std::uint64_t>(*Immediate));
-    else if (const auto *Integer = Operand.getIf<IntegerImmediateOperand>())
-      combineFingerprint(Result, Integer->Value.bitWidth());
-    else if (const auto *Floating = Operand.getIf<FloatingImmediateOperand>())
-      combineFingerprint(Result, static_cast<std::uint64_t>(Floating->Bits.semantics()));
-    else if (const auto *Block = Operand.getIf<BasicBlockOperand>())
-      combineFingerprint(Result, Block->Block == nullptr ? 0 : Block->Block->stableID().value());
+  void changedInstr(MachineInstr &Instruction) override {
+    createdOrChangedInstr(Instruction);
   }
-  return Result;
-}
 
-[[nodiscard]] OpcodeState stateOf(const MachineInstr &Instruction) noexcept {
-  return {Instruction.stableID(), Instruction.opcode(), operandFingerprint(Instruction)};
-}
+private:
+  void createdOrChangedInstr(MachineInstr &Instruction) {
+    if (!isGenericOpcode(Instruction.opcode()))
+      return;
+    if (isArtifact(Instruction))
+      Artifacts.insert(&Instruction);
+    else
+      Instructions.insert(&Instruction);
+  }
 
-[[nodiscard]] std::vector<MachineInstr *> layoutInstructions(MachineFunction &Function) {
-  std::vector<MachineInstr *> Result;
+  InstList &Instructions;
+  ArtifactList &Artifacts;
+};
+
+[[nodiscard]] std::vector<MachineBasicBlock *> reversePostOrderBlocks(MachineFunction &Function) {
+  struct TraversalFrame final {
+    MachineBasicBlock *Block = nullptr;
+    std::vector<MachineBasicBlock *> Successors;
+    std::size_t NextSuccessor = 0;
+  };
+
+  const auto makeFrame = [](MachineBasicBlock &Block) {
+    TraversalFrame Frame;
+    Frame.Block = &Block;
+    for (const MachineSuccessor &Successor : Block.successors()) {
+      if (Successor.Block != nullptr)
+        Frame.Successors.push_back(Successor.Block);
+    }
+    std::ranges::sort(Frame.Successors, {}, [](const MachineBasicBlock *Successor) {
+      return Successor->stableID().value();
+    });
+    return Frame;
+  };
+
+  std::vector<MachineBasicBlock *> Roots;
   for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next())
-    for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr;
-         Instruction = Instruction->next())
-      Result.push_back(Instruction);
-  return Result;
+    Roots.push_back(Block);
+  std::ranges::sort(Roots, {},
+                    [](const MachineBasicBlock *Block) { return Block->stableID().value(); });
+  if (MachineBasicBlock *Entry = Function.firstBlock(); Entry != nullptr) {
+    const auto Position = std::ranges::find(Roots, Entry);
+    if (Position != Roots.end())
+      std::rotate(Roots.begin(), Position, std::next(Position));
+  }
+
+  std::vector<MachineBasicBlock *> PostOrder;
+  std::unordered_set<MachineBasicBlock *> Visited;
+  std::vector<TraversalFrame> Stack;
+  for (MachineBasicBlock *Root : Roots) {
+    if (!Visited.insert(Root).second)
+      continue;
+    Stack.push_back(makeFrame(*Root));
+    while (!Stack.empty()) {
+      TraversalFrame &Frame = Stack.back();
+      if (Frame.NextSuccessor < Frame.Successors.size()) {
+        MachineBasicBlock *Successor = Frame.Successors[Frame.NextSuccessor++];
+        if (Visited.insert(Successor).second)
+          Stack.push_back(makeFrame(*Successor));
+        continue;
+      }
+      PostOrder.push_back(Frame.Block);
+      Stack.pop_back();
+    }
+  }
+  std::ranges::reverse(PostOrder);
+  return PostOrder;
 }
 
 [[nodiscard]] Expected<void> requireProperties(const MachineFunction &Function,
@@ -202,91 +271,136 @@ Expected<MachinePassResult> Legalizer::run(MachineFunction &Function,
 Expected<MachinePassResult> Legalizer::runImplementation(MachineFunction &Function,
                                                          const TargetDescription &Target,
                                                          MachineActionReplay &Actions) {
-
   MachinePassResult Result;
-  std::unordered_set<OpcodeState, OpcodeStateHash> SeenStates;
-
-  std::vector<MachineInstr *> Worklist = layoutInstructions(Function);
-  std::unordered_set<MachineInstrStableID> PendingInstructions;
-  for (const MachineInstr *Instruction : Worklist)
-    PendingInstructions.insert(Instruction->stableID());
-  for (std::size_t Index = 0; Index < Worklist.size(); ++Index) {
-    MachineInstr *Instruction = Worklist[Index];
-    if (Instruction != nullptr)
-      PendingInstructions.erase(Instruction->stableID());
-    if (Instruction == nullptr || Instruction->parent() == nullptr)
-      continue;
+  InstList InstructionWorklist;
+  ArtifactList ArtifactWorklist;
+  std::vector<MachineInstr *> RetryList;
 
-    const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode());
-    if (Descriptor == nullptr)
-      return std::unexpected(
-          passFailure("legalizer encountered an unknown instruction descriptor"));
-    if (!Descriptor->Flags.has(InstructionFlag::Generic))
-      continue;
+  for (MachineBasicBlock *Block : reversePostOrderBlocks(Function)) {
+    for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr;
+         Instruction = Instruction->next()) {
+      if (!isGenericOpcode(Instruction->opcode()))
+        continue;
+      if (isArtifact(*Instruction))
+        ArtifactWorklist.deferredInsert(Instruction);
+      else
+        InstructionWorklist.deferredInsert(Instruction);
+    }
+  }
+  ArtifactWorklist.finalize();
+  InstructionWorklist.finalize();
 
-    auto Validation = Target.validateInstruction(*Instruction, Function.registerInfo());
-    if (!Validation)
-      return std::unexpected(withErrorContext(Validation.error(), "legalizer input is invalid: "));
+  LegalizerWorkListManager WorkListObserver(InstructionWorklist, ArtifactWorklist);
+  GISelObserverWrapper Observer(WorkListObserver);
+  MachineFunctionDelegateInstaller ObserverInstaller(Function, Observer);
+  MachineIRBuilder Builder(Function, Target);
+  LegalizerHelper Helper(Function, *Info, Observer, Builder);
+  LegalizationArtifactCombiner ArtifactCombiner(Builder, Function.registerInfo(), *Info);
 
-    auto Action = Info->classify(*Instruction, Function, Target);
-    if (!Action)
-      return std::unexpected(Action.error());
-    if (*Action == LegalizationAction::Legal)
-      continue;
-    if (*Action == LegalizationAction::Unsupported) {
-      return std::unexpected(
-          passFailure("target legalizer rejected generic instruction '" + Descriptor->Name + "'",
-                      ErrorCode::Unsupported));
-    }
+  do {
+    while (!InstructionWorklist.empty()) {
+      MachineInstr *Instruction = InstructionWorklist.popBack();
+      if (Instruction == nullptr || Instruction->parent() == nullptr)
+        continue;
 
-    const OpcodeState Before = stateOf(*Instruction);
-    if (!SeenStates.insert(Before).second)
-      return std::unexpected(
-          passFailure("target legalizer entered a repeated state for '" + Descriptor->Name + "'",
-                      ErrorCode::InternalError));
+      const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode());
+      if (Descriptor == nullptr)
+        return std::unexpected(
+            passFailure("legalizer encountered an unknown instruction descriptor"));
+      if (!Descriptor->Flags.has(InstructionFlag::Generic))
+        continue;
 
-    auto LoweringValidation = Info->validateLowering(*Instruction, Function, Target, *Action);
-    if (!LoweringValidation)
-      return std::unexpected(
-          withErrorContext(LoweringValidation.error(), "legalizer lowering precondition failed: "));
-    std::string Rule = "legalize.";
-    Rule.append(legalizationActionName(*Action));
-    Rule.push_back('.');
-    Rule.append(Descriptor->Name);
-    auto Accepted = Actions.request(Rule, Function, Instruction->stableID());
-    if (!Accepted)
-      return std::unexpected(Accepted.error());
-    if (!*Accepted) {
-      return std::unexpected(
-          passFailure("required legalizer action was rejected for '" + Descriptor->Name + "'",
-                      ErrorCode::VerificationError));
+      auto Validation = Target.validateInstruction(*Instruction, Function.registerInfo());
+      if (!Validation) {
+        return std::unexpected(
+            withErrorContext(Validation.error(), "legalizer input is invalid: "));
+      }
+
+      auto Decision = Info->getDecision(*Instruction, Function, Target);
+      if (!Decision)
+        return std::unexpected(Decision.error());
+      if (Decision->Action == LegalizationAction::Legal)
+        continue;
+      if (Decision->Action == LegalizationAction::Unsupported) {
+        if (isArtifact(*Instruction)) {
+          RetryList.push_back(Instruction);
+          continue;
+        }
+        return std::unexpected(
+            passFailure("target legalizer rejected generic instruction '" + Descriptor->Name + "'",
+                        ErrorCode::Unsupported));
+      }
+
+      auto LoweringValidation = Info->validateLowering(*Instruction, Function, Target, *Decision);
+      if (!LoweringValidation) {
+        return std::unexpected(withErrorContext(LoweringValidation.error(),
+                                                "legalizer lowering precondition failed: "));
+      }
+      std::string Rule = "legalize.";
+      Rule.append(legalizationActionName(Decision->Action));
+      Rule.push_back('.');
+      Rule.append(Descriptor->Name);
+      auto Accepted = Actions.request(Rule, Function, Instruction->stableID());
+      if (!Accepted)
+        return std::unexpected(Accepted.error());
+      if (!*Accepted) {
+        return std::unexpected(
+            passFailure("required legalizer action was rejected for '" + Descriptor->Name + "'",
+                        ErrorCode::VerificationError));
+      }
+      Expected<void> Lowered = Helper.legalize(*Instruction, *Decision);
+      if (!Lowered) {
+        if (Lowered.error().code() == ErrorCode::Unsupported && Instruction->parent() != nullptr &&
+            isArtifact(*Instruction)) {
+          RetryList.push_back(Instruction);
+          continue;
+        }
+        return std::unexpected(Lowered.error());
+      }
+      Result.Changed = true;
+      Result.Changes.add(MachinePassChange::Instructions);
     }
-    auto Lowered = Info->lower(*Instruction, Function, Target, *Action);
-    if (!Lowered)
-      return std::unexpected(Lowered.error());
-    const bool InstructionWasErased = Instruction->parent() == nullptr;
-    if (!InstructionWasErased) {
-      const OpcodeState After = stateOf(*Instruction);
-      if (After == Before)
-        return std::unexpected(passFailure("target legalizer made no observable progress for '" +
-                                               Descriptor->Name + "'",
-                                           ErrorCode::InternalError));
+
+    if (!RetryList.empty()) {
+      if (!ArtifactWorklist.empty()) {
+        for (MachineInstr *Instruction : RetryList)
+          ArtifactWorklist.insert(Instruction);
+        RetryList.clear();
+      } else {
+        MachineInstr *Instruction = RetryList.front();
+        const InstructionDescriptor *Descriptor =
+            Instruction == nullptr ? nullptr : Target.instructionDescriptor(Instruction->opcode());
+        return std::unexpected(passFailure(
+            "target legalizer rejected generic artifact '" +
+                std::string(Descriptor == nullptr ? "<unknown>" : Descriptor->Name) + "'",
+            ErrorCode::Unsupported));
+      }
     }
-    Result.Changed = true;
-    Result.Changes.add(MachinePassChange::Instructions);
-
-    const std::vector<MachineInstr *> Current = layoutInstructions(Function);
-    for (MachineInstr *Candidate : Current) {
-      const InstructionDescriptor *CandidateDescriptor =
-          Target.instructionDescriptor(Candidate->opcode());
-      if (CandidateDescriptor == nullptr)
-        return std::unexpected(passFailure("legalizer produced an unknown instruction descriptor"));
-      if (CandidateDescriptor->Flags.has(InstructionFlag::Generic) &&
-          PendingInstructions.insert(Candidate->stableID()).second) {
-        Worklist.push_back(Candidate);
+
+    while (!ArtifactWorklist.empty()) {
+      MachineInstr *Instruction = ArtifactWorklist.popBack();
+      if (Instruction == nullptr || Instruction->parent() == nullptr)
+        continue;
+      std::vector<MachineInstr *> DeadInstructions;
+      auto Combined =
+          ArtifactCombiner.tryCombineInstruction(*Instruction, DeadInstructions, Observer);
+      if (!Combined)
+        return std::unexpected(Combined.error());
+      if (*Combined) {
+        for (MachineInstr *Dead : DeadInstructions) {
+          if (Dead == nullptr || Dead->parent() == nullptr)
+            continue;
+          auto Removed = Dead->parent()->removeInstruction(*Dead);
+          if (!Removed)
+            return std::unexpected(Removed.error());
+        }
+        Result.Changed = true;
+        Result.Changes.add(MachinePassChange::Instructions);
+        continue;
       }
+      InstructionWorklist.insert(Instruction);
     }
-  }
+  } while (!InstructionWorklist.empty());
 
   for (MachineInstr *Instruction : layoutInstructions(Function)) {
     const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode());
@@ -294,15 +408,16 @@ Expected<MachinePassResult> Legalizer::runImplementation(MachineFunction &Functi
       return std::unexpected(passFailure("legalizer produced an unknown instruction descriptor"));
     if (!Descriptor->Flags.has(InstructionFlag::Generic))
       continue;
-    auto Action = Info->classify(*Instruction, Function, Target);
-    if (!Action)
-      return std::unexpected(Action.error());
-    if (*Action != LegalizationAction::Legal)
-      return std::unexpected(
-          passFailure("legalizer reached a fixed point with an illegal generic instruction '" +
-                          Descriptor->Name + "'",
-                      *Action == LegalizationAction::Unsupported ? ErrorCode::Unsupported
-                                                                 : ErrorCode::InternalError));
+    auto Decision = Info->getDecision(*Instruction, Function, Target);
+    if (!Decision)
+      return std::unexpected(Decision.error());
+    if (Decision->Action != LegalizationAction::Legal &&
+        Decision->Action != LegalizationAction::Custom)
+      return std::unexpected(passFailure(
+          "legalizer reached a fixed point with an illegal generic instruction '" +
+              Descriptor->Name + "'",
+          Decision->Action == LegalizationAction::Unsupported ? ErrorCode::Unsupported
+                                                              : ErrorCode::InternalError));
     auto FinalValidation = Target.validateInstruction(*Instruction, Function.registerInfo());
     if (!FinalValidation)
       return std::unexpected(
diff --git a/backend/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/backend/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
new file mode 100644
index 0000000..1e52214
--- /dev/null
+++ b/backend/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -0,0 +1,590 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "rcc/CodeGen/GlobalISel/LegalizerHelper.h"
+
+#include "rcc/CodeGen/GlobalISel/GISelChangeObserver.h"
+#include "rcc/CodeGen/MachineBasicBlock.h"
+#include "rcc/CodeGen/MachineFunction.h"
+#include "rcc/CodeGen/MachineInstr.h"
+#include "rcc/CodeGen/MachineMemOperand.h"
+#include "rcc/CodeGen/MachineOperand.h"
+#include "rcc/CodeGen/MachineRegisterInfo.h"
+
+#include <algorithm>
+#include <bit>
+#include <cstddef>
+#include <cstdint>
+#include <optional>
+#include <utility>
+
+namespace rcc {
+namespace {
+
+struct ScalarLayout final {
+  std::uint64_t StoreSize = 0;
+  std::uint64_t AllocationSize = 0;
+  std::uint64_t ABIAlignment = 0;
+};
+
+[[nodiscard]] Expected<std::int64_t> signedImmediate(const MachineOperand &Operand) {
+  if (const std::int64_t *Value = Operand.getIf<std::int64_t>())
+    return *Value;
+  const auto *Integer = Operand.getIf<IntegerImmediateOperand>();
+  if (Integer == nullptr || Integer->Value.bitWidth() == 0 || Integer->Value.bitWidth() > 64) {
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "legalizer requires an immediate no wider than 64 bits");
+  }
+  std::uint64_t Raw = Integer->Value.words().empty() ? 0 : Integer->Value.words().front();
+  const std::uint32_t Width = Integer->Value.bitWidth();
+  if (Width < 64 && (Raw & (std::uint64_t{1} << (Width - 1U))) != 0)
+    Raw |= ~std::uint64_t{0} << Width;
+  return std::bit_cast<std::int64_t>(Raw);
+}
+
+[[nodiscard]] std::optional<ScalarLayout> scalarLayout(LowLevelType Type) noexcept {
+  if (!Type.isScalarValue() || Type.scalarBitWidth() == 0)
+    return std::nullopt;
+  const std::uint64_t StoreSize = (Type.scalarBitWidth() + 7U) / 8U;
+  const std::uint64_t AllocationSize = std::bit_ceil(StoreSize);
+  return ScalarLayout{StoreSize, AllocationSize, std::min<std::uint64_t>(AllocationSize, 8)};
+}
+
+[[nodiscard]] Expected<const MachineMemOperand *> createMemoryAccess(MachineFunction &Function,
+                                                                     MemoryAccessFlag Access,
+                                                                     LowLevelType Type,
+                                                                     const ScalarLayout &Layout) {
+  auto Size = MemorySize::fixed(Layout.StoreSize);
+  auto Align = Alignment::create(Layout.ABIAlignment);
+  if (!Size || !Align)
+    return makeUnexpected(ErrorCode::InternalError,
+                          "legalizer cannot represent scalar memory layout");
+  MachineMemOperandDesc Description;
+  Description.Accesses = {Access};
+  Description.Size = *Size;
+  Description.MemoryType = Type;
+  Description.Align = *Align;
+  return Function.createMemOperand(std::move(Description));
+}
+
+[[nodiscard]] Expected<Register> buildConstantBefore(MachineIRBuilder &Builder,
+                                                     MachineInstr &Anchor, LowLevelType Type,
+                                                     std::uint64_t Value) {
+  auto Result = Builder.function().registerInfo().createVirtualRegister(Type);
+  if (!Result)
+    return std::unexpected(Result.error());
+  auto Immediate = APInt::fromUnsigned(Type.scalarBitWidth(), Value);
+  if (!Immediate)
+    return std::unexpected(Immediate.error());
+  if (auto Point = Builder.setInsertPointBefore(Anchor); !Point)
+    return std::unexpected(Point.error());
+  auto Built = Builder.buildConstant(*Result, std::move(*Immediate));
+  if (!Built)
+    return std::unexpected(Built.error());
+  return *Result;
+}
+
+[[nodiscard]] Expected<Register> buildBinaryBefore(MachineIRBuilder &Builder, MachineInstr &Anchor,
+                                                   GenericOpcode Generic, LowLevelType Type,
+                                                   Register Left, Register Right) {
+  auto Result = Builder.function().registerInfo().createVirtualRegister(Type);
+  if (!Result)
+    return std::unexpected(Result.error());
+  if (auto Point = Builder.setInsertPointBefore(Anchor); !Point)
+    return std::unexpected(Point.error());
+  auto Pending = Builder.buildInstr(opcode(Generic));
+  if (!Pending)
+    return std::unexpected(Pending.error());
+  if (auto Added = Pending->addDef(*Result); !Added)
+    return std::unexpected(Added.error());
+  if (auto Added = Pending->addUse(Left); !Added)
+    return std::unexpected(Added.error());
+  if (auto Added = Pending->addUse(Right); !Added)
+    return std::unexpected(Added.error());
+  auto Built = Pending->build();
+  if (!Built)
+    return std::unexpected(Built.error());
+  return *Result;
+}
+
+} // namespace
+
+LegalizerHelper::LegalizerHelper(MachineFunction &Function, const LegalizerInfo &Info,
+                                 GISelChangeObserver &Observer, MachineIRBuilder &MIRBuilder)
+    : MIRBuilder(MIRBuilder), Observer(Observer), Function(Function), MRI(Function.registerInfo()),
+      Info(Info) {}
+
+Expected<void> LegalizerHelper::widenScalarSrc(MachineInstr &Instruction, LowLevelType WideType,
+                                               std::size_t OperandIndex,
+                                               GenericOpcode ExtensionOpcode) {
+  MachineOperand *Operand = Instruction.operandAt(OperandIndex);
+  const Register *Source = Operand == nullptr ? nullptr : Operand->getIf<Register>();
+  if (Source == nullptr || !Source->isVirtual() || !Operand->isUse())
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "scalar widening source is not a virtual-register use");
+
+  auto WideSource = MRI.createVirtualRegister(WideType);
+  if (!WideSource)
+    return std::unexpected(WideSource.error());
+  if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point)
+    return std::unexpected(Point.error());
+  auto Extension = MIRBuilder.buildInstr(opcode(ExtensionOpcode));
+  if (!Extension)
+    return std::unexpected(Extension.error());
+  if (auto Added = Extension->addDef(*WideSource); !Added)
+    return std::unexpected(Added.error());
+  if (auto Added = Extension->addUse(*Source); !Added)
+    return std::unexpected(Added.error());
+  if (auto Built = Extension->build(); !Built)
+    return std::unexpected(Built.error());
+  return Operand->setRegister(*WideSource);
+}
+
+Expected<void> LegalizerHelper::widenScalarDst(MachineInstr &Instruction, LowLevelType WideType,
+                                               std::size_t OperandIndex,
+                                               GenericOpcode TruncOpcode) {
+  MachineOperand *Operand = Instruction.operandAt(OperandIndex);
+  const Register *Destination = Operand == nullptr ? nullptr : Operand->getIf<Register>();
+  if (Destination == nullptr || !Destination->isVirtual() || !Operand->isDef())
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "scalar widening destination is not a virtual-register definition");
+
+  auto WideDestination = MRI.createVirtualRegister(WideType);
+  if (!WideDestination)
+    return std::unexpected(WideDestination.error());
+  if (auto Point = MIRBuilder.setInsertPointAfter(Instruction); !Point)
+    return std::unexpected(Point.error());
+  auto Truncation = MIRBuilder.buildInstr(opcode(TruncOpcode));
+  if (!Truncation)
+    return std::unexpected(Truncation.error());
+  if (auto Added = Truncation->addDef(*Destination); !Added)
+    return std::unexpected(Added.error());
+  if (auto Added = Truncation->addUse(*WideDestination); !Added)
+    return std::unexpected(Added.error());
+  if (auto Built = Truncation->build(); !Built)
+    return std::unexpected(Built.error());
+  return Operand->setRegister(*WideDestination);
+}
+
+Expected<void> LegalizerHelper::widenScalar(MachineInstr &Instruction, std::size_t TypeIndex,
+                                            LowLevelType WideType) {
+  auto Query = LegalizerInfo::buildQuery(Instruction, Function, MIRBuilder.target());
+  if (!Query)
+    return std::unexpected(Query.error());
+  if (!WideType.isCategoryNeutralScalar() || TypeIndex >= Query->Types.size() ||
+      !Query->Types[TypeIndex].isCategoryNeutralScalar() ||
+      WideType.scalarBitWidth() <= Query->Types[TypeIndex].scalarBitWidth()) {
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "scalar widening has an invalid type index or target type");
+  }
+
+  const auto widenSource = [&](std::size_t OperandIndex, GenericOpcode ExtensionOpcode) {
+    return widenScalarSrc(Instruction, WideType, OperandIndex, ExtensionOpcode);
+  };
+  const auto widenDestination = [&](std::size_t OperandIndex = 0,
+                                    GenericOpcode TruncOpcode = GenericOpcode::G_TRUNC) {
+    return widenScalarDst(Instruction, WideType, OperandIndex, TruncOpcode);
+  };
+  const auto apply = [](Expected<void> Result) -> Expected<void> {
+    if (!Result)
+      return std::unexpected(Result.error());
+    return {};
+  };
+
+  Observer.changingInstr(Instruction);
+  switch (static_cast<GenericOpcode>(Instruction.opcode().value())) {
+  case GenericOpcode::G_ATOMICRMW_XCHG:
+  case GenericOpcode::G_ATOMICRMW_ADD:
+  case GenericOpcode::G_ATOMICRMW_SUB:
+  case GenericOpcode::G_ATOMICRMW_AND:
+  case GenericOpcode::G_ATOMICRMW_OR:
+  case GenericOpcode::G_ATOMICRMW_XOR:
+  case GenericOpcode::G_ATOMICRMW_MIN:
+  case GenericOpcode::G_ATOMICRMW_MAX:
+  case GenericOpcode::G_ATOMICRMW_UMIN:
+  case GenericOpcode::G_ATOMICRMW_UMAX:
+    if (TypeIndex != 0)
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "atomic RMW value widening has an invalid type index");
+    if (auto Result = apply(widenSource(2, GenericOpcode::G_ANYEXT)); !Result)
+      return Result;
+    if (auto Result = apply(widenDestination()); !Result)
+      return Result;
+    break;
+
+  case GenericOpcode::G_ATOMIC_CMPXCHG:
+    if (TypeIndex != 0)
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "compare-exchange widening has an invalid type index");
+    if (auto Result = apply(widenSource(2, GenericOpcode::G_ANYEXT)); !Result)
+      return Result;
+    if (auto Result = apply(widenSource(3, GenericOpcode::G_ANYEXT)); !Result)
+      return Result;
+    if (auto Result = apply(widenDestination()); !Result)
+      return Result;
+    break;
+
+  case GenericOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS:
+    if (TypeIndex == 0) {
+      if (auto Result = apply(widenSource(3, GenericOpcode::G_ANYEXT)); !Result)
+        return Result;
+      if (auto Result = apply(widenSource(4, GenericOpcode::G_ANYEXT)); !Result)
+        return Result;
+      if (auto Result = apply(widenDestination()); !Result)
+        return Result;
+      break;
+    }
+    if (TypeIndex != 1)
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "compare-exchange success widening has an invalid type index");
+    if (auto Result = apply(widenDestination(1)); !Result)
+      return Result;
+    break;
+
+  case GenericOpcode::G_ADD:
+  case GenericOpcode::G_SUB:
+  case GenericOpcode::G_MUL:
+  case GenericOpcode::G_AND:
+  case GenericOpcode::G_OR:
+  case GenericOpcode::G_XOR:
+    if (TypeIndex != 0)
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "binary operation widening has an invalid type index");
+    if (auto Result = apply(widenSource(1, GenericOpcode::G_ANYEXT)); !Result)
+      return Result;
+    if (auto Result = apply(widenSource(2, GenericOpcode::G_ANYEXT)); !Result)
+      return Result;
+    if (auto Result = apply(widenDestination()); !Result)
+      return Result;
+    break;
+
+  case GenericOpcode::G_SDIV:
+  case GenericOpcode::G_SREM:
+  case GenericOpcode::G_SMIN:
+  case GenericOpcode::G_SMAX:
+  case GenericOpcode::G_ABDS:
+    if (TypeIndex != 0)
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "signed operation widening has an invalid type index");
+    if (auto Result = apply(widenSource(1, GenericOpcode::G_SEXT)); !Result)
+      return Result;
+    if (auto Result = apply(widenSource(2, GenericOpcode::G_SEXT)); !Result)
+      return Result;
+    if (auto Result = apply(widenDestination()); !Result)
+      return Result;
+    break;
+
+  case GenericOpcode::G_UDIV:
+  case GenericOpcode::G_UREM:
+  case GenericOpcode::G_UMIN:
+  case GenericOpcode::G_UMAX:
+  case GenericOpcode::G_ABDU:
+    if (TypeIndex != 0)
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "unsigned operation widening has an invalid type index");
+    if (auto Result = apply(widenSource(1, GenericOpcode::G_ZEXT)); !Result)
+      return Result;
+    if (auto Result = apply(widenSource(2, GenericOpcode::G_ZEXT)); !Result)
+      return Result;
+    if (auto Result = apply(widenDestination()); !Result)
+      return Result;
+    break;
+
+  case GenericOpcode::G_SHL:
+  case GenericOpcode::G_LSHR:
+  case GenericOpcode::G_ASHR:
+    if (TypeIndex == 0) {
+      GenericOpcode ExtensionOpcode = GenericOpcode::G_ANYEXT;
+      if (Instruction.opcode() == opcode(GenericOpcode::G_LSHR))
+        ExtensionOpcode = GenericOpcode::G_ZEXT;
+      else if (Instruction.opcode() == opcode(GenericOpcode::G_ASHR))
+        ExtensionOpcode = GenericOpcode::G_SEXT;
+      if (auto Result = apply(widenSource(1, ExtensionOpcode)); !Result)
+        return Result;
+      if (auto Result = apply(widenDestination()); !Result)
+        return Result;
+      break;
+    }
+    if (TypeIndex != 1)
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "shift widening has an invalid type index");
+    if (auto Result = apply(widenSource(2, GenericOpcode::G_ZEXT)); !Result)
+      return Result;
+    break;
+
+  case GenericOpcode::G_ROTL:
+  case GenericOpcode::G_ROTR:
+    if (TypeIndex != 1)
+      return makeUnexpected(ErrorCode::Unsupported, "rotate value widening is not defined");
+    if (auto Result = apply(widenSource(2, GenericOpcode::G_ZEXT)); !Result)
+      return Result;
+    break;
+
+  case GenericOpcode::G_FREEZE:
+  case GenericOpcode::G_CONSTANT_FOLD_BARRIER:
+    if (TypeIndex != 0)
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "unary operation widening has an invalid type index");
+    if (auto Result = apply(widenSource(1, GenericOpcode::G_ANYEXT)); !Result)
+      return Result;
+    if (auto Result = apply(widenDestination()); !Result)
+      return Result;
+    break;
+
+  case GenericOpcode::G_ABS:
+    if (TypeIndex != 0)
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "absolute-value widening has an invalid type index");
+    if (auto Result = apply(widenSource(1, GenericOpcode::G_SEXT)); !Result)
+      return Result;
+    if (auto Result = apply(widenDestination()); !Result)
+      return Result;
+    break;
+
+  case GenericOpcode::G_SELECT:
+    if (TypeIndex == 0) {
+      if (auto Result = apply(widenSource(2, GenericOpcode::G_ANYEXT)); !Result)
+        return Result;
+      if (auto Result = apply(widenSource(3, GenericOpcode::G_ANYEXT)); !Result)
+        return Result;
+      if (auto Result = apply(widenDestination()); !Result)
+        return Result;
+      break;
+    }
+    if (TypeIndex != 1)
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "select widening has an invalid type index");
+    if (auto Result = apply(widenSource(1, GenericOpcode::G_ZEXT)); !Result)
+      return Result;
+    break;
+
+  case GenericOpcode::G_FPEXT:
+    if (TypeIndex != 1)
+      return makeUnexpected(ErrorCode::Unsupported,
+                            "floating-point extension destination widening is not defined");
+    if (auto Result = apply(widenSource(1, GenericOpcode::G_FPEXT)); !Result)
+      return Result;
+    break;
+
+  case GenericOpcode::G_FPTOSI:
+  case GenericOpcode::G_FPTOUI:
+  case GenericOpcode::G_INTRINSIC_LRINT:
+  case GenericOpcode::G_INTRINSIC_LLRINT:
+  case GenericOpcode::G_IS_FPCLASS:
+    if (TypeIndex == 0) {
+      if (auto Result = apply(widenDestination()); !Result)
+        return Result;
+      break;
+    }
+    if (TypeIndex != 1)
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "floating-to-integer widening has an invalid type index");
+    if (auto Result = apply(widenSource(1, GenericOpcode::G_FPEXT)); !Result)
+      return Result;
+    break;
+
+  case GenericOpcode::G_SITOFP:
+  case GenericOpcode::G_UITOFP:
+    if (TypeIndex == 0) {
+      if (auto Result = apply(widenDestination(0, GenericOpcode::G_FPTRUNC)); !Result)
+        return Result;
+      break;
+    }
+    if (TypeIndex != 1)
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "integer-to-floating widening has an invalid type index");
+    if (auto Result = apply(widenSource(1, Instruction.opcode() == opcode(GenericOpcode::G_SITOFP)
+                                               ? GenericOpcode::G_SEXT
+                                               : GenericOpcode::G_ZEXT));
+        !Result)
+      return Result;
+    break;
+
+  default:
+    return makeUnexpected(ErrorCode::Unsupported,
+                          "scalar widening is not implemented for this opcode");
+  }
+
+  Observer.changedInstr(Instruction);
+  return {};
+}
+
+Expected<void> LegalizerHelper::lowerVAArg(MachineInstr &Instruction) {
+  if (Instruction.opcode() != opcode(GenericOpcode::G_VAARG) || Instruction.operandCount() != 3 ||
+      !Instruction.memOperands().empty()) {
+    return makeUnexpected(ErrorCode::VerificationError, "G_VAARG has invalid operands");
+  }
+
+  MachineOperand *DestinationOperand = Instruction.operandAt(0);
+  MachineOperand *ListOperand = Instruction.operandAt(1);
+  MachineOperand *AlignmentOperand = Instruction.operandAt(2);
+  const Register *Destination =
+      DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+  const Register *ListPointer = ListOperand == nullptr ? nullptr : ListOperand->getIf<Register>();
+  if (Destination == nullptr || ListPointer == nullptr || !Destination->isVirtual() ||
+      !ListPointer->isVirtual() || !DestinationOperand->isDef() || !ListOperand->isUse() ||
+      AlignmentOperand == nullptr) {
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "G_VAARG requires virtual-register operands");
+  }
+
+  auto DestinationInfo = MRI.virtualRegisterInfo(*Destination);
+  auto PointerInfo = MRI.virtualRegisterInfo(*ListPointer);
+  auto RequestedAlignment = signedImmediate(*AlignmentOperand);
+  if (!DestinationInfo || !PointerInfo || !DestinationInfo->get().Type ||
+      !PointerInfo->get().Type || !RequestedAlignment || *RequestedAlignment <= 0 ||
+      !std::has_single_bit(static_cast<std::uint64_t>(*RequestedAlignment))) {
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "G_VAARG has invalid type or alignment information");
+  }
+
+  const LowLevelType DestinationType = *DestinationInfo->get().Type;
+  const LowLevelType PointerType = *PointerInfo->get().Type;
+  const auto DestinationLayout = scalarLayout(DestinationType);
+  const auto PointerLayout = scalarLayout(PointerType);
+  if (!DestinationLayout || !PointerLayout || !PointerType.isPointer()) {
+    return makeUnexpected(ErrorCode::Unsupported,
+                          "G_VAARG requires scalar result and pointer list types");
+  }
+
+  auto ListLoadMemory =
+      createMemoryAccess(Function, MemoryAccessFlag::Load, PointerType, *PointerLayout);
+  auto ListStoreMemory =
+      createMemoryAccess(Function, MemoryAccessFlag::Store, PointerType, *PointerLayout);
+  if (!ListLoadMemory)
+    return std::unexpected(ListLoadMemory.error());
+  if (!ListStoreMemory)
+    return std::unexpected(ListStoreMemory.error());
+
+  auto CurrentList = MRI.createVirtualRegister(PointerType);
+  if (!CurrentList)
+    return std::unexpected(CurrentList.error());
+  if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point)
+    return std::unexpected(Point.error());
+  auto ListLoad = MIRBuilder.buildInstr(opcode(GenericOpcode::G_LOAD));
+  if (!ListLoad)
+    return std::unexpected(ListLoad.error());
+  if (auto Added = ListLoad->addDef(*CurrentList); !Added)
+    return std::unexpected(Added.error());
+  if (auto Added = ListLoad->addUse(*ListPointer); !Added)
+    return std::unexpected(Added.error());
+  if (auto Added = ListLoad->addMemOperand(**ListLoadMemory); !Added)
+    return std::unexpected(Added.error());
+  if (auto Built = ListLoad->build(); !Built)
+    return std::unexpected(Built.error());
+
+  Register ValueAddress = *CurrentList;
+  const std::uint64_t AlignmentValue = static_cast<std::uint64_t>(*RequestedAlignment);
+  auto PointerScalarType = LowLevelType::makeCategoryNeutralScalar(PointerType.scalarBitWidth());
+  if (!PointerScalarType)
+    return std::unexpected(PointerScalarType.error());
+  if (AlignmentValue > 1) {
+    auto AlignmentOffset =
+        buildConstantBefore(MIRBuilder, Instruction, *PointerScalarType, AlignmentValue - 1U);
+    if (!AlignmentOffset)
+      return std::unexpected(AlignmentOffset.error());
+    auto BiasedAddress = buildBinaryBefore(MIRBuilder, Instruction, GenericOpcode::G_PTR_ADD,
+                                           PointerType, ValueAddress, *AlignmentOffset);
+    if (!BiasedAddress)
+      return std::unexpected(BiasedAddress.error());
+    auto AlignmentMask =
+        buildConstantBefore(MIRBuilder, Instruction, *PointerScalarType, ~(AlignmentValue - 1U));
+    if (!AlignmentMask)
+      return std::unexpected(AlignmentMask.error());
+    auto AlignedAddress = buildBinaryBefore(MIRBuilder, Instruction, GenericOpcode::G_PTRMASK,
+                                            PointerType, *BiasedAddress, *AlignmentMask);
+    if (!AlignedAddress)
+      return std::unexpected(AlignedAddress.error());
+    ValueAddress = *AlignedAddress;
+  }
+
+  auto Increment = buildConstantBefore(MIRBuilder, Instruction, *PointerScalarType,
+                                       DestinationLayout->AllocationSize);
+  if (!Increment)
+    return std::unexpected(Increment.error());
+  auto NextList = buildBinaryBefore(MIRBuilder, Instruction, GenericOpcode::G_PTR_ADD, PointerType,
+                                    ValueAddress, *Increment);
+  if (!NextList)
+    return std::unexpected(NextList.error());
+
+  auto ListStore = MIRBuilder.buildInstr(opcode(GenericOpcode::G_STORE));
+  if (!ListStore)
+    return std::unexpected(ListStore.error());
+  if (auto Added = ListStore->addUse(*NextList); !Added)
+    return std::unexpected(Added.error());
+  if (auto Added = ListStore->addUse(*ListPointer); !Added)
+    return std::unexpected(Added.error());
+  if (auto Added = ListStore->addMemOperand(**ListStoreMemory); !Added)
+    return std::unexpected(Added.error());
+  if (auto Built = ListStore->build(); !Built)
+    return std::unexpected(Built.error());
+
+  if (!MRI.useEmpty(*Destination)) {
+    auto ValueLoadMemory =
+        createMemoryAccess(Function, MemoryAccessFlag::Load, DestinationType, *DestinationLayout);
+    if (!ValueLoadMemory)
+      return std::unexpected(ValueLoadMemory.error());
+    auto ValueLoad = MIRBuilder.buildInstr(opcode(GenericOpcode::G_LOAD));
+    if (!ValueLoad)
+      return std::unexpected(ValueLoad.error());
+    if (auto Added = ValueLoad->addDef(*Destination); !Added)
+      return std::unexpected(Added.error());
+    if (auto Added = ValueLoad->addUse(ValueAddress); !Added)
+      return std::unexpected(Added.error());
+    if (auto Added = ValueLoad->addMemOperand(**ValueLoadMemory); !Added)
+      return std::unexpected(Added.error());
+    if (auto Built = ValueLoad->build(); !Built)
+      return std::unexpected(Built.error());
+  }
+
+  MachineBasicBlock *Block = Instruction.parent();
+  if (Block == nullptr)
+    return makeUnexpected(ErrorCode::VerificationError, "G_VAARG is detached during legalization");
+  return Block->removeInstruction(Instruction);
+}
+
+Expected<void> LegalizerHelper::lower(MachineInstr &Instruction,
+                                      const LegalizationDecision &Decision) {
+  if (Decision.Action != LegalizationAction::Lower)
+    return makeUnexpected(ErrorCode::InvalidArgument,
+                          "lower received a non-lower legalization action");
+  switch (static_cast<GenericOpcode>(Instruction.opcode().value())) {
+  case GenericOpcode::G_VAARG:
+    return lowerVAArg(Instruction);
+  default: {
+    const InstructionDescriptor *Descriptor =
+        MIRBuilder.target().instructionDescriptor(Instruction.opcode());
+    const std::string_view Name =
+        Descriptor == nullptr ? std::string_view{"<unknown>"} : Descriptor->Name;
+    return makeUnexpected(ErrorCode::Unsupported,
+                          "generic lowering is not implemented for opcode '" + std::string(Name) +
+                              "'");
+  }
+  }
+}
+
+Expected<void> LegalizerHelper::legalize(MachineInstr &Instruction,
+                                         const LegalizationDecision &Decision) {
+  switch (Decision.Action) {
+  case LegalizationAction::WidenScalar:
+    return widenScalar(Instruction, Decision.TypeIndex, Decision.NewType);
+  case LegalizationAction::Custom:
+    return Info.legalizeCustom(*this, Instruction, Decision);
+  case LegalizationAction::Lower:
+    return lower(Instruction, Decision);
+  case LegalizationAction::Legal:
+    return {};
+  case LegalizationAction::Unsupported:
+  case LegalizationAction::Libcall:
+  case LegalizationAction::NarrowScalar:
+  case LegalizationAction::MoreElements:
+  case LegalizationAction::FewerElements:
+  case LegalizationAction::Bitcast:
+  case LegalizationAction::ChangeElementType:
+  case LegalizationAction::ChangeAddressSpace:
+    return makeUnexpected(ErrorCode::Unsupported,
+                          "legalizer helper does not implement the requested action");
+  }
+  return makeUnexpected(ErrorCode::InternalError, "unknown legalization action");
+}
+
+} // namespace rcc
diff --git a/backend/lib/CodeGen/GlobalISel/LegalizerInfo.cpp b/backend/lib/CodeGen/GlobalISel/LegalizerInfo.cpp
new file mode 100644
index 0000000..7d9e6ff
--- /dev/null
+++ b/backend/lib/CodeGen/GlobalISel/LegalizerInfo.cpp
@@ -0,0 +1,477 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "rcc/CodeGen/GlobalISel/LegalizerInfo.h"
+
+#include "rcc/CodeGen/GenericOpcodes.h"
+#include "rcc/CodeGen/InstructionDescriptor.h"
+#include "rcc/CodeGen/MachineFunction.h"
+#include "rcc/CodeGen/MachineInstr.h"
+#include "rcc/CodeGen/MachineOperand.h"
+#include "rcc/CodeGen/MachineRegisterInfo.h"
+#include "rcc/CodeGen/TargetDescription.h"
+
+#include <algorithm>
+#include <bit>
+#include <cstddef>
+#include <cstdint>
+#include <optional>
+#include <ranges>
+#include <tuple>
+#include <utility>
+#include <vector>
+
+namespace rcc {
+namespace {
+
+[[nodiscard]] LegalityPredicate typeInSet(std::size_t TypeIndex, std::vector<LowLevelType> Types) {
+  return [TypeIndex, Types = std::move(Types)](const LegalizationQuery &Query) {
+    return TypeIndex < Query.Types.size() &&
+           std::ranges::find(Types, Query.Types[TypeIndex]) != Types.end();
+  };
+}
+
+[[nodiscard]] LegalityPredicate
+typePairInSet(std::vector<std::pair<LowLevelType, LowLevelType>> Types) {
+  return [Types = std::move(Types)](const LegalizationQuery &Query) {
+    return Query.Types.size() >= 2 &&
+           std::ranges::find(Types, std::pair{Query.Types[0], Query.Types[1]}) != Types.end();
+  };
+}
+
+[[nodiscard]] LegalityPredicate
+typeTripleInSet(std::vector<std::tuple<LowLevelType, LowLevelType, LowLevelType>> Types) {
+  return [Types = std::move(Types)](const LegalizationQuery &Query) {
+    return Query.Types.size() >= 3 &&
+           std::ranges::find(Types, std::tuple{Query.Types[0], Query.Types[1], Query.Types[2]}) !=
+               Types.end();
+  };
+}
+
+[[nodiscard]] LowLevelType scalarType(std::uint32_t BitWidth) {
+  auto Type = LowLevelType::makeCategoryNeutralScalar(BitWidth);
+  return Type ? *Type : LowLevelType{};
+}
+
+[[nodiscard]] std::uint32_t nextPowerOfTwo(std::uint32_t Width, std::uint32_t MinimumBitWidth) {
+  return std::max(std::bit_ceil(Width), MinimumBitWidth);
+}
+
+[[nodiscard]] bool memoryRuleMatches(const LegalizationQuery &Query,
+                                     const LegalizeMemoryRule &Rule) {
+  if (Query.Types.size() < 2 || Query.MemoryOperands.size() != 1 ||
+      Query.Types[0] != Rule.ValueType || Query.Types[1] != Rule.AddressType)
+    return false;
+  const LegalizationMemoryDesc &Memory = Query.MemoryOperands.front();
+  if (Memory.AlignmentInBits < Rule.MinimumAlignmentInBits)
+    return false;
+  if (Memory.Type)
+    return Memory.Type->minimumBitWidth() == Rule.MemoryType.minimumBitWidth();
+  return Memory.Size.isKnown() && !Memory.Size.isScalable() &&
+         Memory.Size.minimumBytes() * 8U == Rule.MemoryType.scalarBitWidth();
+}
+
+} // namespace
+
+LegalizeRule::LegalizeRule(LegalityPredicate Predicate, LegalizationAction Action,
+                           LegalizeMutation Mutation)
+    : Predicate(std::move(Predicate)), Action(Action), Mutation(std::move(Mutation)) {}
+
+bool LegalizeRule::matches(const LegalizationQuery &Query) const {
+  return Predicate(Query);
+}
+
+LegalizationDecision LegalizeRule::apply(const LegalizationQuery &Query) const {
+  if (!Mutation)
+    return {.Action = Action, .TypeIndex = 0, .NewType = {}};
+  auto [TypeIndex, NewType] = Mutation(Query);
+  return {.Action = Action, .TypeIndex = TypeIndex, .NewType = NewType};
+}
+
+LegalizeRuleSet &LegalizeRuleSet::actionIf(LegalizationAction Action, LegalityPredicate Predicate,
+                                           LegalizeMutation Mutation) {
+  Rules.emplace_back(std::move(Predicate), Action, std::move(Mutation));
+  return *this;
+}
+
+LegalizeRuleSet &LegalizeRuleSet::actionFor(LegalizationAction Action,
+                                            std::initializer_list<LowLevelType> Types) {
+  return actionIf(Action, typeInSet(0, std::vector<LowLevelType>(Types)));
+}
+
+LegalizeRuleSet &
+LegalizeRuleSet::actionFor(LegalizationAction Action,
+                           std::initializer_list<std::pair<LowLevelType, LowLevelType>> Types) {
+  return actionIf(Action, typePairInSet(std::vector<std::pair<LowLevelType, LowLevelType>>(Types)));
+}
+
+LegalizeRuleSet &LegalizeRuleSet::actionFor(
+    LegalizationAction Action,
+    std::initializer_list<std::tuple<LowLevelType, LowLevelType, LowLevelType>> Types) {
+  return actionIf(
+      Action,
+      typeTripleInSet(std::vector<std::tuple<LowLevelType, LowLevelType, LowLevelType>>(Types)));
+}
+
+LegalizeRuleSet &LegalizeRuleSet::legalIf(LegalityPredicate Predicate) {
+  return actionIf(LegalizationAction::Legal, std::move(Predicate));
+}
+
+LegalizeRuleSet &LegalizeRuleSet::customIf(LegalityPredicate Predicate) {
+  return actionIf(LegalizationAction::Custom, std::move(Predicate));
+}
+
+LegalizeRuleSet &LegalizeRuleSet::lowerIf(LegalityPredicate Predicate) {
+  return actionIf(LegalizationAction::Lower, std::move(Predicate));
+}
+
+LegalizeRuleSet &LegalizeRuleSet::libcallIf(LegalityPredicate Predicate) {
+  return actionIf(LegalizationAction::Libcall, std::move(Predicate));
+}
+
+LegalizeRuleSet &LegalizeRuleSet::unsupportedIf(LegalityPredicate Predicate) {
+  return actionIf(LegalizationAction::Unsupported, std::move(Predicate));
+}
+
+LegalizeRuleSet &LegalizeRuleSet::widenScalarIf(LegalityPredicate Predicate,
+                                                LegalizeMutation Mutation) {
+  return actionIf(LegalizationAction::WidenScalar, std::move(Predicate), std::move(Mutation));
+}
+
+LegalizeRuleSet &LegalizeRuleSet::narrowScalarIf(LegalityPredicate Predicate,
+                                                 LegalizeMutation Mutation) {
+  return actionIf(LegalizationAction::NarrowScalar, std::move(Predicate), std::move(Mutation));
+}
+
+LegalizeRuleSet &LegalizeRuleSet::moreElementsIf(LegalityPredicate Predicate,
+                                                 LegalizeMutation Mutation) {
+  return actionIf(LegalizationAction::MoreElements, std::move(Predicate), std::move(Mutation));
+}
+
+LegalizeRuleSet &LegalizeRuleSet::fewerElementsIf(LegalityPredicate Predicate,
+                                                  LegalizeMutation Mutation) {
+  return actionIf(LegalizationAction::FewerElements, std::move(Predicate), std::move(Mutation));
+}
+
+LegalizeRuleSet &LegalizeRuleSet::bitcastIf(LegalityPredicate Predicate,
+                                            LegalizeMutation Mutation) {
+  return actionIf(LegalizationAction::Bitcast, std::move(Predicate), std::move(Mutation));
+}
+
+LegalizeRuleSet &LegalizeRuleSet::legalFor(std::initializer_list<LowLevelType> Types) {
+  return actionFor(LegalizationAction::Legal, Types);
+}
+
+LegalizeRuleSet &LegalizeRuleSet::legalFor(bool Enabled,
+                                           std::initializer_list<LowLevelType> Types) {
+  return Enabled ? legalFor(Types) : *this;
+}
+
+LegalizeRuleSet &
+LegalizeRuleSet::legalFor(std::initializer_list<std::pair<LowLevelType, LowLevelType>> Types) {
+  return actionFor(LegalizationAction::Legal, Types);
+}
+
+LegalizeRuleSet &
+LegalizeRuleSet::legalFor(bool Enabled,
+                          std::initializer_list<std::pair<LowLevelType, LowLevelType>> Types) {
+  return Enabled ? legalFor(Types) : *this;
+}
+
+LegalizeRuleSet &LegalizeRuleSet::legalFor(
+    std::initializer_list<std::tuple<LowLevelType, LowLevelType, LowLevelType>> Types) {
+  return actionFor(LegalizationAction::Legal, Types);
+}
+
+LegalizeRuleSet &LegalizeRuleSet::customFor(std::initializer_list<LowLevelType> Types) {
+  return actionFor(LegalizationAction::Custom, Types);
+}
+
+LegalizeRuleSet &LegalizeRuleSet::customFor(bool Enabled,
+                                            std::initializer_list<LowLevelType> Types) {
+  return Enabled ? customFor(Types) : *this;
+}
+
+LegalizeRuleSet &
+LegalizeRuleSet::customFor(std::initializer_list<std::pair<LowLevelType, LowLevelType>> Types) {
+  return actionFor(LegalizationAction::Custom, Types);
+}
+
+LegalizeRuleSet &
+LegalizeRuleSet::customFor(bool Enabled,
+                           std::initializer_list<std::pair<LowLevelType, LowLevelType>> Types) {
+  return Enabled ? customFor(Types) : *this;
+}
+
+LegalizeRuleSet &LegalizeRuleSet::lowerFor(std::initializer_list<LowLevelType> Types) {
+  return actionFor(LegalizationAction::Lower, Types);
+}
+
+LegalizeRuleSet &
+LegalizeRuleSet::lowerFor(std::initializer_list<std::pair<LowLevelType, LowLevelType>> Types) {
+  return actionFor(LegalizationAction::Lower, Types);
+}
+
+LegalizeRuleSet &LegalizeRuleSet::libcallFor(std::initializer_list<LowLevelType> Types) {
+  return actionFor(LegalizationAction::Libcall, Types);
+}
+
+LegalizeRuleSet &LegalizeRuleSet::libcallFor(bool Enabled,
+                                             std::initializer_list<LowLevelType> Types) {
+  return Enabled ? libcallFor(Types) : *this;
+}
+
+LegalizeRuleSet &
+LegalizeRuleSet::libcallFor(std::initializer_list<std::pair<LowLevelType, LowLevelType>> Types) {
+  return actionFor(LegalizationAction::Libcall, Types);
+}
+
+LegalizeRuleSet &
+LegalizeRuleSet::libcallFor(bool Enabled,
+                            std::initializer_list<std::pair<LowLevelType, LowLevelType>> Types) {
+  return Enabled ? libcallFor(Types) : *this;
+}
+
+LegalizeRuleSet &
+LegalizeRuleSet::legalForTypesWithMemDesc(std::initializer_list<LegalizeMemoryRule> MemoryRules) {
+  const std::vector<LegalizeMemoryRule> MemoryRuleList(MemoryRules);
+  return legalIf([MemoryRuleList](const LegalizationQuery &Query) {
+    return std::ranges::any_of(MemoryRuleList, [&Query](const LegalizeMemoryRule &Rule) {
+      return memoryRuleMatches(Query, Rule);
+    });
+  });
+}
+
+LegalizeRuleSet &LegalizeRuleSet::widenScalarToNextPow2(std::size_t TypeIndex,
+                                                        std::uint32_t MinimumBitWidth) {
+  return widenScalarIf(
+      [TypeIndex, MinimumBitWidth](const LegalizationQuery &Query) {
+        if (TypeIndex >= Query.Types.size() || !Query.Types[TypeIndex].isCategoryNeutralScalar())
+          return false;
+        const std::uint32_t Width = Query.Types[TypeIndex].scalarBitWidth();
+        return nextPowerOfTwo(Width, MinimumBitWidth) > Width;
+      },
+      [TypeIndex, MinimumBitWidth](const LegalizationQuery &Query) {
+        return std::pair{TypeIndex, scalarType(nextPowerOfTwo(
+                                        Query.Types[TypeIndex].scalarBitWidth(), MinimumBitWidth))};
+      });
+}
+
+LegalizeRuleSet &LegalizeRuleSet::minScalar(std::size_t TypeIndex, LowLevelType MinimumType) {
+  return widenScalarIf(
+      [TypeIndex, MinimumType](const LegalizationQuery &Query) {
+        return TypeIndex < Query.Types.size() && Query.Types[TypeIndex].isCategoryNeutralScalar() &&
+               Query.Types[TypeIndex].scalarBitWidth() < MinimumType.scalarBitWidth();
+      },
+      [TypeIndex, MinimumType](const LegalizationQuery &) {
+        return std::pair{TypeIndex, MinimumType};
+      });
+}
+
+LegalizeRuleSet &LegalizeRuleSet::minScalar(bool Enabled, std::size_t TypeIndex,
+                                            LowLevelType MinimumType) {
+  return Enabled ? minScalar(TypeIndex, MinimumType) : *this;
+}
+
+LegalizeRuleSet &LegalizeRuleSet::maxScalar(std::size_t TypeIndex, LowLevelType MaximumType) {
+  return narrowScalarIf(
+      [TypeIndex, MaximumType](const LegalizationQuery &Query) {
+        return TypeIndex < Query.Types.size() && Query.Types[TypeIndex].isCategoryNeutralScalar() &&
+               Query.Types[TypeIndex].scalarBitWidth() > MaximumType.scalarBitWidth();
+      },
+      [TypeIndex, MaximumType](const LegalizationQuery &) {
+        return std::pair{TypeIndex, MaximumType};
+      });
+}
+
+LegalizeRuleSet &LegalizeRuleSet::maxScalar(bool Enabled, std::size_t TypeIndex,
+                                            LowLevelType MaximumType) {
+  return Enabled ? maxScalar(TypeIndex, MaximumType) : *this;
+}
+
+LegalizeRuleSet &LegalizeRuleSet::clampScalar(std::size_t TypeIndex, LowLevelType MinimumType,
+                                              LowLevelType MaximumType) {
+  return minScalar(TypeIndex, MinimumType).maxScalar(TypeIndex, MaximumType);
+}
+
+LegalizeRuleSet &LegalizeRuleSet::scalarSameSizeAs(std::size_t TypeIndex,
+                                                   std::size_t ReferenceTypeIndex) {
+  const auto changeToReference = [TypeIndex, ReferenceTypeIndex](const LegalizationQuery &Query) {
+    return std::pair{TypeIndex, scalarType(Query.Types[ReferenceTypeIndex].scalarBitWidth())};
+  };
+  widenScalarIf(
+      [TypeIndex, ReferenceTypeIndex](const LegalizationQuery &Query) {
+        return TypeIndex < Query.Types.size() && ReferenceTypeIndex < Query.Types.size() &&
+               Query.Types[TypeIndex].isCategoryNeutralScalar() &&
+               Query.Types[ReferenceTypeIndex].isCategoryNeutralScalar() &&
+               Query.Types[TypeIndex].scalarBitWidth() <
+                   Query.Types[ReferenceTypeIndex].scalarBitWidth();
+      },
+      changeToReference);
+  return narrowScalarIf(
+      [TypeIndex, ReferenceTypeIndex](const LegalizationQuery &Query) {
+        return TypeIndex < Query.Types.size() && ReferenceTypeIndex < Query.Types.size() &&
+               Query.Types[TypeIndex].isCategoryNeutralScalar() &&
+               Query.Types[ReferenceTypeIndex].isCategoryNeutralScalar() &&
+               Query.Types[TypeIndex].scalarBitWidth() >
+                   Query.Types[ReferenceTypeIndex].scalarBitWidth();
+      },
+      changeToReference);
+}
+
+LegalizeRuleSet &LegalizeRuleSet::lowerIfMemSizeNotByteSizePow2(std::size_t MemoryIndex) {
+  return lowerIf([MemoryIndex](const LegalizationQuery &Query) {
+    if (MemoryIndex >= Query.MemoryOperands.size())
+      return false;
+    const MemorySize Size = Query.MemoryOperands[MemoryIndex].Size;
+    return !Size.isKnown() || Size.isScalable() || !std::has_single_bit(Size.minimumBytes());
+  });
+}
+
+LegalizeRuleSet &LegalizeRuleSet::alwaysLegal() {
+  return legalIf([](const LegalizationQuery &) { return true; });
+}
+
+LegalizeRuleSet &LegalizeRuleSet::lower() {
+  return lowerIf([](const LegalizationQuery &) { return true; });
+}
+
+LegalizeRuleSet &LegalizeRuleSet::libcall() {
+  return libcallIf([](const LegalizationQuery &) { return true; });
+}
+
+std::optional<LegalizationDecision> LegalizeRuleSet::apply(const LegalizationQuery &Query) const {
+  for (const LegalizeRule &Rule : Rules) {
+    if (Rule.matches(Query))
+      return Rule.apply(Query);
+  }
+  return std::nullopt;
+}
+
+LegalizeRuleSet &LegalizerInfo::getActionDefinitionsBuilder(Opcode InstructionOpcode) {
+  return RuleSets[InstructionOpcode];
+}
+
+LegalizeRuleSet &
+LegalizerInfo::getActionDefinitionsBuilder(std::initializer_list<Opcode> InstructionOpcodes) {
+  const Opcode Representative = *InstructionOpcodes.begin();
+  for (const Opcode InstructionOpcode : InstructionOpcodes) {
+    if (InstructionOpcode != Representative)
+      RuleAliases.emplace(InstructionOpcode, Representative);
+  }
+  return getActionDefinitionsBuilder(Representative);
+}
+
+Expected<LegalizationQuery> LegalizerInfo::buildQuery(const MachineInstr &Instruction,
+                                                      const MachineFunction &Function,
+                                                      const TargetDescription &Target) {
+  const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction.opcode());
+  if (Descriptor == nullptr)
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "legalization query requires a registered instruction opcode");
+
+  std::size_t ExplicitCount = 0;
+  while (ExplicitCount < Instruction.operandCount() &&
+         !Instruction.operandAt(ExplicitCount)->isImplicit()) {
+    ++ExplicitCount;
+  }
+  auto OperandDescriptors = resolveExplicitOperandDescriptors(*Descriptor, ExplicitCount);
+  if (!OperandDescriptors)
+    return std::unexpected(OperandDescriptors.error());
+
+  std::size_t TypeCount = 0;
+  for (const OperandDescriptor *Operand : *OperandDescriptors) {
+    if (Operand->TypeConstraint && Operand->TypeConstraint->EqualityGroup) {
+      TypeCount = std::max(TypeCount,
+                           static_cast<std::size_t>(*Operand->TypeConstraint->EqualityGroup) + 1U);
+    }
+  }
+
+  std::vector<std::optional<LowLevelType>> OptionalTypes(TypeCount);
+  std::vector<std::vector<std::size_t>> TypeOperands(TypeCount);
+  for (std::size_t Index = 0; Index < ExplicitCount; ++Index) {
+    const OperandDescriptor &OperandDescriptor = *(*OperandDescriptors)[Index];
+    if (!OperandDescriptor.TypeConstraint || !OperandDescriptor.TypeConstraint->EqualityGroup) {
+      continue;
+    }
+    const std::size_t TypeIndex = *OperandDescriptor.TypeConstraint->EqualityGroup;
+    const Register *Reg = Instruction.operandAt(Index)->getIf<Register>();
+    if (Reg == nullptr || !Reg->isVirtual())
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "legalization type operand must name a virtual register");
+    auto Info = Function.registerInfo().virtualRegisterInfo(*Reg);
+    if (!Info)
+      return std::unexpected(Info.error());
+    if (!Info->get().Type)
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "legalization type operand has no LowLevelType");
+    if (OptionalTypes[TypeIndex] && *OptionalTypes[TypeIndex] != *Info->get().Type)
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "legalization type operands disagree within one type index");
+    OptionalTypes[TypeIndex] = *Info->get().Type;
+    TypeOperands[TypeIndex].push_back(Index);
+  }
+
+  std::vector<LowLevelType> Types;
+  Types.reserve(TypeCount);
+  for (const std::optional<LowLevelType> &Type : OptionalTypes) {
+    if (!Type)
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "legalization query contains a sparse type index");
+    Types.push_back(*Type);
+  }
+
+  std::vector<LegalizationMemoryDesc> MemoryOperands;
+  MemoryOperands.reserve(Instruction.memOperands().size());
+  for (const MachineMemOperand *Memory : Instruction.memOperands()) {
+    const MachineMemOperandDesc &Description = Memory->description();
+    MemoryOperands.push_back({Description.MemoryType, Description.Size,
+                              Description.Align.bytes() * 8U, Description.AddressSpace,
+                              Description.AtomicKind, Description.SuccessOrdering,
+                              Description.FailureOrdering, Description.SyncScope});
+  }
+
+  LegalizationQuery Query(Instruction.opcode(), std::move(Types), std::move(MemoryOperands));
+  Query.TypeOperands = std::move(TypeOperands);
+  return Query;
+}
+
+Expected<LegalizationDecision> LegalizerInfo::getDecision(const MachineInstr &Instruction,
+                                                          const MachineFunction &Function,
+                                                          const TargetDescription &Target) const {
+  auto Query = buildQuery(Instruction, Function, Target);
+  if (!Query)
+    return std::unexpected(Query.error());
+  return getDecision(*Query);
+}
+
+Expected<LegalizationDecision> LegalizerInfo::getDecision(const LegalizationQuery &Query) const {
+  if (!isGenericOpcode(Query.InstructionOpcode))
+    return LegalizationDecision{.Action = LegalizationAction::Legal, .TypeIndex = 0, .NewType = {}};
+
+  Opcode RuleOpcode = Query.InstructionOpcode;
+  if (const auto Alias = RuleAliases.find(RuleOpcode); Alias != RuleAliases.end())
+    RuleOpcode = Alias->second;
+  const auto RuleSet = RuleSets.find(RuleOpcode);
+  if (RuleSet == RuleSets.end())
+    return LegalizationDecision{
+        .Action = LegalizationAction::Unsupported, .TypeIndex = 0, .NewType = {}};
+  const std::optional<LegalizationDecision> Decision = RuleSet->second.apply(Query);
+  return Decision.value_or(LegalizationDecision{
+      .Action = LegalizationAction::Unsupported, .TypeIndex = 0, .NewType = {}});
+}
+
+Expected<bool> LegalizerInfo::isLegal(const LegalizationQuery &Query) const {
+  auto Decision = getDecision(Query);
+  if (!Decision)
+    return std::unexpected(Decision.error());
+  return Decision->Action == LegalizationAction::Legal;
+}
+
+Expected<bool> LegalizerInfo::isUnsupported(const LegalizationQuery &Query) const {
+  auto Decision = getDecision(Query);
+  if (!Decision)
+    return std::unexpected(Decision.error());
+  return Decision->Action == LegalizationAction::Unsupported;
+}
+
+} // namespace rcc
diff --git a/backend/lib/CodeGen/InstructionDescriptor.cpp b/backend/lib/CodeGen/InstructionDescriptor.cpp
index d164cde..027f738 100644
--- a/backend/lib/CodeGen/InstructionDescriptor.cpp
+++ b/backend/lib/CodeGen/InstructionDescriptor.cpp
@@ -302,6 +302,19 @@ struct OperandShape final {
 
 } // namespace
 
+Expected<std::vector<const OperandDescriptor *>>
+resolveExplicitOperandDescriptors(const InstructionDescriptor &Descriptor,
+                                  std::size_t OperandCount) {
+  auto Shape = resolveOperandShape(Descriptor, OperandCount);
+  if (!Shape)
+    return std::unexpected(Shape.error());
+  std::vector<const OperandDescriptor *> Result;
+  Result.reserve(OperandCount);
+  for (std::size_t Index = 0; Index < OperandCount; ++Index)
+    Result.push_back(&descriptorForOperand(Descriptor, *Shape, Index));
+  return Result;
+}
+
 OperandKindSet::OperandKindSet(std::initializer_list<MachineOperand::Kind> Kinds) noexcept {
   for (const MachineOperand::Kind Kind : Kinds)
     add(Kind);
diff --git a/backend/lib/CodeGen/MachineBasicBlock.cpp b/backend/lib/CodeGen/MachineBasicBlock.cpp
index a8d69f6..763a577 100644
--- a/backend/lib/CodeGen/MachineBasicBlock.cpp
+++ b/backend/lib/CodeGen/MachineBasicBlock.cpp
@@ -40,6 +40,8 @@ Expected<void> MachineBasicBlock::appendInstruction(MachineInstr &Instruction) {
   else
     FirstInstruction = &Instruction;
   LastInstruction = &Instruction;
+  if (OwnerFunction->TheDelegate != nullptr)
+    OwnerFunction->TheDelegate->handleInsertion(Instruction);
   return {};
 }
 
@@ -65,6 +67,8 @@ Expected<void> MachineBasicBlock::insertInstructionBefore(MachineInstr &Position
   else
     FirstInstruction = &Instruction;
   Position.PreviousInstruction = &Instruction;
+  if (OwnerFunction->TheDelegate != nullptr)
+    OwnerFunction->TheDelegate->handleInsertion(Instruction);
   return {};
 }
 
@@ -90,6 +94,8 @@ Expected<void> MachineBasicBlock::insertInstructionAfter(MachineInstr &Position,
   else
     LastInstruction = &Instruction;
   Position.NextInstruction = &Instruction;
+  if (OwnerFunction->TheDelegate != nullptr)
+    OwnerFunction->TheDelegate->handleInsertion(Instruction);
   return {};
 }
 
@@ -98,6 +104,9 @@ Expected<void> MachineBasicBlock::removeInstruction(MachineInstr &Instruction) {
     return makeUnexpected(ErrorCode::InvalidArgument, "instruction does not belong to this block");
   }
 
+  if (OwnerFunction->TheDelegate != nullptr)
+    OwnerFunction->TheDelegate->handleRemoval(Instruction);
+
   if (Instruction.PreviousInstruction != nullptr)
     Instruction.PreviousInstruction->NextInstruction = Instruction.NextInstruction;
   else
diff --git a/backend/lib/CodeGen/MachineIRBuilder.cpp b/backend/lib/CodeGen/MachineIRBuilder.cpp
index e59f613..9195d1a 100644
--- a/backend/lib/CodeGen/MachineIRBuilder.cpp
+++ b/backend/lib/CodeGen/MachineIRBuilder.cpp
@@ -2,6 +2,7 @@
 
 #include "rcc/CodeGen/MachineIRBuilder.h"
 
+#include "rcc/CodeGen/GenericOpcodes.h"
 #include "rcc/CodeGen/MachineBasicBlock.h"
 #include "rcc/CodeGen/MachineFrameInfo.h"
 #include "rcc/CodeGen/MachineFunction.h"
@@ -707,4 +708,196 @@ Expected<MachineInstrBuilder> MachineIRBuilder::buildInstr(Opcode InstructionOpc
   return MachineInstrBuilder(*Function, *Target, *Descriptor, *InsertBlock, InsertBefore);
 }
 
+Expected<LowLevelType> MachineIRBuilder::registerType(Register Reg) const {
+  if (!Reg.isVirtual())
+    return makeUnexpected(ErrorCode::InvalidArgument,
+                          "generic builder operand is not a virtual register");
+  auto Info = Function->registerInfo().virtualRegisterInfo(Reg);
+  if (!Info)
+    return std::unexpected(Info.error());
+  if (!Info->get().Type)
+    return makeUnexpected(ErrorCode::InvalidArgument,
+                          "generic builder operand has no LowLevelType");
+  return *Info->get().Type;
+}
+
+Expected<MachineInstr *> MachineIRBuilder::buildUnary(Opcode InstructionOpcode,
+                                                      Register Destination,
+                                                      Register Source) const {
+  auto Pending = buildInstr(InstructionOpcode);
+  if (!Pending)
+    return std::unexpected(Pending.error());
+  if (auto Added = Pending->addDef(Destination); !Added)
+    return std::unexpected(Added.error());
+  if (auto Added = Pending->addUse(Source); !Added)
+    return std::unexpected(Added.error());
+  return Pending->build();
+}
+
+Expected<MachineInstr *> MachineIRBuilder::buildBinary(Opcode InstructionOpcode,
+                                                       Register Destination, Register Left,
+                                                       Register Right) const {
+  auto Pending = buildInstr(InstructionOpcode);
+  if (!Pending)
+    return std::unexpected(Pending.error());
+  if (auto Added = Pending->addDef(Destination); !Added)
+    return std::unexpected(Added.error());
+  if (auto Added = Pending->addUse(Left); !Added)
+    return std::unexpected(Added.error());
+  if (auto Added = Pending->addUse(Right); !Added)
+    return std::unexpected(Added.error());
+  return Pending->build();
+}
+
+Expected<MachineInstr *> MachineIRBuilder::buildTrunc(Register Destination,
+                                                      Register Source) const {
+  return buildUnary(opcode(GenericOpcode::G_TRUNC), Destination, Source);
+}
+
+Expected<MachineInstr *> MachineIRBuilder::buildCopy(Register Destination,
+                                                     Register Source) const {
+  return buildUnary(opcode(GenericOpcode::COPY), Destination, Source);
+}
+
+Expected<MachineInstr *> MachineIRBuilder::buildImplicitDef(Register Destination) const {
+  auto Pending = buildInstr(opcode(GenericOpcode::G_IMPLICIT_DEF));
+  if (!Pending)
+    return std::unexpected(Pending.error());
+  if (auto Added = Pending->addDef(Destination); !Added)
+    return std::unexpected(Added.error());
+  return Pending->build();
+}
+
+Expected<MachineInstr *> MachineIRBuilder::buildAnyExt(Register Destination,
+                                                       Register Source) const {
+  return buildUnary(opcode(GenericOpcode::G_ANYEXT), Destination, Source);
+}
+
+Expected<MachineInstr *> MachineIRBuilder::buildZExt(Register Destination,
+                                                     Register Source) const {
+  return buildUnary(opcode(GenericOpcode::G_ZEXT), Destination, Source);
+}
+
+Expected<MachineInstr *> MachineIRBuilder::buildSExt(Register Destination,
+                                                     Register Source) const {
+  return buildUnary(opcode(GenericOpcode::G_SEXT), Destination, Source);
+}
+
+Expected<MachineInstr *> MachineIRBuilder::buildExtOrTrunc(Opcode ExtensionOpcode,
+                                                           Register Destination,
+                                                           Register Source) const {
+  auto DestinationType = registerType(Destination);
+  if (!DestinationType)
+    return std::unexpected(DestinationType.error());
+  auto SourceType = registerType(Source);
+  if (!SourceType)
+    return std::unexpected(SourceType.error());
+  const std::uint64_t DestinationWidth = DestinationType->minimumBitWidth();
+  const std::uint64_t SourceWidth = SourceType->minimumBitWidth();
+  if (DestinationWidth > SourceWidth)
+    return buildUnary(ExtensionOpcode, Destination, Source);
+  if (DestinationWidth < SourceWidth)
+    return buildTrunc(Destination, Source);
+  return buildUnary(opcode(GenericOpcode::COPY), Destination, Source);
+}
+
+Expected<MachineInstr *> MachineIRBuilder::buildAnyExtOrTrunc(Register Destination,
+                                                              Register Source) const {
+  return buildExtOrTrunc(opcode(GenericOpcode::G_ANYEXT), Destination, Source);
+}
+
+Expected<MachineInstr *> MachineIRBuilder::buildSExtOrTrunc(Register Destination,
+                                                           Register Source) const {
+  return buildExtOrTrunc(opcode(GenericOpcode::G_SEXT), Destination, Source);
+}
+
+Expected<MachineInstr *> MachineIRBuilder::buildConstant(Register Destination,
+                                                         APInt Value) const {
+  auto Pending = buildInstr(opcode(GenericOpcode::G_CONSTANT));
+  if (!Pending)
+    return std::unexpected(Pending.error());
+  if (auto Added = Pending->addDef(Destination); !Added)
+    return std::unexpected(Added.error());
+  if (auto Added = Pending->addIntegerImmediate(std::move(Value)); !Added)
+    return std::unexpected(Added.error());
+  return Pending->build();
+}
+
+Expected<MachineInstr *> MachineIRBuilder::buildConstant(Register Destination,
+                                                         std::int64_t Value) const {
+  auto Type = registerType(Destination);
+  if (!Type)
+    return std::unexpected(Type.error());
+  auto Immediate = APInt::fromSigned(Type->scalarBitWidth(), Value);
+  if (!Immediate)
+    return std::unexpected(Immediate.error());
+  return buildConstant(Destination, std::move(*Immediate));
+}
+
+Expected<MachineInstr *> MachineIRBuilder::buildSExtInReg(Register Destination,
+                                                          Register Source,
+                                                          std::uint32_t Width) const {
+  auto Pending = buildInstr(opcode(GenericOpcode::G_SEXT_INREG));
+  if (!Pending)
+    return std::unexpected(Pending.error());
+  if (auto Added = Pending->addDef(Destination); !Added)
+    return std::unexpected(Added.error());
+  if (auto Added = Pending->addUse(Source); !Added)
+    return std::unexpected(Added.error());
+  if (auto Added = Pending->addImmediate(static_cast<std::int64_t>(Width)); !Added)
+    return std::unexpected(Added.error());
+  return Pending->build();
+}
+
+Expected<MachineInstr *> MachineIRBuilder::buildAnd(Register Destination, Register Left,
+                                                    Register Right) const {
+  return buildBinary(opcode(GenericOpcode::G_AND), Destination, Left, Right);
+}
+
+Expected<MachineInstr *> MachineIRBuilder::buildMergeValues(
+    Register Destination, const std::vector<Register> &Sources) const {
+  auto Pending = buildInstr(opcode(GenericOpcode::G_MERGE_VALUES));
+  if (!Pending)
+    return std::unexpected(Pending.error());
+  if (auto Added = Pending->addDef(Destination); !Added)
+    return std::unexpected(Added.error());
+  for (Register Source : Sources) {
+    if (auto Added = Pending->addUse(Source); !Added)
+      return std::unexpected(Added.error());
+  }
+  return Pending->build();
+}
+
+Expected<MachineInstr *> MachineIRBuilder::buildUnmergeValues(
+    const std::vector<Register> &Destinations, Register Source) const {
+  auto Pending = buildInstr(opcode(GenericOpcode::G_UNMERGE_VALUES));
+  if (!Pending)
+    return std::unexpected(Pending.error());
+  for (Register Destination : Destinations) {
+    if (auto Added = Pending->addDef(Destination); !Added)
+      return std::unexpected(Added.error());
+  }
+  if (auto Added = Pending->addUse(Source); !Added)
+    return std::unexpected(Added.error());
+  return Pending->build();
+}
+
+Expected<MachineInstr *> MachineIRBuilder::buildExtract(Register Destination,
+                                                        Register Source,
+                                                        std::uint64_t Offset) const {
+  if (Offset > static_cast<std::uint64_t>(std::numeric_limits<std::int64_t>::max()))
+    return makeUnexpected(ErrorCode::InvalidArgument,
+                          "generic extract offset does not fit a machine immediate");
+  auto Pending = buildInstr(opcode(GenericOpcode::G_EXTRACT));
+  if (!Pending)
+    return std::unexpected(Pending.error());
+  if (auto Added = Pending->addDef(Destination); !Added)
+    return std::unexpected(Added.error());
+  if (auto Added = Pending->addUse(Source); !Added)
+    return std::unexpected(Added.error());
+  if (auto Added = Pending->addImmediate(static_cast<std::int64_t>(Offset)); !Added)
+    return std::unexpected(Added.error());
+  return Pending->build();
+}
+
 } // namespace rcc
diff --git a/backend/lib/CodeGen/MachineRegisterInfo.cpp b/backend/lib/CodeGen/MachineRegisterInfo.cpp
index a88bfb0..4627f9e 100644
--- a/backend/lib/CodeGen/MachineRegisterInfo.cpp
+++ b/backend/lib/CodeGen/MachineRegisterInfo.cpp
@@ -311,6 +311,27 @@ const MachineOperand *MachineRegisterInfo::getUniqueVRegDef(Register Reg) const
   return UniqueDef;
 }
 
+MachineInstr *MachineRegisterInfo::getVRegDef(Register Reg) noexcept {
+  const MachineOperand *Definition = getUniqueVRegDef(Reg);
+  return Definition == nullptr ? nullptr
+                               : const_cast<MachineInstr *>(Definition->parent());
+}
+
+const MachineInstr *MachineRegisterInfo::getVRegDef(Register Reg) const noexcept {
+  const MachineOperand *Definition = getUniqueVRegDef(Reg);
+  return Definition == nullptr ? nullptr : Definition->parent();
+}
+
+Expected<LowLevelType> MachineRegisterInfo::getType(Register Reg) const {
+  auto Info = virtualRegisterInfo(Reg);
+  if (!Info)
+    return std::unexpected(Info.error());
+  if (!Info->get().Type)
+    return makeUnexpected(ErrorCode::InvalidArgument,
+                          "virtual register has no LowLevelType");
+  return *Info->get().Type;
+}
+
 Expected<std::size_t> MachineRegisterInfo::replaceRegWith(Register OldReg, Register NewReg) {
   if (OldReg.isNoRegister() || NewReg.isNoRegister()) {
     return makeUnexpected(ErrorCode::InvalidArgument,
diff --git a/backend/lib/Support/APInt.cpp b/backend/lib/Support/APInt.cpp
index 898deee..d54a4c2 100644
--- a/backend/lib/Support/APInt.cpp
+++ b/backend/lib/Support/APInt.cpp
@@ -138,6 +138,23 @@ Expected<APInt> APInt::fromSigned(std::uint32_t BitWidth, std::int64_t Value) {
   return create(BitWidth, Words);
 }
 
+Expected<APInt> APInt::getLowBitsSet(std::uint32_t BitWidth,
+                                     std::uint32_t SetBitCount) {
+  if (BitWidth == 0 || BitWidth > MaxBitWidth || SetBitCount > BitWidth)
+    return makeUnexpected(ErrorCode::InvalidArgument,
+                          "APInt low-bit mask has an invalid width");
+  std::vector<std::uint64_t> Words(wordCount(BitWidth), 0);
+  const std::size_t FullWords = SetBitCount / 64U;
+  std::ranges::fill_n(
+      Words.begin(),
+      static_cast<std::vector<std::uint64_t>::difference_type>(FullWords),
+                      std::numeric_limits<std::uint64_t>::max());
+  const std::uint32_t RemainingBits = SetBitCount % 64U;
+  if (RemainingBits != 0)
+    Words[FullWords] = (std::uint64_t{1} << RemainingBits) - 1U;
+  return create(BitWidth, Words);
+}
+
 Expected<APInt> APInt::parseDecimal(std::uint32_t BitWidth, std::string_view Text) {
   if (BitWidth == 0 || BitWidth > MaxBitWidth)
     return makeUnexpected(ErrorCode::ParseError, "APInt bit width is outside the supported range");
@@ -243,6 +260,43 @@ std::uint64_t APInt::word(std::size_t Index) const noexcept {
   return Index < Words.size() ? Words[Index] : 0;
 }
 
+Expected<APInt> APInt::trunc(std::uint32_t NewBitWidth) const {
+  if (NewBitWidth == 0 || NewBitWidth >= BitWidth)
+    return makeUnexpected(ErrorCode::InvalidArgument,
+                          "APInt truncation requires a smaller nonzero width");
+  std::vector<std::uint64_t> ResultWords(wordCount(NewBitWidth), 0);
+  std::ranges::copy_n(
+      Words.begin(),
+      static_cast<std::vector<std::uint64_t>::difference_type>(
+          ResultWords.size()),
+      ResultWords.begin());
+  ResultWords.back() &= highWordMask(NewBitWidth);
+  return create(NewBitWidth, ResultWords);
+}
+
+Expected<APInt> APInt::zext(std::uint32_t NewBitWidth) const {
+  if (NewBitWidth <= BitWidth || NewBitWidth > MaxBitWidth)
+    return makeUnexpected(ErrorCode::InvalidArgument,
+                          "APInt zero extension requires a larger supported width");
+  std::vector<std::uint64_t> ResultWords(wordCount(NewBitWidth), 0);
+  std::ranges::copy(Words, ResultWords.begin());
+  return create(NewBitWidth, ResultWords);
+}
+
+Expected<APInt> APInt::sext(std::uint32_t NewBitWidth) const {
+  if (NewBitWidth <= BitWidth || NewBitWidth > MaxBitWidth)
+    return makeUnexpected(ErrorCode::InvalidArgument,
+                          "APInt sign extension requires a larger supported width");
+  std::vector<std::uint64_t> ResultWords(
+      wordCount(NewBitWidth),
+      isNegative() ? std::numeric_limits<std::uint64_t>::max() : std::uint64_t{0});
+  std::ranges::copy(Words, ResultWords.begin());
+  if (isNegative() && BitWidth % 64U != 0)
+    ResultWords[wordCount(BitWidth) - 1U] |= ~highWordMask(BitWidth);
+  ResultWords.back() &= highWordMask(NewBitWidth);
+  return create(NewBitWidth, ResultWords);
+}
+
 int APInt::compare(const APInt &Other) const noexcept {
   const std::size_t LeftCount = significantWordCount(Words);
   const std::size_t RightCount = significantWordCount(Other.Words);
diff --git a/backend/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp b/backend/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp
index e4c937f..59c992e 100644
--- a/backend/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp
+++ b/backend/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp
@@ -177,10 +177,10 @@ public:
     const std::optional<LowLevelType> Type = resultType(Function, Instruction);
     const auto Integer = [Type] { return Type && isSupportedIntegerType(*Type); };
     const auto Float = [Type] { return Type && isSupportedFloatingScalar(*Type); };
-    auto Classification = Subtarget.getLegalizerInfo()->classify(Instruction, Function, Target);
+    auto Classification = Subtarget.getLegalizerInfo()->getDecision(Instruction, Function, Target);
     if (!Classification)
       return std::unexpected(Classification.error());
-    if (*Classification != LegalizationAction::Legal) {
+    if (Classification->Action != LegalizationAction::Legal) {
       std::string Message = "RISC-V selector cannot select opcode '" +
                             std::string(getGenericOpcodeName(Generic)) + "'";
       if (Type)
diff --git a/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp b/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
index 1a29ebf..4d81404 100644
--- a/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
+++ b/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
@@ -2,9 +2,9 @@
 
 #include "RISCVLegalizerInfo.h"
 
-#include "RISCVCallLowering.h"
 #include "RISCVImmediateMaterialization.h"
 #include "rcc/CodeGen/GenericOpcodes.h"
+#include "rcc/CodeGen/GlobalISel/LegalizerHelper.h"
 #include "rcc/CodeGen/LowLevelType.h"
 #include "rcc/CodeGen/MachineBasicBlock.h"
 #include "rcc/CodeGen/MachineFunction.h"
@@ -59,45 +59,6 @@ operandType(const MachineFunction &Function, const MachineInstr &Instruction, st
   return Operand == nullptr ? std::nullopt : registerType(Function, *Operand);
 }
 
-[[nodiscard]] bool isSupportedFloatingScalar(LowLevelType Type) noexcept {
-  return Type.isCategoryNeutralScalar() &&
-         (Type.scalarBitWidth() == 32 || Type.scalarBitWidth() == 64);
-}
-
-[[nodiscard]] bool isSupportedIntegerType(LowLevelType Type) noexcept {
-  return (Type.isCategoryNeutralScalar() && Type.scalarBitWidth() != 0 &&
-          Type.scalarBitWidth() <= 64) ||
-         (Type.isPointer() && Type.scalarBitWidth() == 64);
-}
-
-[[nodiscard]] bool isSupportedIntegerScalar(LowLevelType Type) noexcept {
-  return Type.isCategoryNeutralScalar() && Type.scalarBitWidth() != 0 &&
-         Type.scalarBitWidth() <= 64;
-}
-
-[[nodiscard]] bool isNativeIntegerScalar(LowLevelType Type) noexcept {
-  return Type.isCategoryNeutralScalar() &&
-         (Type.scalarBitWidth() == 32 || Type.scalarBitWidth() == 64);
-}
-
-[[nodiscard]] bool isScalarGPRType(LowLevelType Type) noexcept {
-  return (Type.isCategoryNeutralScalar() && Type.scalarBitWidth() != 0 &&
-          Type.scalarBitWidth() <= 64) ||
-         (Type.isPointer() && Type.scalarBitWidth() == 64);
-}
-
-[[nodiscard]] bool isI1Type(LowLevelType Type) noexcept {
-  return Type.isCategoryNeutralScalar() && Type.scalarBitWidth() == 1;
-}
-
-[[nodiscard]] bool isScalarIntegerExtension(LowLevelType Destination,
-                                            LowLevelType Source) noexcept {
-  return Destination.isCategoryNeutralScalar() && Source.isCategoryNeutralScalar() &&
-         Destination.scalarBitWidth() != 0 && Source.scalarBitWidth() != 0 &&
-         Source.scalarBitWidth() < Destination.scalarBitWidth() &&
-         Destination.scalarBitWidth() <= 64;
-}
-
 [[nodiscard]] Expected<APInt>
 extendIntegerImmediate(const APInt &Value, std::uint32_t DestinationWidth, bool SignExtend) {
   if (DestinationWidth == 0 || DestinationWidth > 64)
@@ -193,13 +154,6 @@ foldConstantExtension(MachineInstr &Instruction, MachineFunction &Function, Gene
   return {};
 }
 
-struct WideConstantUnmergePlan final {
-  MachineInstr *Extension = nullptr;
-  MachineInstr *Unmerge = nullptr;
-  std::vector<Register> Outputs;
-  bool SignExtend = false;
-};
-
 [[nodiscard]] MachineInstr *singleRegisterUse(const MachineFunction &Function, Register Reg) {
   MachineInstr *UseInstruction = nullptr;
   auto &RegisterInfo = const_cast<MachineRegisterInfo &>(Function.registerInfo());
@@ -213,189 +167,6 @@ struct WideConstantUnmergePlan final {
   return UseInstruction;
 }
 
-[[nodiscard]] std::optional<WideConstantUnmergePlan>
-wideConstantUnmergePlan(const MachineInstr &Constant, const MachineFunction &Function) {
-  const MachineOperand *ResultOperand = Constant.operandAt(0);
-  const Register *Result = ResultOperand == nullptr ? nullptr : ResultOperand->getIf<Register>();
-  if (Result == nullptr || !Result->isVirtual())
-    return std::nullopt;
-
-  MachineInstr *FirstUse = singleRegisterUse(Function, *Result);
-  if (FirstUse == nullptr)
-    return std::nullopt;
-  MachineInstr *Extension = nullptr;
-  MachineInstr *Unmerge = FirstUse;
-  if (FirstUse->opcode() == opcode(GenericOpcode::G_ANYEXT) ||
-      FirstUse->opcode() == opcode(GenericOpcode::G_SEXT) ||
-      FirstUse->opcode() == opcode(GenericOpcode::G_ZEXT)) {
-    Extension = FirstUse;
-    const MachineOperand *ExtendedOperand = Extension->operandAt(0);
-    const Register *Extended =
-        ExtendedOperand == nullptr ? nullptr : ExtendedOperand->getIf<Register>();
-    if (Extended == nullptr || !Extended->isVirtual())
-      return std::nullopt;
-    Unmerge = singleRegisterUse(Function, *Extended);
-    if (Unmerge == nullptr)
-      return std::nullopt;
-  }
-  if (Unmerge->opcode() != opcode(GenericOpcode::G_UNMERGE_VALUES) || Unmerge->operandCount() < 3)
-    return std::nullopt;
-
-  WideConstantUnmergePlan Plan;
-  Plan.Extension = Extension;
-  Plan.Unmerge = Unmerge;
-  Plan.SignExtend = Extension != nullptr && Extension->opcode() != opcode(GenericOpcode::G_ZEXT);
-  for (std::size_t Index = 0; Index + 1U < Unmerge->operandCount(); ++Index) {
-    const MachineOperand *Operand = Unmerge->operandAt(Index);
-    const Register *Reg = Operand == nullptr ? nullptr : Operand->getIf<Register>();
-    if (Reg == nullptr || !Reg->isVirtual() || !Operand->isDef())
-      return std::nullopt;
-    const auto Type = registerType(Function, *Reg);
-    if (!Type || !isSupportedIntegerScalar(*Type))
-      return std::nullopt;
-    Plan.Outputs.push_back(*Reg);
-  }
-  const MachineOperand *SourceOperand = Unmerge->operandAt(Unmerge->operandCount() - 1U);
-  const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>();
-  const Register *ExpectedSource = Result;
-  if (Extension != nullptr) {
-    const MachineOperand *ExtendedOperand = Extension->operandAt(0);
-    ExpectedSource = ExtendedOperand == nullptr ? nullptr : ExtendedOperand->getIf<Register>();
-  }
-  if (Source == nullptr || ExpectedSource == nullptr || *Source != *ExpectedSource)
-    return std::nullopt;
-  return Plan;
-}
-
-[[nodiscard]] Expected<APInt> wideConstantWord(const APInt &Value, std::size_t WordIndex,
-                                               std::uint32_t Width, bool SignExtend) {
-  std::uint64_t Raw = 0;
-  if (WordIndex < Value.words().size())
-    Raw = Value.words()[WordIndex];
-  else if (SignExtend && Value.isNegative())
-    Raw = std::numeric_limits<std::uint64_t>::max();
-  const std::size_t WordBitOffset = WordIndex * 64U;
-  if (SignExtend && Value.isNegative() && WordBitOffset < Value.bitWidth()) {
-    const std::uint32_t Remaining = Value.bitWidth() - static_cast<std::uint32_t>(WordBitOffset);
-    if (Remaining < 64)
-      Raw |= ~std::uint64_t{0} << Remaining;
-  }
-  if (Width < 64)
-    Raw &= (std::uint64_t{1} << Width) - 1U;
-  return APInt::create(Width, std::span<const std::uint64_t>(&Raw, 1));
-}
-
-[[nodiscard]] Expected<void> foldWideConstant(MachineInstr &Instruction, MachineFunction &Function,
-                                              const TargetDescription &Target) {
-  const auto Plan = wideConstantUnmergePlan(Instruction, Function);
-  if (!Plan)
-    return makeUnexpected(ErrorCode::Unsupported,
-                          "RISC-V wide constant requires a G_UNMERGE_VALUES use");
-  const MachineOperand *ImmediateOperand = Instruction.operandAt(1);
-  const auto *Integer =
-      ImmediateOperand == nullptr ? nullptr : ImmediateOperand->getIf<IntegerImmediateOperand>();
-  if (Integer == nullptr)
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "RISC-V wide constant has no integer immediate");
-
-  std::vector<APInt> Words;
-  Words.reserve(Plan->Outputs.size());
-  for (std::size_t Index = 0; Index < Plan->Outputs.size(); ++Index) {
-    const auto Type = registerType(Function, Plan->Outputs[Index]);
-    if (!Type)
-      return makeUnexpected(ErrorCode::VerificationError,
-                            "RISC-V wide constant output has no virtual-register type");
-    auto Word = wideConstantWord(Integer->Value, Index, Type->scalarBitWidth(), Plan->SignExtend);
-    if (!Word)
-      return std::unexpected(Word.error());
-    Words.push_back(std::move(*Word));
-  }
-
-  std::vector<std::size_t> Representatives(Plan->Outputs.size());
-  for (std::size_t Index = 0; Index < Representatives.size(); ++Index) {
-    Representatives[Index] = Index;
-    for (std::size_t Previous = 0; Previous < Index; ++Previous) {
-      if (Words[Previous] == Words[Index]) {
-        Representatives[Index] = Previous;
-        break;
-      }
-    }
-  }
-
-  const Register FirstOutput = Plan->Outputs.front();
-  auto Retargeted = Instruction.operandAt(0)->setRegister(FirstOutput);
-  if (!Retargeted)
-    return std::unexpected(Retargeted.error());
-  auto SetOpcode = Instruction.setOpcode(opcode(GenericOpcode::G_CONSTANT));
-  if (!SetOpcode)
-    return std::unexpected(SetOpcode.error());
-  while (Instruction.operandCount() > 1) {
-    auto Erased = Instruction.eraseOperand(1);
-    if (!Erased)
-      return std::unexpected(Erased.error());
-  }
-  auto Added = Instruction.addOperand(MachineOperand::integerImmediate(std::move(Words.front())));
-  if (!Added)
-    return std::unexpected(Added.error());
-
-  MachineBasicBlock *Block = Instruction.parent();
-  if (Block == nullptr)
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "RISC-V wide constant is detached during legalization");
-  if (Plan->Unmerge != &Instruction) {
-    auto RemovedUnmerge = Plan->Unmerge->parent()->removeInstruction(*Plan->Unmerge);
-    if (!RemovedUnmerge)
-      return std::unexpected(RemovedUnmerge.error());
-  }
-  if (Plan->Extension != nullptr) {
-    auto RemovedExtension = Plan->Extension->parent()->removeInstruction(*Plan->Extension);
-    if (!RemovedExtension)
-      return std::unexpected(RemovedExtension.error());
-  }
-
-  for (std::size_t Index = 1; Index < Plan->Outputs.size(); ++Index) {
-    if (Representatives[Index] == Index)
-      continue;
-    auto Replaced = Function.registerInfo().replaceRegWith(Plan->Outputs[Index],
-                                                           Plan->Outputs[Representatives[Index]]);
-    if (!Replaced)
-      return std::unexpected(Replaced.error());
-  }
-
-  MachineInstr *Last = &Instruction;
-  for (std::size_t Index = 1; Index < Plan->Outputs.size(); ++Index) {
-    if (Representatives[Index] != Index)
-      continue;
-    MachineIRBuilder Builder(Function, Target);
-    auto Point = Builder.setInsertPointAfter(*Last);
-    if (!Point)
-      return std::unexpected(Point.error());
-    auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_CONSTANT));
-    if (!Pending)
-      return std::unexpected(Pending.error());
-    auto Def = Pending->addDef(Plan->Outputs[Index]);
-    if (!Def)
-      return std::unexpected(Def.error());
-    auto Value = Pending->addIntegerImmediate(std::move(Words[Index]));
-    if (!Value)
-      return std::unexpected(Value.error());
-    auto Built = Pending->build();
-    if (!Built)
-      return std::unexpected(Built.error());
-    Last = *Built;
-  }
-  return {};
-}
-
-[[nodiscard]] Expected<Register> buildIntegerConstantBefore(MachineIRBuilder &Builder,
-                                                            MachineInstr &Anchor,
-                                                            std::uint32_t Width,
-                                                            std::uint64_t Value);
-
-[[nodiscard]] Expected<Register> buildBinaryBefore(MachineIRBuilder &Builder, MachineInstr &Anchor,
-                                                   GenericOpcode Generic, LowLevelType Type,
-                                                   Register Left, Register Right);
-
 [[nodiscard]] Expected<Register> buildUnaryBefore(MachineIRBuilder &Builder, MachineInstr &Anchor,
                                                   GenericOpcode Generic, LowLevelType Type,
                                                   Register Source);
@@ -405,123 +176,6 @@ wideConstantUnmergePlan(const MachineInstr &Constant, const MachineFunction &Fun
 [[nodiscard]] Expected<void> rewriteUnary(MachineInstr &Instruction, GenericOpcode Generic,
                                           Register Destination, Register Source);
 
-struct NarrowAnyExtBinaryPlan final {
-  MachineInstr *Binary = nullptr;
-  GenericOpcode Generic = GenericOpcode::Invalid;
-  std::array<Register, 2> WideInputs{};
-  std::vector<MachineInstr *> Artifacts;
-  std::vector<MachineInstr *> Constants;
-  std::uint32_t SourceWidth = 0;
-};
-
-[[nodiscard]] std::optional<NarrowAnyExtBinaryPlan>
-narrowAnyExtBinaryPlan(const MachineInstr &Extension, const MachineFunction &Function) {
-  const MachineOperand *DestinationOperand = Extension.operandAt(0);
-  const MachineOperand *SourceOperand = Extension.operandAt(1);
-  const Register *Destination =
-      DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
-  const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>();
-  if (Destination == nullptr || Source == nullptr || !Destination->isVirtual() ||
-      !Source->isVirtual() || singleRegisterUse(Function, *Source) != &Extension)
-    return std::nullopt;
-  const auto DestinationType = registerType(Function, *Destination);
-  const auto SourceType = registerType(Function, *Source);
-  if (!DestinationType || !SourceType)
-    return std::nullopt;
-  if (!isScalarIntegerExtension(*DestinationType, *SourceType) ||
-      DestinationType->scalarBitWidth() != 64)
-    return std::nullopt;
-
-  const MachineOperand *SourceDefinition = Function.registerInfo().getUniqueVRegDef(*Source);
-  MachineInstr *Binary = SourceDefinition == nullptr
-                             ? nullptr
-                             : const_cast<MachineInstr *>(SourceDefinition->parent());
-  if (Binary == nullptr || Binary->operandCount() != 3)
-    return std::nullopt;
-  const GenericOpcode Generic = static_cast<GenericOpcode>(Binary->opcode().value());
-  if (Generic != GenericOpcode::G_ADD && Generic != GenericOpcode::G_SUB &&
-      Generic != GenericOpcode::G_MUL && Generic != GenericOpcode::G_AND &&
-      Generic != GenericOpcode::G_OR && Generic != GenericOpcode::G_XOR &&
-      Generic != GenericOpcode::G_SHL)
-    return std::nullopt;
-
-  NarrowAnyExtBinaryPlan Plan;
-  Plan.Binary = Binary;
-  Plan.Generic = Generic;
-  Plan.SourceWidth = SourceType->scalarBitWidth();
-  const auto AddArtifact = [&Plan](MachineInstr *Artifact) {
-    if (std::ranges::find(Plan.Artifacts, Artifact) == Plan.Artifacts.end())
-      Plan.Artifacts.push_back(Artifact);
-  };
-  const auto TruncatedWideSource = [&](MachineInstr *Truncation,
-                                       MachineInstr *OnlyUse) -> std::optional<Register> {
-    if (Truncation == nullptr || Truncation->opcode() != opcode(GenericOpcode::G_TRUNC) ||
-        Truncation->operandCount() != 2)
-      return std::nullopt;
-    const MachineOperand *NarrowOperand = Truncation->operandAt(0);
-    const Register *Narrow = NarrowOperand == nullptr ? nullptr : NarrowOperand->getIf<Register>();
-    const MachineOperand *WideOperand = Truncation->operandAt(1);
-    const Register *Wide = WideOperand == nullptr ? nullptr : WideOperand->getIf<Register>();
-    const std::optional<LowLevelType> WideType =
-        WideOperand == nullptr ? std::nullopt : registerType(Function, *WideOperand);
-    if (Narrow == nullptr || Wide == nullptr || !Wide->isVirtual() || !WideType ||
-        *WideType != *DestinationType || singleRegisterUse(Function, *Narrow) != OnlyUse)
-      return std::nullopt;
-    return *Wide;
-  };
-  for (std::size_t Index = 0; Index != Plan.WideInputs.size(); ++Index) {
-    const MachineOperand *Operand = Binary->operandAt(Index + 1U);
-    const Register *Reg = Operand == nullptr ? nullptr : Operand->getIf<Register>();
-    if (Reg == nullptr || !Reg->isVirtual())
-      return std::nullopt;
-    const auto Type = registerType(Function, *Reg);
-    if (!Type)
-      return std::nullopt;
-    if (*Type == *DestinationType) {
-      Plan.WideInputs[Index] = *Reg;
-      continue;
-    }
-    if (*Type != *SourceType)
-      return std::nullopt;
-
-    const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(*Reg);
-    MachineInstr *Def =
-        Definition == nullptr ? nullptr : const_cast<MachineInstr *>(Definition->parent());
-    if (auto Wide = TruncatedWideSource(Def, Binary)) {
-      Plan.WideInputs[Index] = *Wide;
-      AddArtifact(Def);
-      continue;
-    }
-    if (Def != nullptr &&
-        (Def->opcode() == opcode(GenericOpcode::COPY) ||
-         Def->opcode() == opcode(GenericOpcode::G_ANYEXT)) &&
-        Def->operandCount() == 2 && singleRegisterUse(Function, *Reg) == Binary) {
-      const MachineOperand *NarrowOperand = Def->operandAt(1);
-      const Register *Narrow =
-          NarrowOperand == nullptr ? nullptr : NarrowOperand->getIf<Register>();
-      const MachineOperand *NarrowDefinition =
-          Narrow == nullptr ? nullptr : Function.registerInfo().getUniqueVRegDef(*Narrow);
-      MachineInstr *Truncation = NarrowDefinition == nullptr
-                                     ? nullptr
-                                     : const_cast<MachineInstr *>(NarrowDefinition->parent());
-      if (auto Wide = TruncatedWideSource(Truncation, Def)) {
-        Plan.WideInputs[Index] = *Wide;
-        AddArtifact(Def);
-        AddArtifact(Truncation);
-        continue;
-      }
-    }
-    if (Def != nullptr && Def->opcode() == opcode(GenericOpcode::G_CONSTANT) &&
-        singleRegisterUse(Function, *Reg) == Binary) {
-      Plan.WideInputs[Index] = *Reg;
-      Plan.Constants.push_back(Def);
-      continue;
-    }
-    return std::nullopt;
-  }
-  return Plan;
-}
-
 [[nodiscard]] Expected<void> widenConstantToRV64(MachineInstr &Instruction,
                                                  MachineFunction &Function) {
   const MachineOperand *DestinationOperand = Instruction.operandAt(0);
@@ -893,127 +547,6 @@ lowerAtomicCompareExchangeWithSuccess(MachineInstr &Instruction, MachineFunction
   return {};
 }
 
-[[nodiscard]] Expected<void> foldNarrowAnyExtBinary(MachineInstr &Extension,
-                                                    MachineFunction &Function,
-                                                    const TargetDescription &Target) {
-  const auto Plan = narrowAnyExtBinaryPlan(Extension, Function);
-  if (!Plan)
-    return makeUnexpected(ErrorCode::Unsupported,
-                          "RISC-V narrow any-extension does not have a legal binary chain");
-  for (MachineInstr *Constant : Plan->Constants) {
-    auto Widened = widenConstantToRV64(*Constant, Function);
-    if (!Widened)
-      return std::unexpected(Widened.error());
-  }
-  for (std::size_t Index = 0; Index != Plan->WideInputs.size(); ++Index) {
-    auto Replaced = Plan->Binary->operandAt(Index + 1U)->setRegister(Plan->WideInputs[Index]);
-    if (!Replaced)
-      return std::unexpected(Replaced.error());
-  }
-  for (MachineInstr *Artifact : Plan->Artifacts) {
-    if (Artifact->parent() == nullptr)
-      continue;
-    auto Removed = Artifact->parent()->removeInstruction(*Artifact);
-    if (!Removed)
-      return std::unexpected(Removed.error());
-  }
-
-  if (Plan->Generic == GenericOpcode::G_SHL) {
-    Register Amount = Plan->WideInputs[1];
-    if (Plan->SourceWidth < 32 || Plan->SourceWidth > 32) {
-      const std::uint64_t Mask = Plan->SourceWidth == 64
-                                     ? std::numeric_limits<std::uint64_t>::max()
-                                     : (std::uint64_t{1} << Plan->SourceWidth) - 1U;
-      auto WideType = LowLevelType::makeCategoryNeutralScalar(64);
-      if (!WideType)
-        return std::unexpected(WideType.error());
-      MachineIRBuilder Builder(Function, Target);
-      auto MaskValue = buildIntegerConstantBefore(Builder, Extension, 64, Mask);
-      if (!MaskValue)
-        return std::unexpected(MaskValue.error());
-      auto MaskedAmount = buildBinaryBefore(Builder, Extension, GenericOpcode::G_AND, *WideType,
-                                            Amount, *MaskValue);
-      if (!MaskedAmount)
-        return std::unexpected(MaskedAmount.error());
-      Amount = *MaskedAmount;
-    }
-    auto SetOpcode =
-        Extension.setOpcode(Plan->SourceWidth == 32 ? instructionOpcode(InstructionOpcode::G_SLLW)
-                                                    : opcode(GenericOpcode::G_SHL));
-    if (!SetOpcode)
-      return std::unexpected(SetOpcode.error());
-    while (Extension.operandCount() > 1) {
-      auto Erased = Extension.eraseOperand(1);
-      if (!Erased)
-        return std::unexpected(Erased.error());
-    }
-    for (Register Input : {Plan->WideInputs[0], Amount}) {
-      auto Operand =
-          MachineOperand::registerOperand(Input, RegisterOperandFlags{RegisterOperandFlag::Use});
-      if (!Operand)
-        return std::unexpected(Operand.error());
-      auto Added = Extension.addOperand(std::move(*Operand));
-      if (!Added)
-        return std::unexpected(Added.error());
-    }
-    MachineBasicBlock *BinaryBlock = Plan->Binary->parent();
-    if (BinaryBlock == nullptr)
-      return makeUnexpected(ErrorCode::VerificationError,
-                            "RISC-V narrow shift is detached during legalization");
-    return BinaryBlock->removeInstruction(*Plan->Binary);
-  }
-
-  if (Plan->SourceWidth == 32 &&
-      (Plan->Generic == GenericOpcode::G_ADD || Plan->Generic == GenericOpcode::G_SUB)) {
-    auto WideType = LowLevelType::makeCategoryNeutralScalar(64);
-    if (!WideType)
-      return std::unexpected(WideType.error());
-    auto Temporary = Function.registerInfo().createVirtualRegister(*WideType);
-    if (!Temporary)
-      return std::unexpected(Temporary.error());
-    auto Retargeted = Plan->Binary->operandAt(0)->setRegister(*Temporary);
-    if (!Retargeted)
-      return std::unexpected(Retargeted.error());
-    auto SetOpcode = Extension.setOpcode(opcode(GenericOpcode::G_SEXT_INREG));
-    if (!SetOpcode)
-      return std::unexpected(SetOpcode.error());
-    auto ReplacedSource = Extension.operandAt(1)->setRegister(*Temporary);
-    if (!ReplacedSource)
-      return std::unexpected(ReplacedSource.error());
-    auto Width = Extension.addOperand(MachineOperand::immediate(32));
-    if (!Width)
-      return std::unexpected(Width.error());
-    return {};
-  }
-
-  auto SetOpcode = Extension.setOpcode(opcode(Plan->Generic));
-  if (!SetOpcode)
-    return std::unexpected(SetOpcode.error());
-  while (Extension.operandCount() > 1) {
-    auto Erased = Extension.eraseOperand(1);
-    if (!Erased)
-      return std::unexpected(Erased.error());
-  }
-  for (Register Input : Plan->WideInputs) {
-    auto Operand =
-        MachineOperand::registerOperand(Input, RegisterOperandFlags{RegisterOperandFlag::Use});
-    if (!Operand)
-      return std::unexpected(Operand.error());
-    auto Added = Extension.addOperand(std::move(*Operand));
-    if (!Added)
-      return std::unexpected(Added.error());
-  }
-  MachineBasicBlock *BinaryBlock = Plan->Binary->parent();
-  if (BinaryBlock == nullptr)
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "RISC-V narrow binary instruction is detached during legalization");
-  auto Removed = BinaryBlock->removeInstruction(*Plan->Binary);
-  if (!Removed)
-    return std::unexpected(Removed.error());
-  (void)Target;
-  return {};
-}
-
 [[nodiscard]] Expected<void> lowerConstantToPool(MachineInstr &Instruction,
                                                  MachineFunction &Function,
                                                  const TargetDescription &Target) {
@@ -1097,63 +630,6 @@ lowerAtomicCompareExchangeWithSuccess(MachineInstr &Instruction, MachineFunction
   return {};
 }
 
-[[nodiscard]] bool isSupportedNonAtomicMemoryAccess(const MachineInstr &Instruction) noexcept {
-  if (Instruction.memOperands().size() != 1)
-    return false;
-  const MachineMemOperand &Memory = *Instruction.memOperands().front();
-  const MachineMemOperandDesc &Description = Memory.description();
-  return !Memory.isAtomic() && Description.AddressSpace == 0 && Description.Size.isKnown() &&
-         !Description.Size.isScalable() && Description.Size.minimumBytes() != 0;
-}
-
-[[nodiscard]] bool isSupportedAtomicMemoryAccess(const MachineInstr &Instruction,
-                                                 AtomicOperation Operation,
-                                                 std::uint64_t Bytes) noexcept {
-  if (Instruction.memOperands().size() != 1 || (Bytes != 4 && Bytes != 8))
-    return false;
-  const MachineMemOperand &Memory = *Instruction.memOperands().front();
-  const MachineMemOperandDesc &Description = Memory.description();
-  return Description.AtomicKind == Operation && Description.AddressSpace == 0 &&
-         Description.Size.isKnown() && !Description.Size.isScalable() &&
-         Description.Size.minimumBytes() == Bytes && Description.Align.bytes() >= Bytes &&
-         Description.SyncScope.value() == 0;
-}
-
-[[nodiscard]] bool isSupportedAtomicContainerAccess(const MachineInstr &Instruction,
-                                                    AtomicOperation Operation,
-                                                    std::uint64_t ContainerBytes) noexcept {
-  if (Instruction.memOperands().size() != 1 || ContainerBytes == 0)
-    return false;
-  const MachineMemOperandDesc &Description = Instruction.memOperands().front()->description();
-  if (Description.AtomicKind != Operation || Description.AddressSpace != 0 ||
-      !Description.Size.isKnown() || Description.Size.isScalable() ||
-      Description.SyncScope.value() != 0)
-    return false;
-  const std::uint64_t MemoryBytes = Description.Size.minimumBytes();
-  return (MemoryBytes == 1 || MemoryBytes == 2 || MemoryBytes == 4 || MemoryBytes == 8) &&
-         MemoryBytes <= ContainerBytes && Description.Align.bytes() >= MemoryBytes;
-}
-
-[[nodiscard]] std::optional<std::uint64_t>
-memoryAccessBytes(const MachineInstr &Instruction) noexcept {
-  if (Instruction.memOperands().size() != 1)
-    return std::nullopt;
-  const MachineMemOperandDesc &Description = Instruction.memOperands().front()->description();
-  if (!Description.Size.isKnown() || Description.Size.isScalable())
-    return std::nullopt;
-  return Description.Size.minimumBytes();
-}
-
-[[nodiscard]] bool memoryMatchesType(const MachineInstr &Instruction, LowLevelType Type) noexcept {
-  const auto Bytes = memoryAccessBytes(Instruction);
-  if (!Bytes || (!isSupportedIntegerType(Type) && !isSupportedFloatingScalar(Type)))
-    return false;
-  if (isSupportedFloatingScalar(Type))
-    return Type.scalarBitWidth() == 32 ? *Bytes == 4 : *Bytes == 8;
-  const std::uint64_t ValueBytes = (Type.scalarBitWidth() + 7U) / 8U;
-  return (*Bytes == 1 || *Bytes == 2 || *Bytes == 4 || *Bytes == 8) && *Bytes <= ValueBytes;
-}
-
 [[nodiscard]] Expected<std::int64_t> signedImmediate(const MachineOperand &Operand) {
   if (const std::int64_t *Value = Operand.getIf<std::int64_t>())
     return *Value;
@@ -1168,19 +644,6 @@ memoryAccessBytes(const MachineInstr &Instruction) noexcept {
   return std::bit_cast<std::int64_t>(Raw);
 }
 
-[[nodiscard]] Expected<void> replaceRegisterUses(MachineFunction &Function, Register OldReg,
-                                                 Register NewReg) {
-  std::vector<MachineOperand *> Uses;
-  for (MachineOperand &Operand : Function.registerInfo().useOperands(OldReg))
-    Uses.push_back(&Operand);
-  for (MachineOperand *Operand : Uses) {
-    auto Replaced = Operand->setRegister(NewReg);
-    if (!Replaced)
-      return std::unexpected(Replaced.error());
-  }
-  return {};
-}
-
 [[nodiscard]] Expected<void> eraseInstruction(MachineInstr &Instruction) {
   MachineBasicBlock *Block = Instruction.parent();
   if (Block == nullptr)
@@ -1189,67 +652,6 @@ memoryAccessBytes(const MachineInstr &Instruction) noexcept {
   return Block->removeInstruction(Instruction);
 }
 
-[[nodiscard]] bool allDefsUnused(const MachineInstr &Instruction, const MachineFunction &Function) {
-  for (const MachineOperand &Operand : Instruction.operands()) {
-    const Register *Reg = Operand.getIf<Register>();
-    if (Reg != nullptr && Operand.isDef() && Reg->isVirtual() &&
-        !Function.registerInfo().useEmpty(*Reg))
-      return false;
-  }
-  return true;
-}
-
-void eraseDeadArtifacts(std::span<MachineInstr *const> Candidates,
-                        const MachineFunction &Function) {
-  bool Changed = false;
-  do {
-    Changed = false;
-    for (auto It = Candidates.rbegin(); It != Candidates.rend(); ++It) {
-      MachineInstr *Instruction = *It;
-      if (Instruction == nullptr || Instruction->parent() == nullptr ||
-          !allDefsUnused(*Instruction, Function))
-        continue;
-      const Opcode Value = Instruction->opcode();
-      if (Value != opcode(GenericOpcode::G_TRUNC) && Value != opcode(GenericOpcode::G_ANYEXT) &&
-          Value != opcode(GenericOpcode::G_SEXT) && Value != opcode(GenericOpcode::G_ZEXT) &&
-          Value != opcode(GenericOpcode::G_MERGE_VALUES) &&
-          Value != opcode(GenericOpcode::G_UNMERGE_VALUES) && Value != opcode(GenericOpcode::COPY))
-        continue;
-      if (eraseInstruction(*Instruction))
-        Changed = true;
-    }
-  } while (Changed);
-}
-
-[[nodiscard]] Expected<Register> buildIntegerConstantBefore(MachineIRBuilder &Builder,
-                                                            MachineInstr &Anchor,
-                                                            std::uint32_t Width,
-                                                            std::uint64_t Value) {
-  auto Type = LowLevelType::makeCategoryNeutralScalar(Width);
-  if (!Type)
-    return std::unexpected(Type.error());
-  auto Result = Builder.function().registerInfo().createVirtualRegister(*Type);
-  if (!Result)
-    return std::unexpected(Result.error());
-  auto Immediate = APInt::fromUnsigned(Width, Value);
-  if (!Immediate)
-    return std::unexpected(Immediate.error());
-  auto Point = Builder.setInsertPointBefore(Anchor);
-  if (!Point)
-    return std::unexpected(Point.error());
-  auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_CONSTANT));
-  if (!Pending)
-    return std::unexpected(Pending.error());
-  if (auto Added = Pending->addDef(*Result); !Added)
-    return std::unexpected(Added.error());
-  if (auto Added = Pending->addIntegerImmediate(std::move(*Immediate)); !Added)
-    return std::unexpected(Added.error());
-  auto Built = Pending->build();
-  if (!Built)
-    return std::unexpected(Built.error());
-  return *Result;
-}
-
 [[nodiscard]] Expected<Register> buildUnaryBefore(MachineIRBuilder &Builder, MachineInstr &Anchor,
                                                   GenericOpcode Generic, LowLevelType Type,
                                                   Register Source) {
@@ -1272,30 +674,6 @@ void eraseDeadArtifacts(std::span<MachineInstr *const> Candidates,
   return *Result;
 }
 
-[[nodiscard]] Expected<Register> buildBinaryBefore(MachineIRBuilder &Builder, MachineInstr &Anchor,
-                                                   GenericOpcode Generic, LowLevelType Type,
-                                                   Register Left, Register Right) {
-  auto Result = Builder.function().registerInfo().createVirtualRegister(Type);
-  if (!Result)
-    return std::unexpected(Result.error());
-  auto Point = Builder.setInsertPointBefore(Anchor);
-  if (!Point)
-    return std::unexpected(Point.error());
-  auto Pending = Builder.buildInstr(opcode(Generic));
-  if (!Pending)
-    return std::unexpected(Pending.error());
-  if (auto Added = Pending->addDef(*Result); !Added)
-    return std::unexpected(Added.error());
-  if (auto Added = Pending->addUse(Left); !Added)
-    return std::unexpected(Added.error());
-  if (auto Added = Pending->addUse(Right); !Added)
-    return std::unexpected(Added.error());
-  auto Built = Pending->build();
-  if (!Built)
-    return std::unexpected(Built.error());
-  return *Result;
-}
-
 [[nodiscard]] Expected<void> rewriteUnary(MachineInstr &Instruction, GenericOpcode Generic,
                                           Register Destination, Register Source) {
   auto SetOpcode = Instruction.setOpcode(opcode(Generic));
@@ -1319,2469 +697,415 @@ void eraseDeadArtifacts(std::span<MachineInstr *const> Candidates,
   return Instruction.addOperand(std::move(*Use));
 }
 
-[[nodiscard]] Expected<void> widenNarrowMultiply(MachineInstr &Instruction,
-                                                 MachineFunction &Function,
-                                                 const TargetDescription &Target) {
-  const std::optional<LowLevelType> Type = resultType(Function, Instruction);
-  const MachineOperand *DestinationOperand = Instruction.operandAt(0);
-  const MachineOperand *LeftOperand = Instruction.operandAt(1);
-  const MachineOperand *RightOperand = Instruction.operandAt(2);
-  const Register *Destination =
-      DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
-  const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf<Register>();
-  const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf<Register>();
-  if (!Type || Destination == nullptr || Left == nullptr || Right == nullptr ||
-      Type->scalarBitWidth() >= 64) {
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "RISC-V narrow multiply has invalid operands");
-  }
-
-  MachineInstr *Extension = singleRegisterUse(Function, *Destination);
-  if (Extension != nullptr && Extension->opcode() == opcode(GenericOpcode::G_ANYEXT)) {
-    const auto Plan = narrowAnyExtBinaryPlan(*Extension, Function);
-    if (Plan && Plan->Binary == &Instruction)
-      return foldNarrowAnyExtBinary(*Extension, Function, Target);
-  }
-
-  auto WideType = LowLevelType::makeCategoryNeutralScalar(64);
-  if (!WideType)
-    return std::unexpected(WideType.error());
-  MachineIRBuilder Builder(Function, Target);
-  auto WideLeft = buildUnaryBefore(Builder, Instruction, GenericOpcode::G_ANYEXT, *WideType, *Left);
-  if (!WideLeft)
-    return std::unexpected(WideLeft.error());
-  auto WideRight =
-      buildUnaryBefore(Builder, Instruction, GenericOpcode::G_ANYEXT, *WideType, *Right);
-  if (!WideRight)
-    return std::unexpected(WideRight.error());
-  auto Product = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_MUL, *WideType, *WideLeft,
-                                   *WideRight);
-  if (!Product)
-    return std::unexpected(Product.error());
-  return rewriteUnary(Instruction, GenericOpcode::G_TRUNC, *Destination, *Product);
-}
-
-[[nodiscard]] Expected<void> lowerMultiplyLibcall(MachineInstr &Instruction,
-                                                  MachineFunction &Function,
-                                                  const TargetDescription &Target,
-                                                  CallingConvention Convention,
-                                                  const TargetCallLowering &CallLowering) {
-  const std::optional<LowLevelType> Type = resultType(Function, Instruction);
-  const MachineOperand *DestinationOperand = Instruction.operandAt(0);
-  const MachineOperand *LeftOperand = Instruction.operandAt(1);
-  const MachineOperand *RightOperand = Instruction.operandAt(2);
-  const Register *Destination =
-      DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
-  const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf<Register>();
-  const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf<Register>();
-  if (!Type || Destination == nullptr || Left == nullptr || Right == nullptr ||
-      Type->scalarBitWidth() != 64) {
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "RISC-V multiply libcall requires three s64 registers");
-  }
-  auto ABIAlignment = Alignment::create(8);
-  auto Callee = MachineCallTarget::externalSymbol("__muldi3");
-  if (!ABIAlignment || !Callee)
-    return makeUnexpected(ErrorCode::InternalError,
-                          "RISC-V multiply libcall ABI facts are unavailable");
-  const auto Value = [Type = *Type, Alignment = *ABIAlignment](Register Reg, std::uint32_t Number) {
-    return ABIValue{
-        .Value = Reg,
-        .Type = Type,
-        .Category = ABIValueCategory::Integer,
-        .OriginalValue = Number,
-        .FragmentIndex = 0,
-        .FragmentCount = 1,
-        .FragmentOffsetBytes = 0,
-        .OriginalAlignment = Alignment,
-        .Extension = ABIExtensionKind::None,
-        .Flags = {},
-        .ByValSizeBytes = 0,
-        .ByValAlignment = Alignment,
-    };
-  };
-
-  CallLoweringRequest Request;
-  Request.Convention = callingConventionID(Convention);
-  Request.Callee = std::move(*Callee);
-  Request.Arguments = {Value(*Left, 0), Value(*Right, 1)};
-  Request.Results = {Value(*Destination, 0)};
-  Request.FixedArgumentCount = Request.Arguments.size();
-
-  MachineIRBuilder Builder(Function, Target);
-  auto Point = Builder.setInsertPointBefore(Instruction);
-  if (!Point)
-    return std::unexpected(Point.error());
-  auto Lowered = CallLowering.lowerCall(Builder, Request);
-  if (!Lowered)
-    return std::unexpected(Lowered.error());
-  return eraseInstruction(Instruction);
-}
-
-[[nodiscard]] Expected<void> rewriteBinary(MachineInstr &Instruction, GenericOpcode Generic,
-                                           Register Destination, Register Left, Register Right) {
-  auto Rewritten = rewriteUnary(Instruction, Generic, Destination, Left);
-  if (!Rewritten)
-    return std::unexpected(Rewritten.error());
-  auto Use = MachineOperand::registerOperand(Right, RegisterOperandFlags{RegisterOperandFlag::Use});
-  if (!Use)
-    return std::unexpected(Use.error());
-  return Instruction.addOperand(std::move(*Use));
-}
-
-[[nodiscard]] bool isKnownZeroExtended(const MachineFunction &Function, Register Reg,
-                                       std::uint32_t ActiveBits, unsigned Depth = 0) {
-  if (Depth > 8 || !Reg.isVirtual())
-    return false;
-  const auto Type = registerType(Function, Reg);
-  if (!Type)
-    return false;
-  const std::uint32_t Width = Type->scalarBitWidth();
-  if (Width <= ActiveBits)
-    return true;
-  const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(Reg);
-  const MachineInstr *Def = Definition == nullptr ? nullptr : Definition->parent();
-  if (Def == nullptr)
-    return false;
-  if (Def->opcode() == opcode(GenericOpcode::G_ASSERT_ZEXT)) {
-    const MachineOperand *BitsOperand = Def->operandAt(2);
-    auto Bits = BitsOperand == nullptr
-                    ? Expected<std::int64_t>(std::unexpected(
-                          Error(ErrorCode::InvalidArgument, "missing asserted extension width")))
-                    : signedImmediate(*BitsOperand);
-    return Bits && *Bits >= 0 && static_cast<std::uint64_t>(*Bits) <= ActiveBits;
-  }
-  if (Def->opcode() == opcode(GenericOpcode::G_ZEXT)) {
-    const MachineOperand *SourceOperand = Def->operandAt(1);
-    const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>();
-    if (Source == nullptr)
-      return false;
-    const auto SourceType = registerType(Function, *Source);
-    return SourceType && SourceType->scalarBitWidth() <= ActiveBits;
-  }
-  if (Def->opcode() == opcode(GenericOpcode::G_TRUNC)) {
-    const MachineOperand *SourceOperand = Def->operandAt(1);
-    const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>();
-    return Source != nullptr && isKnownZeroExtended(Function, *Source, ActiveBits, Depth + 1U);
-  }
-  return false;
-}
-
-[[nodiscard]] Expected<Register> zeroExtendToRV64(MachineIRBuilder &Builder, MachineInstr &Anchor,
-                                                  MachineFunction &Function, Register Source,
-                                                  std::vector<MachineInstr *> &DeadCandidates) {
-  const auto SourceType = registerType(Function, Source);
-  if (!SourceType)
-    return makeUnexpected(ErrorCode::VerificationError, "RISC-V merge source has no scalar type");
-  if (SourceType->scalarBitWidth() == 64)
-    return Source;
-  const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(Source);
-  MachineInstr *Def =
-      Definition == nullptr ? nullptr : const_cast<MachineInstr *>(Definition->parent());
-  if (Def != nullptr && Def->opcode() == opcode(GenericOpcode::G_TRUNC)) {
-    const MachineOperand *WideOperand = Def->operandAt(1);
-    const Register *Wide = WideOperand == nullptr ? nullptr : WideOperand->getIf<Register>();
-    if (Wide != nullptr) {
-      const auto WideType = registerType(Function, *Wide);
-      if (WideType && WideType->scalarBitWidth() == 64 &&
-          isKnownZeroExtended(Function, *Wide, SourceType->scalarBitWidth())) {
-        DeadCandidates.push_back(Def);
-        return *Wide;
-      }
-    }
-  }
-  auto WideType = LowLevelType::makeCategoryNeutralScalar(64);
-  if (!WideType)
-    return std::unexpected(WideType.error());
-  return buildUnaryBefore(Builder, Anchor, GenericOpcode::G_ZEXT, *WideType, Source);
-}
-
-[[nodiscard]] MachineInstr *singleUseInstruction(const MachineFunction &Function, Register Reg) {
-  MachineInstr *Result = nullptr;
-  auto &RegisterInfo = const_cast<MachineRegisterInfo &>(Function.registerInfo());
-  for (MachineOperand &Operand : RegisterInfo.useOperands(Reg)) {
-    MachineInstr *Instruction = Operand.parent();
-    if (Instruction == nullptr || Instruction->parent() == nullptr)
-      continue;
-    if (Result != nullptr && Result != Instruction)
-      return nullptr;
-    Result = Instruction;
-  }
-  return Result;
-}
-
-[[nodiscard]] Expected<void> lowerWideMergeValues(MachineInstr &Instruction,
-                                                  MachineFunction &Function) {
-  const MachineOperand *DestinationOperand = Instruction.operandAt(0);
-  const Register *Destination =
-      DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
-  const MachineOperand *LowOperand = Instruction.operandAt(1);
-  const Register *Low = LowOperand == nullptr ? nullptr : LowOperand->getIf<Register>();
-  if (Destination == nullptr || Low == nullptr)
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "RISC-V wide merge has invalid register operands");
-
-  std::vector<MachineInstr *> Truncations;
-  auto &RegisterInfo = Function.registerInfo();
-  std::vector<MachineOperand *> Uses;
-  for (MachineOperand &Use : RegisterInfo.useOperands(*Destination))
-    Uses.push_back(&Use);
-  for (MachineOperand *Use : Uses) {
-    MachineInstr *Trunc = Use->parent();
-    if (Trunc == nullptr || Trunc->parent() == nullptr ||
-        Trunc->opcode() != opcode(GenericOpcode::G_TRUNC) || Trunc->operandAt(1) != Use)
-      return makeUnexpected(ErrorCode::Unsupported,
-                            "RISC-V wide merge is only representable through low-part truncation");
-    const MachineOperand *TruncResultOperand = Trunc->operandAt(0);
-    const Register *TruncResult =
-        TruncResultOperand == nullptr ? nullptr : TruncResultOperand->getIf<Register>();
-    if (TruncResult == nullptr)
-      return makeUnexpected(ErrorCode::VerificationError,
-                            "RISC-V wide merge truncation has no result");
-    const auto LowType = registerType(Function, *Low);
-    const auto ResultType = registerType(Function, *TruncResult);
-    if (!LowType || !ResultType || ResultType->scalarBitWidth() > LowType->scalarBitWidth())
-      return makeUnexpected(ErrorCode::Unsupported,
-                            "RISC-V wide merge truncation spans more than its low source");
-    if (*ResultType == *LowType) {
-      auto Replaced = replaceRegisterUses(Function, *TruncResult, *Low);
-      if (!Replaced)
-        return std::unexpected(Replaced.error());
-      Truncations.push_back(Trunc);
-    } else {
-      auto Replaced = Use->setRegister(*Low);
-      if (!Replaced)
-        return std::unexpected(Replaced.error());
-    }
-  }
-  for (MachineInstr *Trunc : Truncations) {
-    auto Erased = eraseInstruction(*Trunc);
-    if (!Erased)
-      return std::unexpected(Erased.error());
-  }
-  return eraseInstruction(Instruction);
-}
-
-[[nodiscard]] Expected<void> lowerMergeValues(MachineInstr &Instruction, MachineFunction &Function,
-                                              const TargetDescription &Target) {
-  const std::optional<LowLevelType> DestinationType = resultType(Function, Instruction);
-  if (!DestinationType || DestinationType->scalarBitWidth() == 0)
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "RISC-V merge destination has no scalar type");
-  if (DestinationType->scalarBitWidth() > 64)
-    return lowerWideMergeValues(Instruction, Function);
-
-  const MachineOperand *DestinationOperand = Instruction.operandAt(0);
-  const Register *Destination =
-      DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
-  if (Destination == nullptr || Instruction.operandCount() < 3)
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "RISC-V merge has invalid register operands");
-
-  Register Result = *Destination;
-  MachineInstr *FoldedExtension = nullptr;
-  if (DestinationType->scalarBitWidth() < 64) {
-    MachineInstr *Use = singleUseInstruction(Function, *Destination);
-    if (Use != nullptr && Use->opcode() == opcode(GenericOpcode::G_ZEXT) &&
-        operandType(Function, *Use, 0)
-            .transform([](LowLevelType Type) { return Type.scalarBitWidth() == 64; })
-            .value_or(false)) {
-      const MachineOperand *ExtendedOperand = Use->operandAt(0);
-      const Register *Extended =
-          ExtendedOperand == nullptr ? nullptr : ExtendedOperand->getIf<Register>();
-      if (Extended != nullptr) {
-        Result = *Extended;
-        FoldedExtension = Use;
-      }
-    }
-  }
-
-  auto WideType = LowLevelType::makeCategoryNeutralScalar(64);
-  if (!WideType)
-    return std::unexpected(WideType.error());
-  MachineIRBuilder Builder(Function, Target);
-  std::vector<MachineInstr *> DeadCandidates;
-  std::vector<Register> Terms;
-  Terms.reserve(Instruction.operandCount() - 1U);
-  for (std::size_t Index = 1; Index < Instruction.operandCount(); ++Index) {
-    const MachineOperand *Operand = Instruction.operandAt(Index);
-    const Register *Source = Operand == nullptr ? nullptr : Operand->getIf<Register>();
-    if (Source == nullptr)
-      return makeUnexpected(ErrorCode::VerificationError, "RISC-V merge source is not a register");
-    auto WideSource = zeroExtendToRV64(Builder, Instruction, Function, *Source, DeadCandidates);
-    if (!WideSource)
-      return std::unexpected(WideSource.error());
-    if (Index == 1) {
-      Terms.push_back(*WideSource);
-      continue;
-    }
-    const std::optional<LowLevelType> SourceType = registerType(Function, *Operand);
-    if (!SourceType)
-      return makeUnexpected(ErrorCode::VerificationError, "RISC-V merge source has no scalar type");
-    const std::uint64_t Shift =
-        static_cast<std::uint64_t>(SourceType->scalarBitWidth()) * (Index - 1U);
-    auto Amount = buildIntegerConstantBefore(Builder, Instruction, 64, Shift);
-    if (!Amount)
-      return std::unexpected(Amount.error());
-    auto Shifted = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_SHL, *WideType,
-                                     *WideSource, *Amount);
-    if (!Shifted)
-      return std::unexpected(Shifted.error());
-    Terms.push_back(*Shifted);
-  }
-
-  Register Accumulator = Terms.front();
-  for (std::size_t Index = 1; Index + 1U < Terms.size(); ++Index) {
-    auto Combined = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_OR, *WideType,
-                                      Accumulator, Terms[Index]);
-    if (!Combined)
-      return std::unexpected(Combined.error());
-    Accumulator = *Combined;
-  }
-  if (DestinationType->scalarBitWidth() == 64 || FoldedExtension != nullptr) {
-    auto Rewritten =
-        rewriteBinary(Instruction, GenericOpcode::G_OR, Result, Accumulator, Terms.back());
-    if (!Rewritten)
-      return std::unexpected(Rewritten.error());
-  } else {
-    auto Combined = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_OR, *WideType,
-                                      Accumulator, Terms.back());
-    if (!Combined)
-      return std::unexpected(Combined.error());
-    auto Rewritten = rewriteUnary(Instruction, GenericOpcode::G_TRUNC, *Destination, *Combined);
-    if (!Rewritten)
-      return std::unexpected(Rewritten.error());
-  }
-  if (FoldedExtension != nullptr) {
-    auto Erased = eraseInstruction(*FoldedExtension);
-    if (!Erased)
-      return std::unexpected(Erased.error());
-  }
-  eraseDeadArtifacts(DeadCandidates, Function);
-  return {};
-}
-
-[[nodiscard]] Expected<void> foldWideExtendedUnmerge(MachineInstr &Instruction,
-                                                     MachineFunction &Function) {
-  const std::size_t OutputCount = Instruction.operandCount() - 1U;
-  const MachineOperand *SourceOperand = Instruction.operandAt(OutputCount);
-  const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>();
-  const MachineOperand *Definition =
-      Source == nullptr ? nullptr : Function.registerInfo().getUniqueVRegDef(*Source);
-  MachineInstr *Extension =
-      Definition == nullptr ? nullptr : const_cast<MachineInstr *>(Definition->parent());
-  if (Extension == nullptr || (Extension->opcode() != opcode(GenericOpcode::G_ZEXT) &&
-                               Extension->opcode() != opcode(GenericOpcode::G_ANYEXT) &&
-                               Extension->opcode() != opcode(GenericOpcode::G_SEXT)))
-    return makeUnexpected(ErrorCode::Unsupported,
-                          "RISC-V wide unmerge requires an extension-defined source");
-  const MachineOperand *NarrowOperand = Extension->operandAt(1);
-  const Register *Narrow = NarrowOperand == nullptr ? nullptr : NarrowOperand->getIf<Register>();
-  if (Narrow == nullptr)
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "RISC-V wide unmerge extension has no register source");
-  const auto NarrowType = registerType(Function, *Narrow);
-  if (!NarrowType)
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "RISC-V wide unmerge extension source has no type");
-  const std::uint32_t NarrowWidth = NarrowType->scalarBitWidth();
-
-  std::vector<MachineInstr *> DeadUsers;
-  for (std::size_t Index = 0; Index < OutputCount; ++Index) {
-    const MachineOperand *OutputOperand = Instruction.operandAt(Index);
-    const Register *Output = OutputOperand == nullptr ? nullptr : OutputOperand->getIf<Register>();
-    if (Output == nullptr || Function.registerInfo().useEmpty(*Output))
-      continue;
-    const auto OutputType = registerType(Function, *Output);
-    if (!OutputType)
-      return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide unmerge output has no type");
-    const std::uint64_t Offset = static_cast<std::uint64_t>(Index) * OutputType->scalarBitWidth();
-    if (Offset != 0)
-      return makeUnexpected(ErrorCode::Unsupported,
-                            "RISC-V wide unmerge cannot materialize a used high wide part");
-    if (OutputType->scalarBitWidth() == NarrowWidth) {
-      auto Replaced = replaceRegisterUses(Function, *Output, *Narrow);
-      if (!Replaced)
-        return std::unexpected(Replaced.error());
-      continue;
-    }
-    if (OutputType->scalarBitWidth() < NarrowWidth)
-      return makeUnexpected(ErrorCode::Unsupported,
-                            "RISC-V wide unmerge output unexpectedly narrows its source");
-    std::vector<MachineOperand *> Uses;
-    for (MachineOperand &Use : Function.registerInfo().useOperands(*Output))
-      Uses.push_back(&Use);
-    for (MachineOperand *Use : Uses) {
-      MachineInstr *Trunc = Use->parent();
-      if (Trunc == nullptr || Trunc->parent() == nullptr ||
-          Trunc->opcode() != opcode(GenericOpcode::G_TRUNC) || Trunc->operandAt(1) != Use)
-        return makeUnexpected(
-            ErrorCode::Unsupported,
-            "RISC-V wide unmerge result is only representable through truncation");
-      const MachineOperand *ResultOperand = Trunc->operandAt(0);
-      const Register *Result =
-          ResultOperand == nullptr ? nullptr : ResultOperand->getIf<Register>();
-      if (Result == nullptr)
-        return makeUnexpected(ErrorCode::VerificationError,
-                              "RISC-V wide unmerge truncation has no result");
-      const auto ResultType = registerType(Function, *Result);
-      if (!ResultType || ResultType->scalarBitWidth() > NarrowWidth)
-        return makeUnexpected(ErrorCode::Unsupported,
-                              "RISC-V wide unmerge truncation exceeds its source width");
-      if (ResultType->scalarBitWidth() == NarrowWidth) {
-        auto Replaced = replaceRegisterUses(Function, *Result, *Narrow);
-        if (!Replaced)
-          return std::unexpected(Replaced.error());
-        DeadUsers.push_back(Trunc);
-      } else {
-        auto Replaced = Use->setRegister(*Narrow);
-        if (!Replaced)
-          return std::unexpected(Replaced.error());
-      }
-    }
-  }
-  for (MachineInstr *User : DeadUsers) {
-    auto Erased = eraseInstruction(*User);
-    if (!Erased)
-      return std::unexpected(Erased.error());
-  }
-  auto Erased = eraseInstruction(Instruction);
-  if (!Erased)
-    return std::unexpected(Erased.error());
-  if (Function.registerInfo().useEmpty(*Source))
-    return eraseInstruction(*Extension);
-  return {};
-}
-
-[[nodiscard]] Expected<void> lowerUnmergeValues(MachineInstr &Instruction,
-                                                MachineFunction &Function,
-                                                const TargetDescription &Target) {
-  if (Instruction.operandCount() < 3)
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "RISC-V unmerge has invalid register operands");
-  const std::size_t OutputCount = Instruction.operandCount() - 1U;
-  const std::optional<LowLevelType> OutputType = operandType(Function, Instruction, 0);
-  const std::optional<LowLevelType> SourceType = operandType(Function, Instruction, OutputCount);
-  if (!OutputType || !SourceType)
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "RISC-V unmerge operands have no scalar types");
-  if (SourceType->scalarBitWidth() > 64 || OutputType->scalarBitWidth() > 64)
-    return foldWideExtendedUnmerge(Instruction, Function);
-
-  const MachineOperand *SourceOperand = Instruction.operandAt(OutputCount);
-  const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>();
-  if (Source == nullptr)
-    return makeUnexpected(ErrorCode::VerificationError, "RISC-V unmerge source is not a register");
-  auto WideType = LowLevelType::makeCategoryNeutralScalar(64);
-  if (!WideType)
-    return std::unexpected(WideType.error());
-  MachineIRBuilder Builder(Function, Target);
-  std::vector<MachineInstr *> FoldedExtensions;
-  for (std::size_t Index = 0; Index < OutputCount; ++Index) {
-    const MachineOperand *OutputOperand = Instruction.operandAt(Index);
-    const Register *Output = OutputOperand == nullptr ? nullptr : OutputOperand->getIf<Register>();
-    if (Output == nullptr || Function.registerInfo().useEmpty(*Output))
-      continue;
-    Register Extracted = *Source;
-    const std::uint64_t Offset = static_cast<std::uint64_t>(Index) * OutputType->scalarBitWidth();
-    if (Offset != 0) {
-      auto Amount = buildIntegerConstantBefore(Builder, Instruction, 64, Offset);
-      if (!Amount)
-        return std::unexpected(Amount.error());
-      auto Shifted = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_LSHR, *WideType,
-                                       *Source, *Amount);
-      if (!Shifted)
-        return std::unexpected(Shifted.error());
-      Extracted = *Shifted;
-    }
-
-    MachineInstr *Extension = singleUseInstruction(Function, *Output);
-    const bool FoldZExt =
-        Extension != nullptr && Extension->opcode() == opcode(GenericOpcode::G_ZEXT) &&
-        operandType(Function, *Extension, 0)
-            .transform([](LowLevelType Type) { return Type.scalarBitWidth() == 64; })
-            .value_or(false);
-    if (FoldZExt) {
-      const MachineOperand *ExtendedOperand = Extension->operandAt(0);
-      const Register *Extended =
-          ExtendedOperand == nullptr ? nullptr : ExtendedOperand->getIf<Register>();
-      if (Extended == nullptr)
-        return makeUnexpected(ErrorCode::VerificationError,
-                              "RISC-V unmerge extension has no destination");
-      const std::uint64_t Mask = OutputType->scalarBitWidth() == 64
-                                     ? std::numeric_limits<std::uint64_t>::max()
-                                     : (std::uint64_t{1} << OutputType->scalarBitWidth()) - 1U;
-      auto MaskReg = buildIntegerConstantBefore(Builder, Instruction, 64, Mask);
-      if (!MaskReg)
-        return std::unexpected(MaskReg.error());
-      auto Masked = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_AND, *WideType,
-                                      Extracted, *MaskReg);
-      if (!Masked)
-        return std::unexpected(Masked.error());
-      auto Replaced = replaceRegisterUses(Function, *Extended, *Masked);
-      if (!Replaced)
-        return std::unexpected(Replaced.error());
-      FoldedExtensions.push_back(Extension);
-      continue;
-    }
-    auto Truncated =
-        buildUnaryBefore(Builder, Instruction, GenericOpcode::G_TRUNC, *OutputType, Extracted);
-    if (!Truncated)
-      return std::unexpected(Truncated.error());
-    auto Replaced = replaceRegisterUses(Function, *Output, *Truncated);
-    if (!Replaced)
-      return std::unexpected(Replaced.error());
-  }
-  for (MachineInstr *Extension : FoldedExtensions) {
-    auto Erased = eraseInstruction(*Extension);
-    if (!Erased)
-      return std::unexpected(Erased.error());
-  }
-  return eraseInstruction(Instruction);
-}
-
-struct WideValueParts final {
-  std::uint32_t BitWidth = 0;
-  std::vector<Register> Parts;
-  std::vector<MachineInstr *> Artifacts;
-};
-
-[[nodiscard]] Expected<WideValueParts>
-collectWideValueParts(Register Value, MachineFunction &Function, unsigned Depth = 0) {
-  if (Depth > 16)
-    return makeUnexpected(ErrorCode::Unsupported, "RISC-V wide-value artifact chain is too deep");
-  const auto Type = registerType(Function, Value);
-  if (!Type)
-    return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide value has no scalar type");
-  if (!Type->isCategoryNeutralScalar() || Type->scalarBitWidth() == 0)
-    return makeUnexpected(ErrorCode::Unsupported,
-                          "RISC-V wide-value decomposition requires an integer scalar");
-  if (Type->scalarBitWidth() <= 64)
-    return WideValueParts{.BitWidth = Type->scalarBitWidth(), .Parts = {Value}, .Artifacts = {}};
-
-  const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(Value);
-  MachineInstr *Def =
-      Definition == nullptr ? nullptr : const_cast<MachineInstr *>(Definition->parent());
-  if (Def == nullptr)
-    return makeUnexpected(ErrorCode::Unsupported,
-                          "RISC-V wide value has no decomposable definition");
-  if (Def->opcode() == opcode(GenericOpcode::G_MERGE_VALUES)) {
-    WideValueParts Result;
-    Result.BitWidth = Type->scalarBitWidth();
-    for (std::size_t Index = 1; Index < Def->operandCount(); ++Index) {
-      const MachineOperand *Operand = Def->operandAt(Index);
-      const Register *Part = Operand == nullptr ? nullptr : Operand->getIf<Register>();
-      const std::optional<LowLevelType> PartType =
-          Operand == nullptr ? std::nullopt : registerType(Function, *Operand);
-      if (Part == nullptr || !PartType || PartType->scalarBitWidth() != 64)
-        return makeUnexpected(ErrorCode::Unsupported,
-                              "RISC-V wide merge must already use 64-bit limbs");
-      Result.Parts.push_back(*Part);
-    }
-    Result.Artifacts.push_back(Def);
-    return Result;
-  }
-  if (Def->opcode() == opcode(GenericOpcode::G_TRUNC)) {
-    const MachineOperand *SourceOperand = Def->operandAt(1);
-    const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>();
-    if (Source == nullptr)
-      return makeUnexpected(ErrorCode::VerificationError,
-                            "RISC-V wide truncation has no source register");
-    auto Result = collectWideValueParts(*Source, Function, Depth + 1U);
-    if (!Result)
-      return std::unexpected(Result.error());
-    Result->BitWidth = Type->scalarBitWidth();
-    const std::size_t RequiredParts = (Type->scalarBitWidth() + 63U) / 64U;
-    if (Result->Parts.size() < RequiredParts)
-      return makeUnexpected(ErrorCode::VerificationError,
-                            "RISC-V wide truncation source has too few limbs");
-    Result->Parts.resize(RequiredParts);
-    Result->Artifacts.push_back(Def);
-    return Result;
-  }
-  return makeUnexpected(ErrorCode::Unsupported,
-                        "RISC-V wide value is not defined by merge/trunc artifacts");
-}
-
-struct WideResultPlan final {
-  std::vector<Register> Outputs;
-  MachineInstr *Extension = nullptr;
-  MachineInstr *Unmerge = nullptr;
-};
-
-[[nodiscard]] Expected<WideResultPlan> wideResultPlan(Register Result, MachineFunction &Function,
-                                                      std::size_t RequiredParts) {
-  MachineInstr *Use = singleUseInstruction(Function, Result);
-  WideResultPlan Plan;
-  if (Use != nullptr && (Use->opcode() == opcode(GenericOpcode::G_ANYEXT) ||
-                         Use->opcode() == opcode(GenericOpcode::G_SEXT) ||
-                         Use->opcode() == opcode(GenericOpcode::G_ZEXT))) {
-    Plan.Extension = Use;
-    const MachineOperand *ExtendedOperand = Use->operandAt(0);
-    const Register *Extended =
-        ExtendedOperand == nullptr ? nullptr : ExtendedOperand->getIf<Register>();
-    Use = Extended == nullptr ? nullptr : singleUseInstruction(Function, *Extended);
-  }
-  if (Use == nullptr || Use->opcode() != opcode(GenericOpcode::G_UNMERGE_VALUES))
-    return makeUnexpected(ErrorCode::Unsupported,
-                          "RISC-V wide result must feed an unmerge artifact");
-  Plan.Unmerge = Use;
-  const std::size_t OutputCount = Use->operandCount() - 1U;
-  if (OutputCount < RequiredParts)
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "RISC-V wide result unmerge has too few outputs");
-  Plan.Outputs.reserve(RequiredParts);
-  for (std::size_t Index = 0; Index < RequiredParts; ++Index) {
-    const MachineOperand *Operand = Use->operandAt(Index);
-    const Register *Output = Operand == nullptr ? nullptr : Operand->getIf<Register>();
-    const std::optional<LowLevelType> Type =
-        Operand == nullptr ? std::nullopt : registerType(Function, *Operand);
-    if (Output == nullptr || !Type || Type->scalarBitWidth() != 64)
-      return makeUnexpected(ErrorCode::Unsupported,
-                            "RISC-V wide result must unmerge to 64-bit limbs");
-    Plan.Outputs.push_back(*Output);
-  }
-  return Plan;
-}
-
-[[nodiscard]] Expected<Register> buildIntegerCompareBefore(MachineIRBuilder &Builder,
-                                                           MachineInstr &Anchor,
-                                                           IntegerPredicate Predicate,
-                                                           Register Left, Register Right) {
-  auto Type = LowLevelType::makeCategoryNeutralScalar(64);
-  if (!Type)
-    return std::unexpected(Type.error());
-  auto Result = Builder.function().registerInfo().createVirtualRegister(*Type);
-  if (!Result)
-    return std::unexpected(Result.error());
-  auto Point = Builder.setInsertPointBefore(Anchor);
-  if (!Point)
-    return std::unexpected(Point.error());
-  auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_ICMP));
-  if (!Pending)
-    return std::unexpected(Pending.error());
-  if (auto Added = Pending->addDef(*Result); !Added)
-    return std::unexpected(Added.error());
-  if (auto Added = Pending->addIntegerPredicate(Predicate); !Added)
-    return std::unexpected(Added.error());
-  if (auto Added = Pending->addUse(Left); !Added)
-    return std::unexpected(Added.error());
-  if (auto Added = Pending->addUse(Right); !Added)
-    return std::unexpected(Added.error());
-  auto Built = Pending->build();
-  if (!Built)
-    return std::unexpected(Built.error());
-  return *Result;
-}
-
-[[nodiscard]] Expected<Register> buildSelectBefore(MachineIRBuilder &Builder, MachineInstr &Anchor,
-                                                   LowLevelType Type, Register Condition,
-                                                   Register TrueValue, Register FalseValue) {
-  auto Result = Builder.function().registerInfo().createVirtualRegister(Type);
-  if (!Result)
-    return std::unexpected(Result.error());
-  auto Point = Builder.setInsertPointBefore(Anchor);
-  if (!Point)
-    return std::unexpected(Point.error());
-  auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_SELECT));
-  if (!Pending)
-    return std::unexpected(Pending.error());
-  if (auto Added = Pending->addDef(*Result); !Added)
-    return std::unexpected(Added.error());
-  if (auto Added = Pending->addUse(Condition); !Added)
-    return std::unexpected(Added.error());
-  if (auto Added = Pending->addUse(TrueValue); !Added)
-    return std::unexpected(Added.error());
-  if (auto Added = Pending->addUse(FalseValue); !Added)
-    return std::unexpected(Added.error());
-  auto Built = Pending->build();
-  if (!Built)
-    return std::unexpected(Built.error());
-  return *Result;
-}
-
-struct TwoLimbShiftResult final {
-  Register Low;
-  Register High;
-};
-
-[[nodiscard]] Expected<TwoLimbShiftResult>
-emitTwoLimbLeftShiftBody(MachineIRBuilder &Builder, MachineInstr &Anchor, Register Low,
-                         Register High, Register Amount, Register AmountExcess, Register AmountLack,
-                         Register Zero, Register IsShort, Register IsZero) {
-  auto Type = LowLevelType::makeCategoryNeutralScalar(64);
-  if (!Type)
-    return std::unexpected(Type.error());
-  auto LowShort = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SHL, *Type, Low, Amount);
-  if (!LowShort)
-    return std::unexpected(LowShort.error());
-  auto CrossLow = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_LSHR, *Type, Low, AmountLack);
-  if (!CrossLow)
-    return std::unexpected(CrossLow.error());
-  auto HighShort = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SHL, *Type, High, Amount);
-  if (!HighShort)
-    return std::unexpected(HighShort.error());
-  auto ShortHigh =
-      buildBinaryBefore(Builder, Anchor, GenericOpcode::G_OR, *Type, *CrossLow, *HighShort);
-  if (!ShortHigh)
-    return std::unexpected(ShortHigh.error());
-  auto LongHigh =
-      buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SHL, *Type, Low, AmountExcess);
-  if (!LongHigh)
-    return std::unexpected(LongHigh.error());
-  auto ResultLow = buildSelectBefore(Builder, Anchor, *Type, IsShort, *LowShort, Zero);
-  if (!ResultLow)
-    return std::unexpected(ResultLow.error());
-  auto ShiftedHigh = buildSelectBefore(Builder, Anchor, *Type, IsShort, *ShortHigh, *LongHigh);
-  if (!ShiftedHigh)
-    return std::unexpected(ShiftedHigh.error());
-  auto ResultHigh = buildSelectBefore(Builder, Anchor, *Type, IsZero, High, *ShiftedHigh);
-  if (!ResultHigh)
-    return std::unexpected(ResultHigh.error());
-  return TwoLimbShiftResult{.Low = *ResultLow, .High = *ResultHigh};
-}
-
-[[nodiscard]] Expected<TwoLimbShiftResult> emitTwoLimbLeftShift(MachineIRBuilder &Builder,
-                                                                MachineInstr &Anchor, Register Low,
-                                                                Register High, Register Amount) {
-  auto Type = LowLevelType::makeCategoryNeutralScalar(64);
-  if (!Type)
-    return std::unexpected(Type.error());
-  auto Boundary = buildIntegerConstantBefore(Builder, Anchor, 64, 64);
-  if (!Boundary)
-    return std::unexpected(Boundary.error());
-  auto AmountExcess =
-      buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, Amount, *Boundary);
-  if (!AmountExcess)
-    return std::unexpected(AmountExcess.error());
-  auto AmountLack =
-      buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, *Boundary, Amount);
-  if (!AmountLack)
-    return std::unexpected(AmountLack.error());
-  auto Zero = buildIntegerConstantBefore(Builder, Anchor, 64, 0);
-  if (!Zero)
-    return std::unexpected(Zero.error());
-  auto IsShort = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan,
-                                           Amount, *Boundary);
-  if (!IsShort)
-    return std::unexpected(IsShort.error());
-  auto IsZero = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::Equal, Amount, *Zero);
-  if (!IsZero)
-    return std::unexpected(IsZero.error());
-  return emitTwoLimbLeftShiftBody(Builder, Anchor, Low, High, Amount, *AmountExcess, *AmountLack,
-                                  *Zero, *IsShort, *IsZero);
-}
-
-[[nodiscard]] Expected<Register>
-emitLowLimbOfTwoLimbLogicalRightShift(MachineIRBuilder &Builder, MachineInstr &Anchor, Register Low,
-                                      Register High, Register Amount, Register Boundary,
-                                      Register Zero) {
-  auto Type = LowLevelType::makeCategoryNeutralScalar(64);
-  if (!Type)
-    return std::unexpected(Type.error());
-  auto AmountExcess =
-      buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, Amount, Boundary);
-  if (!AmountExcess)
-    return std::unexpected(AmountExcess.error());
-  auto AmountLack =
-      buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, Boundary, Amount);
-  if (!AmountLack)
-    return std::unexpected(AmountLack.error());
-  auto IsShort = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan,
-                                           Amount, Boundary);
-  if (!IsShort)
-    return std::unexpected(IsShort.error());
-  auto IsZero = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::Equal, Amount, Zero);
-  if (!IsZero)
-    return std::unexpected(IsZero.error());
-  auto LowShort = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_LSHR, *Type, Low, Amount);
-  if (!LowShort)
-    return std::unexpected(LowShort.error());
-  auto HighShort =
-      buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SHL, *Type, High, *AmountLack);
-  if (!HighShort)
-    return std::unexpected(HighShort.error());
-  auto ShortValue =
-      buildBinaryBefore(Builder, Anchor, GenericOpcode::G_OR, *Type, *LowShort, *HighShort);
-  if (!ShortValue)
-    return std::unexpected(ShortValue.error());
-  auto LongValue =
-      buildBinaryBefore(Builder, Anchor, GenericOpcode::G_LSHR, *Type, High, *AmountExcess);
-  if (!LongValue)
-    return std::unexpected(LongValue.error());
-  auto Shifted = buildSelectBefore(Builder, Anchor, *Type, *IsShort, *ShortValue, *LongValue);
-  if (!Shifted)
-    return std::unexpected(Shifted.error());
-  return buildSelectBefore(Builder, Anchor, *Type, *IsZero, Low, *Shifted);
-}
-
-[[nodiscard]] Expected<std::vector<Register>>
-emitThreeLimbLeftShift(MachineIRBuilder &Builder, MachineInstr &Anchor,
-                       std::span<const Register> Parts, Register Amount) {
-  if (Parts.size() != 3)
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "RISC-V three-limb shift requires exactly three limbs");
-  auto Type = LowLevelType::makeCategoryNeutralScalar(64);
-  if (!Type)
-    return std::unexpected(Type.error());
-
-  auto OuterBoundary = buildIntegerConstantBefore(Builder, Anchor, 64, 128);
-  if (!OuterBoundary)
-    return std::unexpected(OuterBoundary.error());
-  auto OuterExcess =
-      buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, Amount, *OuterBoundary);
-  if (!OuterExcess)
-    return std::unexpected(OuterExcess.error());
-  auto OuterLack =
-      buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, *OuterBoundary, Amount);
-  if (!OuterLack)
-    return std::unexpected(OuterLack.error());
-  auto Zero = buildIntegerConstantBefore(Builder, Anchor, 64, 0);
-  if (!Zero)
-    return std::unexpected(Zero.error());
-  auto OuterIsShort = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan,
-                                                Amount, *OuterBoundary);
-  if (!OuterIsShort)
-    return std::unexpected(OuterIsShort.error());
-  auto OuterIsZero =
-      buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::Equal, Amount, *Zero);
-  if (!OuterIsZero)
-    return std::unexpected(OuterIsZero.error());
-
-  auto InnerBoundary = buildIntegerConstantBefore(Builder, Anchor, 64, 64);
-  if (!InnerBoundary)
-    return std::unexpected(InnerBoundary.error());
-  auto InnerExcess =
-      buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, Amount, *InnerBoundary);
-  if (!InnerExcess)
-    return std::unexpected(InnerExcess.error());
-  auto InnerLack =
-      buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, *InnerBoundary, Amount);
-  if (!InnerLack)
-    return std::unexpected(InnerLack.error());
-  auto InnerIsShort = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan,
-                                                Amount, *InnerBoundary);
-  if (!InnerIsShort)
-    return std::unexpected(InnerIsShort.error());
-  auto InnerIsZero =
-      buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::Equal, Amount, *Zero);
-  if (!InnerIsZero)
-    return std::unexpected(InnerIsZero.error());
-  auto Lower = emitTwoLimbLeftShiftBody(Builder, Anchor, Parts[0], Parts[1], Amount, *InnerExcess,
-                                        *InnerLack, *Zero, *InnerIsShort, *InnerIsZero);
-  if (!Lower)
-    return std::unexpected(Lower.error());
-
-  auto Cross = emitLowLimbOfTwoLimbLogicalRightShift(Builder, Anchor, Parts[0], Parts[1],
-                                                     *OuterLack, *InnerBoundary, *Zero);
-  if (!Cross)
-    return std::unexpected(Cross.error());
-  auto HighIsShort = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan,
-                                               Amount, *InnerBoundary);
-  if (!HighIsShort)
-    return std::unexpected(HighIsShort.error());
-  auto HighShort =
-      buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SHL, *Type, Parts[2], Amount);
-  if (!HighShort)
-    return std::unexpected(HighShort.error());
-  auto BoundedHigh = buildSelectBefore(Builder, Anchor, *Type, *HighIsShort, *HighShort, *Zero);
-  if (!BoundedHigh)
-    return std::unexpected(BoundedHigh.error());
-  auto ShortHigh =
-      buildBinaryBefore(Builder, Anchor, GenericOpcode::G_OR, *Type, *Cross, *BoundedHigh);
-  if (!ShortHigh)
-    return std::unexpected(ShortHigh.error());
-
-  auto LongIsShort = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan,
-                                               *OuterExcess, *InnerBoundary);
-  if (!LongIsShort)
-    return std::unexpected(LongIsShort.error());
-  auto LongShift =
-      buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SHL, *Type, Parts[0], *OuterExcess);
-  if (!LongShift)
-    return std::unexpected(LongShift.error());
-  auto LongHigh = buildSelectBefore(Builder, Anchor, *Type, *LongIsShort, *LongShift, *Zero);
-  if (!LongHigh)
-    return std::unexpected(LongHigh.error());
-
-  auto ResultLow = buildSelectBefore(Builder, Anchor, *Type, *OuterIsShort, Lower->Low, *Zero);
-  if (!ResultLow)
-    return std::unexpected(ResultLow.error());
-  auto ResultMiddle = buildSelectBefore(Builder, Anchor, *Type, *OuterIsShort, Lower->High, *Zero);
-  if (!ResultMiddle)
-    return std::unexpected(ResultMiddle.error());
-  auto ShiftedHigh =
-      buildSelectBefore(Builder, Anchor, *Type, *OuterIsShort, *ShortHigh, *LongHigh);
-  if (!ShiftedHigh)
-    return std::unexpected(ShiftedHigh.error());
-  auto ResultHigh = buildSelectBefore(Builder, Anchor, *Type, *OuterIsZero, Parts[2], *ShiftedHigh);
-  if (!ResultHigh)
-    return std::unexpected(ResultHigh.error());
-  return std::vector<Register>{*ResultLow, *ResultMiddle, *ResultHigh};
-}
-
-[[nodiscard]] Expected<void> lowerWideShiftLeft(MachineInstr &Instruction,
-                                                MachineFunction &Function,
-                                                const TargetDescription &Target) {
-  const MachineOperand *DestinationOperand = Instruction.operandAt(0);
-  const MachineOperand *ValueOperand = Instruction.operandAt(1);
-  const MachineOperand *AmountOperand = Instruction.operandAt(2);
-  const Register *Destination =
-      DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
-  const Register *Value = ValueOperand == nullptr ? nullptr : ValueOperand->getIf<Register>();
-  const Register *Amount = AmountOperand == nullptr ? nullptr : AmountOperand->getIf<Register>();
-  if (Destination == nullptr || Value == nullptr || Amount == nullptr)
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "RISC-V wide shift has invalid register operands");
-  auto ValueParts = collectWideValueParts(*Value, Function);
-  if (!ValueParts)
-    return std::unexpected(ValueParts.error());
-  auto AmountParts = collectWideValueParts(*Amount, Function);
-  if (!AmountParts)
-    return std::unexpected(AmountParts.error());
-  if (ValueParts->Parts.size() < 2 || AmountParts->Parts.empty())
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "RISC-V wide shift has an invalid limb decomposition");
-  auto ResultPlan = wideResultPlan(*Destination, Function, ValueParts->Parts.size());
-  if (!ResultPlan)
-    return std::unexpected(ResultPlan.error());
-
-  MachineIRBuilder Builder(Function, Target);
-  Expected<std::vector<Register>> Results =
-      ValueParts->Parts.size() == 2
-          ? emitTwoLimbLeftShift(Builder, Instruction, ValueParts->Parts[0], ValueParts->Parts[1],
-                                 AmountParts->Parts[0])
-                .transform([](TwoLimbShiftResult Result) {
-                  return std::vector<Register>{Result.Low, Result.High};
-                })
-      : ValueParts->Parts.size() == 3
-          ? emitThreeLimbLeftShift(Builder, Instruction, ValueParts->Parts, AmountParts->Parts[0])
-          : Expected<std::vector<Register>>(std::unexpected(
-                Error(ErrorCode::Unsupported,
-                      "RISC-V wide shift currently requires two or three 64-bit limbs")));
-  if (!Results)
-    return std::unexpected(Results.error());
-  for (std::size_t Index = 0; Index < Results->size(); ++Index) {
-    auto Replaced = replaceRegisterUses(Function, ResultPlan->Outputs[Index], (*Results)[Index]);
-    if (!Replaced)
-      return std::unexpected(Replaced.error());
-  }
-  if (auto Erased = eraseInstruction(*ResultPlan->Unmerge); !Erased)
-    return std::unexpected(Erased.error());
-  if (ResultPlan->Extension != nullptr) {
-    if (auto Erased = eraseInstruction(*ResultPlan->Extension); !Erased)
-      return std::unexpected(Erased.error());
-  }
-  if (auto Erased = eraseInstruction(Instruction); !Erased)
-    return std::unexpected(Erased.error());
-  for (std::size_t Index = 1; Index < AmountParts->Parts.size(); ++Index) {
-    const MachineOperand *Definition =
-        Function.registerInfo().getUniqueVRegDef(AmountParts->Parts[Index]);
-    if (Definition != nullptr && Definition->parent() != nullptr)
-      AmountParts->Artifacts.push_back(const_cast<MachineInstr *>(Definition->parent()));
-  }
-  ValueParts->Artifacts.insert(ValueParts->Artifacts.end(), AmountParts->Artifacts.begin(),
-                               AmountParts->Artifacts.end());
-  eraseDeadArtifacts(ValueParts->Artifacts, Function);
-  return {};
-}
-
-[[nodiscard]] Expected<std::vector<Register>> emitWideAdd(MachineIRBuilder &Builder,
-                                                          MachineInstr &Anchor,
-                                                          std::span<const Register> Left,
-                                                          std::span<const Register> Right) {
-  if (Left.size() != Right.size() || Left.size() < 2)
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "RISC-V wide add requires matching multi-limb operands");
-  auto Type = LowLevelType::makeCategoryNeutralScalar(64);
-  if (!Type)
-    return std::unexpected(Type.error());
-  std::vector<Register> Results;
-  Results.reserve(Left.size());
-  auto Low = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_ADD, *Type, Left[0], Right[0]);
-  if (!Low)
-    return std::unexpected(Low.error());
-  auto Carry = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan, *Low,
-                                         Right[0]);
-  if (!Carry)
-    return std::unexpected(Carry.error());
-  auto LowCopy = buildUnaryBefore(Builder, Anchor, GenericOpcode::COPY, *Type, *Low);
-  if (!LowCopy)
-    return std::unexpected(LowCopy.error());
-  Results.push_back(*LowCopy);
-  for (std::size_t Index = 1; Index < Left.size(); ++Index) {
-    auto Base =
-        buildBinaryBefore(Builder, Anchor, GenericOpcode::G_ADD, *Type, Left[Index], Right[Index]);
-    if (!Base)
-      return std::unexpected(Base.error());
-    Register BaseCarry;
-    if (Index + 1U < Left.size()) {
-      auto Compared = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan,
-                                                *Base, Left[Index]);
-      if (!Compared)
-        return std::unexpected(Compared.error());
-      BaseCarry = *Compared;
-    }
-    auto Sum = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_ADD, *Type, *Base, *Carry);
-    if (!Sum)
-      return std::unexpected(Sum.error());
-    if (Index + 1U < Left.size()) {
-      auto Zero = buildIntegerConstantBefore(Builder, Anchor, 64, 0);
-      if (!Zero)
-        return std::unexpected(Zero.error());
-      auto IsZero =
-          buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::Equal, *Sum, *Zero);
-      if (!IsZero)
-        return std::unexpected(IsZero.error());
-      auto CarryFromIncoming =
-          buildBinaryBefore(Builder, Anchor, GenericOpcode::G_AND, *Type, *IsZero, *Carry);
-      if (!CarryFromIncoming)
-        return std::unexpected(CarryFromIncoming.error());
-      auto NextCarry = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_OR, *Type, BaseCarry,
-                                         *CarryFromIncoming);
-      if (!NextCarry)
-        return std::unexpected(NextCarry.error());
-      Carry = *NextCarry;
-    }
-    auto Copy = buildUnaryBefore(Builder, Anchor, GenericOpcode::COPY, *Type, *Sum);
-    if (!Copy)
-      return std::unexpected(Copy.error());
-    Results.push_back(*Copy);
-  }
-  return Results;
-}
-
-[[nodiscard]] Expected<std::vector<Register>> emitWideSub(MachineIRBuilder &Builder,
-                                                          MachineInstr &Anchor,
-                                                          std::span<const Register> Left,
-                                                          std::span<const Register> Right) {
-  if (Left.size() != Right.size() || Left.size() < 2)
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "RISC-V wide sub requires matching multi-limb operands");
-  auto Type = LowLevelType::makeCategoryNeutralScalar(64);
-  if (!Type)
-    return std::unexpected(Type.error());
-  std::vector<Register> Results;
-  Results.reserve(Left.size());
-  auto Low = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, Left[0], Right[0]);
-  if (!Low)
-    return std::unexpected(Low.error());
-  Results.push_back(*Low);
-  auto Borrow = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan,
-                                          Left[0], Right[0]);
-  if (!Borrow)
-    return std::unexpected(Borrow.error());
-  for (std::size_t Index = 1; Index < Left.size(); ++Index) {
-    auto Base =
-        buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, Left[Index], Right[Index]);
-    if (!Base)
-      return std::unexpected(Base.error());
-    Register BaseBorrow;
-    if (Index + 1U < Left.size()) {
-      auto Compared = buildIntegerCompareBefore(
-          Builder, Anchor, IntegerPredicate::UnsignedGreaterThan, *Base, Left[Index]);
-      if (!Compared)
-        return std::unexpected(Compared.error());
-      BaseBorrow = *Compared;
-    }
-    auto Difference =
-        buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, *Base, *Borrow);
-    if (!Difference)
-      return std::unexpected(Difference.error());
-    Results.push_back(*Difference);
-    if (Index + 1U < Left.size()) {
-      auto Zero = buildIntegerConstantBefore(Builder, Anchor, 64, 0);
-      if (!Zero)
-        return std::unexpected(Zero.error());
-      auto IsZero =
-          buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::Equal, *Base, *Zero);
-      if (!IsZero)
-        return std::unexpected(IsZero.error());
-      auto BorrowFromIncoming =
-          buildBinaryBefore(Builder, Anchor, GenericOpcode::G_AND, *Type, *IsZero, *Borrow);
-      if (!BorrowFromIncoming)
-        return std::unexpected(BorrowFromIncoming.error());
-      auto NextBorrow = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_OR, *Type, BaseBorrow,
-                                          *BorrowFromIncoming);
-      if (!NextBorrow)
-        return std::unexpected(NextBorrow.error());
-      Borrow = *NextBorrow;
-    }
-  }
-  return Results;
-}
-
-[[nodiscard]] Expected<std::vector<Register>>
-emitWideBitwise(MachineIRBuilder &Builder, MachineInstr &Anchor, GenericOpcode Generic,
-                std::span<const Register> Left, std::span<const Register> Right) {
-  if (Left.size() != Right.size() || Left.size() < 2)
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "RISC-V wide bitwise operation requires matching multi-limb operands");
-  auto Type = LowLevelType::makeCategoryNeutralScalar(64);
-  if (!Type)
-    return std::unexpected(Type.error());
-  std::vector<Register> Results;
-  Results.reserve(Left.size());
-  for (std::size_t Index = 0; Index < Left.size(); ++Index) {
-    auto Part = buildBinaryBefore(Builder, Anchor, Generic, *Type, Left[Index], Right[Index]);
-    if (!Part)
-      return std::unexpected(Part.error());
-    Results.push_back(*Part);
-  }
-  return Results;
-}
-
-[[nodiscard]] Expected<void> lowerWideBinary(MachineInstr &Instruction, MachineFunction &Function,
-                                             const TargetDescription &Target,
-                                             GenericOpcode Generic) {
-  const MachineOperand *DestinationOperand = Instruction.operandAt(0);
-  const MachineOperand *LeftOperand = Instruction.operandAt(1);
-  const MachineOperand *RightOperand = Instruction.operandAt(2);
-  const Register *Destination =
-      DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
-  const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf<Register>();
-  const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf<Register>();
-  if (Destination == nullptr || Left == nullptr || Right == nullptr)
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "RISC-V wide binary instruction has invalid operands");
-  auto LeftParts = collectWideValueParts(*Left, Function);
-  if (!LeftParts)
-    return std::unexpected(LeftParts.error());
-  auto RightParts = collectWideValueParts(*Right, Function);
-  if (!RightParts)
-    return std::unexpected(RightParts.error());
-  if (LeftParts->Parts.size() != RightParts->Parts.size())
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "RISC-V wide binary operands have different limb counts");
-  auto ResultPlan = wideResultPlan(*Destination, Function, LeftParts->Parts.size());
-  if (!ResultPlan)
-    return std::unexpected(ResultPlan.error());
-
-  MachineIRBuilder Builder(Function, Target);
-  Expected<std::vector<Register>> Results =
-      Generic == GenericOpcode::G_ADD
-          ? emitWideAdd(Builder, Instruction, LeftParts->Parts, RightParts->Parts)
-      : Generic == GenericOpcode::G_SUB
-          ? emitWideSub(Builder, Instruction, LeftParts->Parts, RightParts->Parts)
-          : emitWideBitwise(Builder, Instruction, Generic, LeftParts->Parts, RightParts->Parts);
-  if (!Results)
-    return std::unexpected(Results.error());
-  for (std::size_t Index = 0; Index < Results->size(); ++Index) {
-    auto Replaced = replaceRegisterUses(Function, ResultPlan->Outputs[Index], (*Results)[Index]);
-    if (!Replaced)
-      return std::unexpected(Replaced.error());
-  }
-  if (auto Erased = eraseInstruction(*ResultPlan->Unmerge); !Erased)
-    return std::unexpected(Erased.error());
-  if (ResultPlan->Extension != nullptr) {
-    if (auto Erased = eraseInstruction(*ResultPlan->Extension); !Erased)
-      return std::unexpected(Erased.error());
-  }
-  if (auto Erased = eraseInstruction(Instruction); !Erased)
-    return std::unexpected(Erased.error());
-  LeftParts->Artifacts.insert(LeftParts->Artifacts.end(), RightParts->Artifacts.begin(),
-                              RightParts->Artifacts.end());
-  eraseDeadArtifacts(LeftParts->Artifacts, Function);
-  return {};
-}
-
-[[nodiscard]] bool canLowerNarrowBranchCondition(const MachineInstr &Instruction,
-                                                 const MachineFunction &Function) {
-  if (Instruction.opcode() != opcode(GenericOpcode::G_BRCOND) || Instruction.operandCount() < 2)
-    return false;
-  const MachineOperand *ConditionOperand = Instruction.operandAt(0);
-  const Register *Condition =
-      ConditionOperand == nullptr ? nullptr : ConditionOperand->getIf<Register>();
-  if (Condition == nullptr || !Condition->isVirtual() ||
-      singleRegisterUse(Function, *Condition) != &Instruction)
-    return false;
-  const auto ConditionType = registerType(Function, *Condition);
-  const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(*Condition);
-  const MachineInstr *Truncation = Definition == nullptr ? nullptr : Definition->parent();
-  if (!ConditionType || ConditionType->scalarBitWidth() != 1 || Truncation == nullptr ||
-      Truncation->opcode() != opcode(GenericOpcode::G_TRUNC) || Truncation->operandCount() != 2)
-    return false;
-  return operandType(Function, *Truncation, 1)
-      .transform(
-          [](LowLevelType Type) { return isScalarGPRType(Type) && Type.scalarBitWidth() == 64; })
-      .value_or(false);
-}
-
-[[nodiscard]] Expected<void> lowerNarrowBranchCondition(MachineInstr &Instruction,
-                                                        MachineFunction &Function,
-                                                        const TargetDescription &Target) {
-  if (!canLowerNarrowBranchCondition(Instruction, Function))
-    return makeUnexpected(ErrorCode::Unsupported,
-                          "RISC-V branch condition is not a narrow truncation");
-  const MachineOperand *ConditionOperand = Instruction.operandAt(0);
-  const Register Condition = *ConditionOperand->getIf<Register>();
-  const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(Condition);
-  MachineInstr *Truncation = const_cast<MachineInstr *>(Definition->parent());
-  const Register Source = *Truncation->operandAt(1)->getIf<Register>();
-  auto Type = LowLevelType::makeCategoryNeutralScalar(64);
-  if (!Type)
-    return std::unexpected(Type.error());
-  MachineIRBuilder Builder(Function, Target);
-  auto One = buildIntegerConstantBefore(Builder, Instruction, 64, 1);
-  if (!One)
-    return std::unexpected(One.error());
-  auto Masked = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_AND, *Type, Source, *One);
-  if (!Masked)
-    return std::unexpected(Masked.error());
-  auto Replaced = Instruction.operandAt(0)->setRegister(*Masked);
-  if (!Replaced)
-    return std::unexpected(Replaced.error());
-  if (Function.registerInfo().useEmpty(Condition))
-    return eraseInstruction(*Truncation);
-  return {};
-}
-
-struct NarrowPhiIncoming final {
-  Register Value;
-  MachineBasicBlock *Block = nullptr;
-  MachineInstr *ReusableTruncation = nullptr;
-};
-
-[[nodiscard]] MachineInstr *firstInstructionAfterLeadingCopies(MachineBasicBlock &Block) {
-  MachineInstr *Instruction = Block.firstInstruction();
-  while (Instruction != nullptr && Instruction->opcode() == opcode(GenericOpcode::COPY))
-    Instruction = Instruction->next();
-  return Instruction;
-}
-
-[[nodiscard]] Expected<Register>
-buildIncomingConversion(MachineFunction &Function, const TargetDescription &Target,
-                        MachineBasicBlock &Block, MachineInstr *Anchor, GenericOpcode Generic,
-                        LowLevelType Type, Register Source) {
-  auto Result = Function.registerInfo().createVirtualRegister(Type);
-  if (!Result)
-    return std::unexpected(Result.error());
-  MachineIRBuilder Builder(Function, Target);
-  auto Point =
-      Anchor == nullptr ? Builder.setInsertPoint(Block) : Builder.setInsertPointBefore(*Anchor);
-  if (!Point)
-    return std::unexpected(Point.error());
-  auto Pending = Builder.buildInstr(opcode(Generic));
-  if (!Pending)
-    return std::unexpected(Pending.error());
-  if (auto Added = Pending->addDef(*Result); !Added)
-    return std::unexpected(Added.error());
-  if (auto Added = Pending->addUse(Source); !Added)
-    return std::unexpected(Added.error());
-  auto Built = Pending->build();
-  if (!Built)
-    return std::unexpected(Built.error());
-  return *Result;
-}
-
-[[nodiscard]] Expected<void> lowerNarrowPhi(MachineInstr &Instruction, MachineFunction &Function,
-                                            const TargetDescription &Target) {
-  const std::optional<LowLevelType> OriginalType = resultType(Function, Instruction);
-  const MachineOperand *DestinationOperand = Instruction.operandAt(0);
-  const Register *Destination =
-      DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
-  if (!OriginalType || Destination == nullptr || OriginalType->scalarBitWidth() >= 32 ||
-      Instruction.operandCount() < 3 || (Instruction.operandCount() % 2U) == 0)
-    return makeUnexpected(ErrorCode::VerificationError, "RISC-V narrow PHI has invalid operands");
-  auto LegalType = LowLevelType::makeCategoryNeutralScalar(32);
-  if (!LegalType)
-    return std::unexpected(LegalType.error());
-
-  std::vector<NarrowPhiIncoming> Incoming;
-  Incoming.reserve((Instruction.operandCount() - 1U) / 2U);
-  for (std::size_t Index = 1; Index < Instruction.operandCount(); Index += 2) {
-    const MachineOperand *ValueOperand = Instruction.operandAt(Index);
-    const Register *Value = ValueOperand == nullptr ? nullptr : ValueOperand->getIf<Register>();
-    const MachineOperand *BlockOperand = Instruction.operandAt(Index + 1U);
-    const auto *Block =
-        BlockOperand == nullptr ? nullptr : BlockOperand->getIf<BasicBlockOperand>();
-    if (Value == nullptr || Block == nullptr || Block->Block == nullptr)
-      return makeUnexpected(ErrorCode::VerificationError,
-                            "RISC-V narrow PHI has an invalid incoming edge");
-    MachineInstr *Reusable = nullptr;
-    const MachineOperand *Definition =
-        Value->isVirtual() ? Function.registerInfo().getUniqueVRegDef(*Value) : nullptr;
-    MachineInstr *Def =
-        Definition == nullptr ? nullptr : const_cast<MachineInstr *>(Definition->parent());
-    if (Def != nullptr && Def->opcode() == opcode(GenericOpcode::G_TRUNC) &&
-        Def->operandCount() == 2 && singleRegisterUse(Function, *Value) == &Instruction &&
-        operandType(Function, *Def, 1)
-            .transform([](LowLevelType Type) { return Type.scalarBitWidth() >= 32; })
-            .value_or(false)) {
-      Reusable = Def;
-    }
-    Incoming.push_back(
-        NarrowPhiIncoming{.Value = *Value, .Block = Block->Block, .ReusableTruncation = Reusable});
-  }
-  for (NarrowPhiIncoming &Entry : Incoming) {
-    if (Entry.ReusableTruncation == nullptr)
-      continue;
-    const std::size_t Count = static_cast<std::size_t>(std::ranges::count(
-        Incoming, Entry.ReusableTruncation, &NarrowPhiIncoming::ReusableTruncation));
-    if (Count != 1)
-      Entry.ReusableTruncation = nullptr;
-  }
-  for (NarrowPhiIncoming &Entry : Incoming) {
-    if (Entry.ReusableTruncation == nullptr)
-      continue;
-    MachineBasicBlock *OriginalBlock = Entry.ReusableTruncation->parent();
-    if (OriginalBlock == nullptr)
-      return makeUnexpected(ErrorCode::VerificationError,
-                            "RISC-V narrow PHI input truncation is detached");
-    auto Removed = OriginalBlock->removeInstruction(*Entry.ReusableTruncation);
-    if (!Removed)
-      return std::unexpected(Removed.error());
-  }
-
-  std::vector<std::pair<MachineBasicBlock *, MachineInstr *>> Anchors;
-  for (const NarrowPhiIncoming &Entry : Incoming) {
-    if (std::ranges::find(Anchors, Entry.Block, &decltype(Anchors)::value_type::first) ==
-        Anchors.end()) {
-      Anchors.emplace_back(Entry.Block, firstInstructionAfterLeadingCopies(*Entry.Block));
-    }
-  }
-  std::vector<Register> LegalIncoming;
-  LegalIncoming.reserve(Incoming.size());
-  for (NarrowPhiIncoming &Entry : Incoming) {
-    const auto AnchorPosition =
-        std::ranges::find(Anchors, Entry.Block, &decltype(Anchors)::value_type::first);
-    MachineInstr *Anchor = AnchorPosition->second;
-    if (Entry.ReusableTruncation != nullptr) {
-      auto SetType = Function.registerInfo().setType(Entry.Value, *LegalType);
-      if (!SetType)
-        return std::unexpected(SetType.error());
-      auto Inserted =
-          Anchor == nullptr
-              ? Entry.Block->appendInstruction(*Entry.ReusableTruncation)
-              : Entry.Block->insertInstructionBefore(*Anchor, *Entry.ReusableTruncation);
-      if (!Inserted)
-        return std::unexpected(Inserted.error());
-      LegalIncoming.push_back(Entry.Value);
-      continue;
-    }
-    const auto ValueType = registerType(Function, Entry.Value);
-    if (!ValueType)
-      return makeUnexpected(ErrorCode::VerificationError,
-                            "RISC-V narrow PHI input has no scalar type");
-    const std::uint32_t Width = ValueType->scalarBitWidth();
-    const GenericOpcode Conversion = Width < 32   ? GenericOpcode::G_ANYEXT
-                                     : Width > 32 ? GenericOpcode::G_TRUNC
-                                                  : GenericOpcode::COPY;
-    auto Converted = buildIncomingConversion(Function, Target, *Entry.Block, Anchor, Conversion,
-                                             *LegalType, Entry.Value);
-    if (!Converted)
-      return std::unexpected(Converted.error());
-    LegalIncoming.push_back(*Converted);
-  }
-
-  auto LegalResult = Function.registerInfo().createVirtualRegister(*LegalType);
-  if (!LegalResult)
-    return std::unexpected(LegalResult.error());
-  MachineIRBuilder Builder(Function, Target);
-  auto Point = Builder.setInsertPointBefore(Instruction);
-  if (!Point)
-    return std::unexpected(Point.error());
-  auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_PHI));
-  if (!Pending)
-    return std::unexpected(Pending.error());
-  if (auto Added = Pending->addDef(*LegalResult); !Added)
-    return std::unexpected(Added.error());
-  for (std::size_t Index = 0; Index < Incoming.size(); ++Index) {
-    if (auto Added = Pending->addUse(LegalIncoming[Index]); !Added)
-      return std::unexpected(Added.error());
-    if (auto Added = Pending->addBasicBlock(*Incoming[Index].Block); !Added)
-      return std::unexpected(Added.error());
-  }
-  auto Built = Pending->build();
-  if (!Built)
-    return std::unexpected(Built.error());
-  auto Replaced = replaceRegisterUses(Function, *Destination, *LegalResult);
-  if (!Replaced)
-    return std::unexpected(Replaced.error());
-  return eraseInstruction(Instruction);
-}
-
-[[nodiscard]] Expected<void> lowerWidePhi(MachineInstr &Instruction, MachineFunction &Function,
-                                          const TargetDescription &Target) {
-  const MachineOperand *DestinationOperand = Instruction.operandAt(0);
-  const Register *Destination =
-      DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
-  if (Destination == nullptr || Instruction.operandCount() < 5 ||
-      (Instruction.operandCount() % 2U) == 0)
-    return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide PHI has invalid operands");
-
-  std::vector<WideValueParts> IncomingParts;
-  std::vector<MachineBasicBlock *> IncomingBlocks;
-  for (std::size_t Index = 1; Index < Instruction.operandCount(); Index += 2) {
-    const MachineOperand *ValueOperand = Instruction.operandAt(Index);
-    const Register *Value = ValueOperand == nullptr ? nullptr : ValueOperand->getIf<Register>();
-    const MachineOperand *BlockOperand = Instruction.operandAt(Index + 1U);
-    const auto *Block =
-        BlockOperand == nullptr ? nullptr : BlockOperand->getIf<BasicBlockOperand>();
-    if (Value == nullptr || Block == nullptr || Block->Block == nullptr)
-      return makeUnexpected(ErrorCode::VerificationError,
-                            "RISC-V wide PHI has an invalid incoming edge");
-    auto Parts = collectWideValueParts(*Value, Function);
-    if (!Parts)
-      return std::unexpected(Parts.error());
-    if (!IncomingParts.empty() && Parts->Parts.size() != IncomingParts.front().Parts.size())
-      return makeUnexpected(ErrorCode::VerificationError,
-                            "RISC-V wide PHI incoming values have different limb counts");
-    IncomingParts.push_back(std::move(*Parts));
-    IncomingBlocks.push_back(Block->Block);
-  }
-  const std::size_t PartCount = IncomingParts.front().Parts.size();
-  auto ResultPlan = wideResultPlan(*Destination, Function, PartCount);
-  if (!ResultPlan)
-    return std::unexpected(ResultPlan.error());
-  auto Type = LowLevelType::makeCategoryNeutralScalar(64);
-  if (!Type)
-    return std::unexpected(Type.error());
-  MachineIRBuilder Builder(Function, Target);
-  std::vector<Register> Results;
-  Results.reserve(PartCount);
-  for (std::size_t PartIndex = 0; PartIndex < PartCount; ++PartIndex) {
-    auto Result = Function.registerInfo().createVirtualRegister(*Type);
-    if (!Result)
-      return std::unexpected(Result.error());
-    auto Point = Builder.setInsertPointBefore(Instruction);
-    if (!Point)
-      return std::unexpected(Point.error());
-    auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_PHI));
-    if (!Pending)
-      return std::unexpected(Pending.error());
-    if (auto Added = Pending->addDef(*Result); !Added)
-      return std::unexpected(Added.error());
-    for (std::size_t IncomingIndex = 0; IncomingIndex < IncomingParts.size(); ++IncomingIndex) {
-      if (auto Added = Pending->addUse(IncomingParts[IncomingIndex].Parts[PartIndex]); !Added)
-        return std::unexpected(Added.error());
-      if (auto Added = Pending->addBasicBlock(*IncomingBlocks[IncomingIndex]); !Added)
-        return std::unexpected(Added.error());
-    }
-    auto Built = Pending->build();
-    if (!Built)
-      return std::unexpected(Built.error());
-    Results.push_back(*Result);
-  }
-  for (std::size_t Index = 0; Index < PartCount; ++Index) {
-    auto Replaced = replaceRegisterUses(Function, ResultPlan->Outputs[Index], Results[Index]);
-    if (!Replaced)
-      return std::unexpected(Replaced.error());
-  }
-  if (auto Erased = eraseInstruction(*ResultPlan->Unmerge); !Erased)
-    return std::unexpected(Erased.error());
-  if (ResultPlan->Extension != nullptr) {
-    if (auto Erased = eraseInstruction(*ResultPlan->Extension); !Erased)
-      return std::unexpected(Erased.error());
-  }
-  if (auto Erased = eraseInstruction(Instruction); !Erased)
-    return std::unexpected(Erased.error());
-  std::vector<MachineInstr *> Artifacts;
-  for (WideValueParts &Parts : IncomingParts)
-    Artifacts.insert(Artifacts.end(), Parts.Artifacts.begin(), Parts.Artifacts.end());
-  eraseDeadArtifacts(Artifacts, Function);
-  return {};
-}
-
-[[nodiscard]] bool canFoldWideMergeImmediately(const MachineInstr &Instruction,
-                                               const MachineFunction &Function) {
-  const MachineOperand *LowOperand = Instruction.operandAt(1);
-  const Register *Low = LowOperand == nullptr ? nullptr : LowOperand->getIf<Register>();
-  if (Low == nullptr)
-    return false;
-  const auto LowType = registerType(Function, *Low);
-  if (!LowType)
-    return false;
-  const MachineOperand *DestinationOperand = Instruction.operandAt(0);
-  const Register *Destination =
-      DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
-  if (Destination == nullptr)
-    return false;
-  auto &RegisterInfo = const_cast<MachineRegisterInfo &>(Function.registerInfo());
-  for (MachineOperand &Use : RegisterInfo.useOperands(*Destination)) {
-    MachineInstr *Trunc = Use.parent();
-    if (Trunc == nullptr || Trunc->opcode() != opcode(GenericOpcode::G_TRUNC) ||
-        Trunc->operandAt(1) != &Use)
-      return false;
-    const std::optional<LowLevelType> Result = resultType(Function, *Trunc);
-    if (!Result || Result->scalarBitWidth() > LowType->scalarBitWidth())
-      return false;
-  }
-  return true;
-}
-
-struct RV64ScalarLayout final {
-  std::uint64_t StoreSize = 0;
-  std::uint64_t AllocationSize = 0;
-  std::uint64_t ABIAlignment = 0;
-};
-
-[[nodiscard]] std::optional<RV64ScalarLayout> rv64ScalarLayout(LowLevelType Type) noexcept {
-  if (Type.isPointer()) {
-    return Type.addressSpace() == 0 && Type.scalarBitWidth() == 64
-               ? std::optional<RV64ScalarLayout>{{8, 8, 8}}
-               : std::nullopt;
-  }
-  if ((!isSupportedIntegerType(Type) && !isSupportedFloatingScalar(Type)) ||
-      Type.scalarBitWidth() == 0) {
-    return std::nullopt;
-  }
-  const std::uint64_t StoreBytes = (Type.scalarBitWidth() + 7U) / 8U;
-  const std::uint64_t AllocationBytes = std::bit_ceil(StoreBytes);
-  return RV64ScalarLayout{StoreBytes, AllocationBytes, std::min<std::uint64_t>(AllocationBytes, 8)};
-}
-
-[[nodiscard]] Expected<const MachineMemOperand *>
-createMemoryAccess(MachineFunction &Function, MemoryAccessFlag Access, LowLevelType Type,
-                   std::uint64_t SizeBytes, std::uint64_t AlignmentBytes) {
-  auto Size = MemorySize::fixed(SizeBytes);
-  auto Align = Alignment::create(AlignmentBytes);
-  if (!Size || !Align)
-    return makeUnexpected(ErrorCode::InternalError, "RISC-V memory facts are invalid");
-  MachineMemOperandDesc Description;
-  Description.Accesses = {Access};
-  Description.Size = *Size;
-  Description.MemoryType = Type;
-  Description.Align = *Align;
-  return Function.createMemOperand(std::move(Description));
-}
-
-struct VAArgPlan final {
-  Register Destination;
-  Register ListPointer;
-  LowLevelType DestinationType;
-  LowLevelType PointerType;
-  std::uint64_t Alignment = 0;
-  RV64ScalarLayout ValueLayout;
-};
-
-[[nodiscard]] std::optional<VAArgPlan> vaargPlan(const MachineInstr &Instruction,
-                                                 const MachineFunction &Function) {
-  if (Instruction.opcode() != opcode(GenericOpcode::G_VAARG) || Instruction.operandCount() != 3 ||
-      !Instruction.memOperands().empty()) {
-    return std::nullopt;
-  }
-  const MachineOperand *DestinationOperand = Instruction.operandAt(0);
-  const MachineOperand *ListOperand = Instruction.operandAt(1);
-  const MachineOperand *AlignmentOperand = Instruction.operandAt(2);
-  const Register *Destination =
-      DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
-  const Register *ListPointer = ListOperand == nullptr ? nullptr : ListOperand->getIf<Register>();
-  const std::optional<LowLevelType> DestinationType = resultType(Function, Instruction);
-  const std::optional<LowLevelType> PointerType = operandType(Function, Instruction, 1);
-  const auto AlignmentValue = AlignmentOperand == nullptr
-                                  ? Expected<std::int64_t>(std::unexpected(
-                                        Error(ErrorCode::InvalidArgument, "missing alignment")))
-                                  : signedImmediate(*AlignmentOperand);
-  const std::optional<RV64ScalarLayout> ValueLayout =
-      DestinationType ? rv64ScalarLayout(*DestinationType) : std::nullopt;
-  if (Destination == nullptr || ListPointer == nullptr || !Destination->isVirtual() ||
-      !ListPointer->isVirtual() || !DestinationOperand->isDef() || !ListOperand->isUse() ||
-      !DestinationType || !PointerType || !PointerType->isPointer() ||
-      PointerType->addressSpace() != 0 || PointerType->scalarBitWidth() != 64 || !AlignmentValue ||
-      *AlignmentValue <= 0 || !std::has_single_bit(static_cast<std::uint64_t>(*AlignmentValue)) ||
-      !ValueLayout) {
-    return std::nullopt;
-  }
-
-  return VAArgPlan{*Destination,
-                   *ListPointer,
-                   *DestinationType,
-                   *PointerType,
-                   static_cast<std::uint64_t>(*AlignmentValue),
-                   *ValueLayout};
-}
-
-[[nodiscard]] Expected<void> lowerVAArg(MachineInstr &Instruction, MachineFunction &Function,
-                                        const TargetDescription &Target) {
-  const auto Plan = vaargPlan(Instruction, Function);
-  if (!Plan) {
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "RISC-V va_arg legalization requires a scalar result, p0 list, and "
-                          "power-of-two alignment");
-  }
-
-  auto ListLoadMemory =
-      createMemoryAccess(Function, MemoryAccessFlag::Load, Plan->PointerType, 8, 8);
-  if (!ListLoadMemory)
-    return std::unexpected(ListLoadMemory.error());
-  auto ListStoreMemory =
-      createMemoryAccess(Function, MemoryAccessFlag::Store, Plan->PointerType, 8, 8);
-  if (!ListStoreMemory)
-    return std::unexpected(ListStoreMemory.error());
-
-  auto CurrentList = Function.registerInfo().createVirtualRegister(Plan->PointerType);
-  if (!CurrentList)
-    return std::unexpected(CurrentList.error());
-  MachineIRBuilder Builder(Function, Target);
-  if (auto Point = Builder.setInsertPointBefore(Instruction); !Point)
-    return std::unexpected(Point.error());
-  auto ListLoad = Builder.buildInstr(opcode(GenericOpcode::G_LOAD));
-  if (!ListLoad)
-    return std::unexpected(ListLoad.error());
-  if (auto Added = ListLoad->addDef(*CurrentList); !Added)
-    return std::unexpected(Added.error());
-  if (auto Added = ListLoad->addUse(Plan->ListPointer); !Added)
-    return std::unexpected(Added.error());
-  if (auto Added = ListLoad->addMemOperand(**ListLoadMemory); !Added)
-    return std::unexpected(Added.error());
-  if (auto Built = ListLoad->build(); !Built)
-    return std::unexpected(Built.error());
-
-  Register ValueAddress = *CurrentList;
-  if (Plan->Alignment > 1) {
-    auto AlignmentOffset =
-        buildIntegerConstantBefore(Builder, Instruction, 64, Plan->Alignment - 1U);
-    if (!AlignmentOffset)
-      return std::unexpected(AlignmentOffset.error());
-    auto BiasedAddress = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_PTR_ADD,
-                                           Plan->PointerType, ValueAddress, *AlignmentOffset);
-    if (!BiasedAddress)
-      return std::unexpected(BiasedAddress.error());
-    auto AlignmentMask =
-        buildIntegerConstantBefore(Builder, Instruction, 64, ~(Plan->Alignment - std::uint64_t{1}));
-    if (!AlignmentMask)
-      return std::unexpected(AlignmentMask.error());
-    auto AlignedAddress = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_PTRMASK,
-                                            Plan->PointerType, *BiasedAddress, *AlignmentMask);
-    if (!AlignedAddress)
-      return std::unexpected(AlignedAddress.error());
-    ValueAddress = *AlignedAddress;
-  }
-
-  auto Increment =
-      buildIntegerConstantBefore(Builder, Instruction, 64, Plan->ValueLayout.AllocationSize);
-  if (!Increment)
-    return std::unexpected(Increment.error());
-  auto NextList = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_PTR_ADD,
-                                    Plan->PointerType, ValueAddress, *Increment);
-  if (!NextList)
-    return std::unexpected(NextList.error());
-  auto ListStore = Builder.buildInstr(opcode(GenericOpcode::G_STORE));
-  if (!ListStore)
-    return std::unexpected(ListStore.error());
-  if (auto Added = ListStore->addUse(*NextList); !Added)
-    return std::unexpected(Added.error());
-  if (auto Added = ListStore->addUse(Plan->ListPointer); !Added)
-    return std::unexpected(Added.error());
-  if (auto Added = ListStore->addMemOperand(**ListStoreMemory); !Added)
-    return std::unexpected(Added.error());
-  if (auto Built = ListStore->build(); !Built)
-    return std::unexpected(Built.error());
-
-  if (!Function.registerInfo().useEmpty(Plan->Destination)) {
-    auto ValueLoadMemory =
-        createMemoryAccess(Function, MemoryAccessFlag::Load, Plan->DestinationType,
-                           Plan->ValueLayout.StoreSize, Plan->ValueLayout.ABIAlignment);
-    if (!ValueLoadMemory)
-      return std::unexpected(ValueLoadMemory.error());
-    auto ValueLoad = Builder.buildInstr(opcode(GenericOpcode::G_LOAD));
-    if (!ValueLoad)
-      return std::unexpected(ValueLoad.error());
-    if (auto Added = ValueLoad->addDef(Plan->Destination); !Added)
-      return std::unexpected(Added.error());
-    if (auto Added = ValueLoad->addUse(ValueAddress); !Added)
-      return std::unexpected(Added.error());
-    if (auto Added = ValueLoad->addMemOperand(**ValueLoadMemory); !Added)
-      return std::unexpected(Added.error());
-    if (auto Built = ValueLoad->build(); !Built)
-      return std::unexpected(Built.error());
-  }
-  return eraseInstruction(Instruction);
-}
-
-struct VACopyPlan final {
-  Register Destination;
-  Register Source;
-  LowLevelType PointerType;
-};
-
-[[nodiscard]] std::optional<VACopyPlan> vacopyPlan(const MachineInstr &Instruction,
-                                                   const MachineFunction &Function) {
-  if (Instruction.opcode() != opcode(GenericOpcode::G_INTRINSIC_W_SIDE_EFFECTS) ||
-      Instruction.operandCount() != 3 || !Instruction.memOperands().empty()) {
-    return std::nullopt;
-  }
-  const MachineOperand *IntrinsicIDOperand = Instruction.operandAt(0);
-  const MachineOperand *DestinationOperand = Instruction.operandAt(1);
-  const MachineOperand *SourceOperand = Instruction.operandAt(2);
-  const auto *Intrinsic =
-      IntrinsicIDOperand == nullptr ? nullptr : IntrinsicIDOperand->getIf<IntrinsicOperand>();
-  const Register *Destination =
-      DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
-  const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>();
-  const std::optional<LowLevelType> DestinationType = operandType(Function, Instruction, 1);
-  const std::optional<LowLevelType> SourceType = operandType(Function, Instruction, 2);
-  if (Intrinsic == nullptr || Intrinsic->ID != intrinsicID(GenericIntrinsic::VACopy) ||
-      Destination == nullptr || Source == nullptr || !Destination->isVirtual() ||
-      !Source->isVirtual() || !DestinationOperand->isUse() || !SourceOperand->isUse() ||
-      !DestinationType || !SourceType || *DestinationType != *SourceType ||
-      !DestinationType->isPointer() || DestinationType->addressSpace() != 0 ||
-      DestinationType->scalarBitWidth() != 64) {
-    return std::nullopt;
-  }
-  return VACopyPlan{*Destination, *Source, *DestinationType};
-}
-
-[[nodiscard]] Expected<void> lowerVACopy(MachineInstr &Instruction, MachineFunction &Function,
-                                         const TargetDescription &Target) {
-  const auto Plan = vacopyPlan(Instruction, Function);
-  if (!Plan) {
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "RISC-V va_copy legalization requires two p0 operands");
-  }
-  auto Size = MemorySize::fixed(8);
-  auto Align = Alignment::create(8);
-  if (!Size || !Align)
-    return makeUnexpected(ErrorCode::InternalError, "RISC-V pointer memory facts are invalid");
-
-  MachineMemOperandDesc LoadDescription;
-  LoadDescription.Accesses = {MemoryAccessFlag::Load};
-  LoadDescription.Size = *Size;
-  LoadDescription.MemoryType = Plan->PointerType;
-  LoadDescription.Align = *Align;
-  auto LoadMemory = Function.createMemOperand(std::move(LoadDescription));
-  if (!LoadMemory)
-    return std::unexpected(LoadMemory.error());
-
-  MachineMemOperandDesc StoreDescription;
-  StoreDescription.Accesses = {MemoryAccessFlag::Store};
-  StoreDescription.Size = *Size;
-  StoreDescription.MemoryType = Plan->PointerType;
-  StoreDescription.Align = *Align;
-  auto StoreMemory = Function.createMemOperand(std::move(StoreDescription));
-  if (!StoreMemory)
-    return std::unexpected(StoreMemory.error());
-
-  auto Value = Function.registerInfo().createVirtualRegister(Plan->PointerType);
-  if (!Value)
-    return std::unexpected(Value.error());
-  MachineIRBuilder Builder(Function, Target);
-  if (auto Point = Builder.setInsertPointBefore(Instruction); !Point)
-    return std::unexpected(Point.error());
-
-  auto Load = Builder.buildInstr(opcode(GenericOpcode::G_LOAD));
-  if (!Load)
-    return std::unexpected(Load.error());
-  if (auto Added = Load->addDef(*Value); !Added)
-    return std::unexpected(Added.error());
-  if (auto Added = Load->addUse(Plan->Source); !Added)
-    return std::unexpected(Added.error());
-  if (auto Added = Load->addMemOperand(**LoadMemory); !Added)
-    return std::unexpected(Added.error());
-  if (auto Built = Load->build(); !Built)
-    return std::unexpected(Built.error());
-
-  auto Store = Builder.buildInstr(opcode(GenericOpcode::G_STORE));
-  if (!Store)
-    return std::unexpected(Store.error());
-  if (auto Added = Store->addUse(*Value); !Added)
-    return std::unexpected(Added.error());
-  if (auto Added = Store->addUse(Plan->Destination); !Added)
-    return std::unexpected(Added.error());
-  if (auto Added = Store->addMemOperand(**StoreMemory); !Added)
-    return std::unexpected(Added.error());
-  if (auto Built = Store->build(); !Built)
-    return std::unexpected(Built.error());
-  return eraseInstruction(Instruction);
-}
-
 } // namespace
 
-Expected<LegalizationAction> RISCVLegalizerInfo::classify(const MachineInstr &Instruction,
-                                                          const MachineFunction &Function,
-                                                          const TargetDescription &) const {
-  if (!Subtarget.features().has(RISCVExtension::I)) {
-    return makeUnexpected(ErrorCode::InternalError,
-                          "RISC-V legalizer requires the base I extension");
-  }
-  if (!isGenericOpcode(Instruction.opcode()))
-    return LegalizationAction::Legal;
-
-  const GenericOpcode Generic = static_cast<GenericOpcode>(Instruction.opcode().value());
-  const std::optional<LowLevelType> Type = resultType(Function, Instruction);
-  switch (Generic) {
-  case GenericOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
-    return vacopyPlan(Instruction, Function) ? LegalizationAction::Custom
-                                             : LegalizationAction::Unsupported;
-  case GenericOpcode::G_VAARG:
-    return vaargPlan(Instruction, Function) ? LegalizationAction::Custom
-                                            : LegalizationAction::Unsupported;
-  case GenericOpcode::G_FRAME_INDEX:
-    return Type && Type->isPointer() && Type->addressSpace() == 0 && Type->scalarBitWidth() == 64
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  case GenericOpcode::G_BR:
-    return LegalizationAction::Legal;
-  case GenericOpcode::G_BRCOND: {
-    const std::optional<LowLevelType> Condition = operandType(Function, Instruction, 0);
-    if (!Condition || !isScalarGPRType(*Condition))
-      return LegalizationAction::Unsupported;
-    return canLowerNarrowBranchCondition(Instruction, Function) ? LegalizationAction::Custom
-                                                                : LegalizationAction::Legal;
-  }
-  case GenericOpcode::G_PHI: {
-    if (!Type)
-      return LegalizationAction::Unsupported;
-    for (std::size_t Index = 1; Index < Instruction.operandCount(); Index += 2) {
-      const std::optional<LowLevelType> Incoming = operandType(Function, Instruction, Index);
-      if (!Incoming || *Incoming != *Type)
-        return LegalizationAction::Unsupported;
-    }
-    if (Type->isCategoryNeutralScalar() && Type->scalarBitWidth() > 64)
-      return LegalizationAction::Custom;
-    if (Type->isCategoryNeutralScalar() && Type->scalarBitWidth() < 32)
-      return LegalizationAction::Custom;
-    if (!isSupportedIntegerType(*Type))
-      return LegalizationAction::Unsupported;
-    return LegalizationAction::Legal;
-  }
-  case GenericOpcode::G_IMPLICIT_DEF:
-    return Type && isSupportedIntegerType(*Type) ? LegalizationAction::Legal
-                                                 : LegalizationAction::Unsupported;
-  case GenericOpcode::G_STORE:
-    if (Instruction.operandCount() == 0)
-      return LegalizationAction::Unsupported;
-    if (const std::optional<LowLevelType> ValueType =
-            registerType(Function, *Instruction.operandAt(0));
-        ValueType) {
-      const std::optional<LowLevelType> AddressType = operandType(Function, Instruction, 1);
-      if (!AddressType || !AddressType->isPointer() || AddressType->addressSpace() != 0 ||
-          AddressType->scalarBitWidth() != 64) {
-        return LegalizationAction::Unsupported;
-      }
-      if (Instruction.memOperands().size() == 1 && Instruction.memOperands().front()->isAtomic()) {
-        const std::uint64_t Bytes = ValueType->scalarBitWidth() / 8U;
-        return isSupportedIntegerType(*ValueType) &&
-                       isSupportedAtomicMemoryAccess(Instruction, AtomicOperation::Store, Bytes)
-                   ? LegalizationAction::Legal
-                   : LegalizationAction::Unsupported;
-      }
-      if (isSupportedIntegerType(*ValueType) && isSupportedNonAtomicMemoryAccess(Instruction) &&
-          memoryMatchesType(Instruction, *ValueType)) {
-        return LegalizationAction::Legal;
-      }
-    }
-    return LegalizationAction::Unsupported;
-  case GenericOpcode::G_SEXTLOAD:
-  case GenericOpcode::G_ZEXTLOAD: {
-    const auto Bytes = memoryAccessBytes(Instruction);
-    const std::optional<LowLevelType> AddressType = operandType(Function, Instruction, 1);
-    return Type && Bytes && AddressType && isSupportedIntegerType(*Type) &&
-                   AddressType->isPointer() && AddressType->addressSpace() == 0 &&
-                   AddressType->scalarBitWidth() == 64 &&
-                   *Bytes < ((Type->scalarBitWidth() + 7U) / 8U) &&
-                   (*Bytes == 1 || *Bytes == 2 || *Bytes == 4) &&
-                   isSupportedNonAtomicMemoryAccess(Instruction)
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_FPEXTLOAD: {
-    const auto Bytes = memoryAccessBytes(Instruction);
-    const std::optional<LowLevelType> AddressType = operandType(Function, Instruction, 1);
-    return Type && Bytes && AddressType && Type->isCategoryNeutralScalar() &&
-                   Type->scalarBitWidth() == 64 && AddressType->isPointer() &&
-                   AddressType->addressSpace() == 0 && AddressType->scalarBitWidth() == 64 &&
-                   *Bytes == 4 && isSupportedNonAtomicMemoryAccess(Instruction)
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_FPTRUNCSTORE: {
-    const std::optional<LowLevelType> ValueType = operandType(Function, Instruction, 0);
-    const std::optional<LowLevelType> AddressType = operandType(Function, Instruction, 1);
-    const auto Bytes = memoryAccessBytes(Instruction);
-    return ValueType && AddressType && Bytes && ValueType->isCategoryNeutralScalar() &&
-                   ValueType->scalarBitWidth() == 64 && AddressType->isPointer() &&
-                   AddressType->addressSpace() == 0 && AddressType->scalarBitWidth() == 64 &&
-                   *Bytes == 4 && isSupportedNonAtomicMemoryAccess(Instruction)
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_LOAD:
-    if (Type) {
-      const std::optional<LowLevelType> AddressType = operandType(Function, Instruction, 1);
-      if (!AddressType || !AddressType->isPointer() || AddressType->addressSpace() != 0 ||
-          AddressType->scalarBitWidth() != 64) {
-        return LegalizationAction::Unsupported;
-      }
-      if (Instruction.memOperands().size() == 1 && Instruction.memOperands().front()->isAtomic()) {
-        const std::uint64_t Bytes = Type->scalarBitWidth() / 8U;
-        return isSupportedIntegerType(*Type) &&
-                       isSupportedAtomicMemoryAccess(Instruction, AtomicOperation::Load, Bytes)
-                   ? LegalizationAction::Legal
-                   : LegalizationAction::Unsupported;
-      }
-      if (isSupportedIntegerType(*Type) && isSupportedNonAtomicMemoryAccess(Instruction) &&
-          memoryMatchesType(Instruction, *Type)) {
-        return LegalizationAction::Legal;
-      }
-    }
-    return LegalizationAction::Unsupported;
-  case GenericOpcode::G_FADD:
-  case GenericOpcode::G_FSUB:
-  case GenericOpcode::G_FMUL:
-  case GenericOpcode::G_FDIV: {
-    const std::optional<LowLevelType> Left = operandType(Function, Instruction, 1);
-    const std::optional<LowLevelType> Right = operandType(Function, Instruction, 2);
-    return Type && Left && Right && isSupportedFloatingScalar(*Type) && *Left == *Type &&
-                   *Right == *Type
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_FMA: {
-    const std::optional<LowLevelType> First = operandType(Function, Instruction, 1);
-    const std::optional<LowLevelType> Second = operandType(Function, Instruction, 2);
-    const std::optional<LowLevelType> Third = operandType(Function, Instruction, 3);
-    return Type && First && Second && Third && isSupportedFloatingScalar(*Type) &&
-                   *First == *Type && *Second == *Type && *Third == *Type
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_FNEG:
-  case GenericOpcode::G_FABS:
-  case GenericOpcode::G_FSQRT: {
-    const std::optional<LowLevelType> Source = operandType(Function, Instruction, 1);
-    return Type && Source && isSupportedFloatingScalar(*Type) && *Source == *Type
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_FCOPYSIGN: {
-    const std::optional<LowLevelType> Left = operandType(Function, Instruction, 1);
-    const std::optional<LowLevelType> Right = operandType(Function, Instruction, 2);
-    return Type && Left && Right && isSupportedFloatingScalar(*Type) && *Left == *Type &&
-                   *Right == *Type
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_FCONSTANT: {
-    const MachineOperand *Immediate = Instruction.operandAt(1);
-    const auto *Floating =
-        Immediate == nullptr ? nullptr : Immediate->getIf<FloatingImmediateOperand>();
-    return Type && isSupportedFloatingScalar(*Type) && Floating != nullptr &&
-                   Floating->Bits.bitWidth() == Type->scalarBitWidth()
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_ASSERT_SEXT:
-  case GenericOpcode::G_ASSERT_ZEXT:
-  case GenericOpcode::G_ASSERT_ALIGN: {
-    const std::optional<LowLevelType> Source = operandType(Function, Instruction, 1);
-    return Type && Source && *Type == *Source && isScalarGPRType(*Type) &&
-                   Instruction.operandCount() == 3
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_FREEZE:
-  case GenericOpcode::G_CONSTANT_FOLD_BARRIER: {
-    const std::optional<LowLevelType> Source = operandType(Function, Instruction, 1);
-    return Type && Source && *Type == *Source && isSupportedIntegerType(*Type)
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_PTRTOINT: {
-    const std::optional<LowLevelType> Source = operandType(Function, Instruction, 1);
-    return Type && Source && Type->isCategoryNeutralScalar() && Type->scalarBitWidth() == 64 &&
-                   Source->isPointer() && Source->scalarBitWidth() == 64
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_INTTOPTR: {
-    const std::optional<LowLevelType> Source = operandType(Function, Instruction, 1);
-    return Type && Source && Type->isPointer() && Type->scalarBitWidth() == 64 &&
-                   Source->isCategoryNeutralScalar() && Source->scalarBitWidth() == 64
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_BITCAST: {
-    const std::optional<LowLevelType> Source = operandType(Function, Instruction, 1);
-    return Type && Source && *Type != *Source && Type->scalarBitWidth() == Source->scalarBitWidth()
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_ANYEXT:
-  case GenericOpcode::G_SEXT:
-  case GenericOpcode::G_ZEXT: {
-    const std::optional<LowLevelType> Source = operandType(Function, Instruction, 1);
-    if (!Type || !Source)
-      return LegalizationAction::Unsupported;
-    if (!isScalarIntegerExtension(*Type, *Source)) {
-      const MachineOperand *DestinationOperand = Instruction.operandAt(0);
-      const Register *Destination =
-          DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
-      MachineInstr *OnlyUse =
-          Destination == nullptr ? nullptr : singleUseInstruction(Function, *Destination);
-      return Type->isCategoryNeutralScalar() && Source->isCategoryNeutralScalar() &&
-                     Type->scalarBitWidth() > 64 &&
-                     Source->scalarBitWidth() < Type->scalarBitWidth() && OnlyUse != nullptr &&
-                     OnlyUse->opcode() == opcode(GenericOpcode::G_UNMERGE_VALUES)
-                 ? LegalizationAction::Legal
-                 : LegalizationAction::Unsupported;
-    }
-    const MachineOperand *SourceOperand = Instruction.operandAt(1);
-    const Register *SourceRegister =
-        SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>();
-    const MachineOperand *SourceDefinition =
-        SourceRegister == nullptr || !SourceRegister->isVirtual()
-            ? nullptr
-            : Function.registerInfo().getUniqueVRegDef(*SourceRegister);
-    const MachineInstr *SourceDef =
-        SourceDefinition == nullptr ? nullptr : SourceDefinition->parent();
-    if (SourceDef != nullptr && SourceDef->opcode() == opcode(GenericOpcode::G_CONSTANT))
-      return LegalizationAction::Custom;
-    if (Generic == GenericOpcode::G_ANYEXT &&
-        narrowAnyExtBinaryPlan(Instruction, Function).has_value())
-      return LegalizationAction::Custom;
-    return LegalizationAction::Legal;
-  }
-  case GenericOpcode::G_TRUNC: {
-    const std::optional<LowLevelType> Source = operandType(Function, Instruction, 1);
-    if (Type && Source && Type->isCategoryNeutralScalar() && Source->isCategoryNeutralScalar() &&
-        Type->scalarBitWidth() > 64 && Source->scalarBitWidth() > Type->scalarBitWidth())
-      return LegalizationAction::Legal;
-    return Type && Source && isSupportedIntegerScalar(*Type) && isSupportedIntegerScalar(*Source) &&
-                   Type->scalarBitWidth() != 0 && Source->scalarBitWidth() >= Type->scalarBitWidth()
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_MERGE_VALUES:
-  case GenericOpcode::G_UNMERGE_VALUES: {
-    if (Instruction.operandCount() < 3)
-      return LegalizationAction::Unsupported;
-    const std::size_t SmallTypeIndex = Generic == GenericOpcode::G_MERGE_VALUES ? 1 : 0;
-    const std::size_t BigTypeIndex =
-        Generic == GenericOpcode::G_MERGE_VALUES ? 0 : Instruction.operandCount() - 1U;
-    const std::optional<LowLevelType> SmallType =
-        operandType(Function, Instruction, SmallTypeIndex);
-    const std::optional<LowLevelType> BigType = operandType(Function, Instruction, BigTypeIndex);
-    if (SmallType && BigType && Generic == GenericOpcode::G_MERGE_VALUES &&
-        BigType->scalarBitWidth() > 64 && !canFoldWideMergeImmediately(Instruction, Function))
-      return LegalizationAction::Legal;
-    return SmallType && BigType && SmallType->isCategoryNeutralScalar() &&
-                   BigType->isCategoryNeutralScalar() && SmallType->scalarBitWidth() != 0 &&
-                   BigType->scalarBitWidth() != 0
-               ? LegalizationAction::Custom
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_SEXT_INREG: {
-    const std::optional<LowLevelType> Source = operandType(Function, Instruction, 1);
-    const MachineOperand *WidthOperand = Instruction.operandAt(2);
-    const auto Width =
-        WidthOperand == nullptr
-            ? Expected<std::int64_t>(std::unexpected(
-                  Error(ErrorCode::InvalidArgument, "missing in-register extension width")))
-            : signedImmediate(*WidthOperand);
-    return Type && Source && *Type == *Source && isScalarGPRType(*Type) && Width && *Width > 0 &&
-                   static_cast<std::uint64_t>(*Width) <= Type->scalarBitWidth()
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_ICMP: {
-    const MachineOperand *Predicate = Instruction.operandAt(1);
-    const auto *IntegerPredicate =
-        Predicate == nullptr ? nullptr : Predicate->getIf<PredicateOperand>();
-    const std::optional<LowLevelType> Left = operandType(Function, Instruction, 2);
-    const std::optional<LowLevelType> Right = operandType(Function, Instruction, 3);
-    if (!Type || !isSupportedIntegerScalar(*Type) || IntegerPredicate == nullptr ||
-        IntegerPredicate->integerPredicate() == nullptr || !Left || !Right || *Left != *Right ||
-        !isSupportedIntegerType(*Left)) {
-      return LegalizationAction::Unsupported;
-    }
-    return Type->scalarBitWidth() == 64 && Left->scalarBitWidth() == 64
-               ? LegalizationAction::Legal
-               : LegalizationAction::Custom;
-  }
-  case GenericOpcode::G_FCMP: {
-    const MachineOperand *Predicate = Instruction.operandAt(1);
-    const auto *FloatingPredicate =
-        Predicate == nullptr ? nullptr : Predicate->getIf<PredicateOperand>();
-    const std::optional<LowLevelType> Left = operandType(Function, Instruction, 2);
-    const std::optional<LowLevelType> Right = operandType(Function, Instruction, 3);
-    return Type && isI1Type(*Type) && FloatingPredicate != nullptr &&
-                   FloatingPredicate->floatingPredicate() != nullptr && Left && Right &&
-                   *Left == *Right && isSupportedFloatingScalar(*Left)
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_FPEXT:
-  case GenericOpcode::G_FPTRUNC: {
-    const std::optional<LowLevelType> Source = operandType(Function, Instruction, 1);
-    const bool IsExtension = Generic == GenericOpcode::G_FPEXT;
-    return Type && Source && isSupportedFloatingScalar(*Type) &&
-                   isSupportedFloatingScalar(*Source) &&
-                   ((IsExtension && Source->scalarBitWidth() == 32 &&
-                     Type->scalarBitWidth() == 64) ||
-                    (!IsExtension && Source->scalarBitWidth() == 64 &&
-                     Type->scalarBitWidth() == 32))
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_FPTOSI:
-  case GenericOpcode::G_FPTOUI: {
-    const std::optional<LowLevelType> Source = operandType(Function, Instruction, 1);
-    return Type && Source && isSupportedIntegerType(*Type) && isSupportedFloatingScalar(*Source)
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_SITOFP:
-  case GenericOpcode::G_UITOFP: {
-    const std::optional<LowLevelType> Source = operandType(Function, Instruction, 1);
-    return Type && Source && isSupportedFloatingScalar(*Type) && isSupportedIntegerType(*Source)
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_FMINNUM:
-  case GenericOpcode::G_FMAXNUM: {
-    const std::optional<LowLevelType> Left = operandType(Function, Instruction, 1);
-    const std::optional<LowLevelType> Right = operandType(Function, Instruction, 2);
-    return Type && Left && Right && isSupportedFloatingScalar(*Type) && *Left == *Type &&
-                   *Right == *Type
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_SELECT: {
-    const std::optional<LowLevelType> Condition = operandType(Function, Instruction, 1);
-    const std::optional<LowLevelType> TrueValue = operandType(Function, Instruction, 2);
-    const std::optional<LowLevelType> FalseValue = operandType(Function, Instruction, 3);
-    return Type && Condition && TrueValue && FalseValue && isScalarGPRType(*Condition) &&
-                   *TrueValue == *Type && *FalseValue == *Type && isSupportedIntegerType(*Type)
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_PTRMASK: {
-    const std::optional<LowLevelType> Pointer = operandType(Function, Instruction, 1);
-    const std::optional<LowLevelType> Mask = operandType(Function, Instruction, 2);
-    return Type && Pointer && Mask && Type->isPointer() && Type->addressSpace() == 0 &&
-                   Type->scalarBitWidth() == 64 && *Pointer == *Type &&
-                   isSupportedIntegerScalar(*Mask) && Mask->scalarBitWidth() == 64
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_ABS: {
-    const std::optional<LowLevelType> Source = operandType(Function, Instruction, 1);
-    return Type && Source && *Source == *Type && isSupportedIntegerScalar(*Type)
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_SDIVREM:
-  case GenericOpcode::G_UDIVREM: {
-    const std::optional<LowLevelType> Remainder = operandType(Function, Instruction, 1);
-    const std::optional<LowLevelType> Left = operandType(Function, Instruction, 2);
-    const std::optional<LowLevelType> Right = operandType(Function, Instruction, 3);
-    return Type && Remainder && Left && Right && isSupportedIntegerScalar(*Type) &&
-                   *Remainder == *Type && *Left == *Type && *Right == *Type
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_SMULH:
-  case GenericOpcode::G_UMULH: {
-    const std::optional<LowLevelType> Left = operandType(Function, Instruction, 1);
-    const std::optional<LowLevelType> Right = operandType(Function, Instruction, 2);
-    return Type && Left && Right && isNativeIntegerScalar(*Type) && *Left == *Type &&
-                   *Right == *Type
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_SMIN:
-  case GenericOpcode::G_SMAX:
-  case GenericOpcode::G_UMIN:
-  case GenericOpcode::G_UMAX: {
-    const std::optional<LowLevelType> Left = operandType(Function, Instruction, 1);
-    const std::optional<LowLevelType> Right = operandType(Function, Instruction, 2);
-    return Type && Left && Right && isSupportedIntegerScalar(*Type) && *Left == *Type &&
-                   *Right == *Type
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_UADDO:
-  case GenericOpcode::G_USUBO:
-  case GenericOpcode::G_SADDO:
-  case GenericOpcode::G_SSUBO:
-  case GenericOpcode::G_UMULO:
-  case GenericOpcode::G_SMULO: {
-    const std::optional<LowLevelType> Overflow = operandType(Function, Instruction, 1);
-    const std::optional<LowLevelType> Left = operandType(Function, Instruction, 2);
-    const std::optional<LowLevelType> Right = operandType(Function, Instruction, 3);
-    return Type && Overflow && Left && Right && isNativeIntegerScalar(*Type) &&
-                   isI1Type(*Overflow) && *Left == *Type && *Right == *Type
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_UADDE:
-  case GenericOpcode::G_USUBE:
-  case GenericOpcode::G_SADDE:
-  case GenericOpcode::G_SSUBE: {
-    const std::optional<LowLevelType> CarryOut = operandType(Function, Instruction, 1);
-    const std::optional<LowLevelType> Left = operandType(Function, Instruction, 2);
-    const std::optional<LowLevelType> Right = operandType(Function, Instruction, 3);
-    const std::optional<LowLevelType> CarryIn = operandType(Function, Instruction, 4);
-    return Type && CarryOut && Left && Right && CarryIn && isNativeIntegerScalar(*Type) &&
-                   isI1Type(*CarryOut) && isI1Type(*CarryIn) && *Left == *Type && *Right == *Type
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_ATOMICRMW_XCHG:
-  case GenericOpcode::G_ATOMICRMW_ADD:
-  case GenericOpcode::G_ATOMICRMW_SUB:
-  case GenericOpcode::G_ATOMICRMW_AND:
-  case GenericOpcode::G_ATOMICRMW_OR:
-  case GenericOpcode::G_ATOMICRMW_XOR:
-  case GenericOpcode::G_ATOMICRMW_MAX:
-  case GenericOpcode::G_ATOMICRMW_MIN:
-  case GenericOpcode::G_ATOMICRMW_UMAX:
-  case GenericOpcode::G_ATOMICRMW_UMIN: {
-    const std::optional<LowLevelType> Address = operandType(Function, Instruction, 1);
-    const std::optional<LowLevelType> Value = operandType(Function, Instruction, 2);
-    const std::uint64_t Bytes = Type ? Type->scalarBitWidth() / 8U : 0;
-    return Type && Address && Value && isNativeIntegerScalar(*Type) && *Value == *Type &&
-                   Address->isPointer() && Address->addressSpace() == 0 &&
-                   Address->scalarBitWidth() == 64 &&
-                   isSupportedAtomicMemoryAccess(Instruction, AtomicOperation::ReadModifyWrite,
-                                                 Bytes)
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_ATOMIC_CMPXCHG: {
-    const std::optional<LowLevelType> Address = operandType(Function, Instruction, 1);
-    const std::optional<LowLevelType> Compare = operandType(Function, Instruction, 2);
-    const std::optional<LowLevelType> NewValue = operandType(Function, Instruction, 3);
-    if (!Type || !Address || !Compare || !NewValue || *Compare != *Type || *NewValue != *Type ||
-        !Type->isCategoryNeutralScalar() ||
-        (Type->scalarBitWidth() != 8 && Type->scalarBitWidth() != 16 &&
-         Type->scalarBitWidth() != 32 && Type->scalarBitWidth() != 64) ||
-        !Address->isPointer() || Address->addressSpace() != 0 || Address->scalarBitWidth() != 64 ||
-        !isSupportedAtomicContainerAccess(Instruction, AtomicOperation::CompareExchange,
-                                          (Type->scalarBitWidth() + 7U) / 8U)) {
-      return LegalizationAction::Unsupported;
-    }
-    return Type->scalarBitWidth() == 64 ? LegalizationAction::Legal : LegalizationAction::Custom;
-  }
-  case GenericOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
-    const std::optional<LowLevelType> Success = operandType(Function, Instruction, 1);
-    const std::optional<LowLevelType> Address = operandType(Function, Instruction, 2);
-    const std::optional<LowLevelType> Compare = operandType(Function, Instruction, 3);
-    const std::optional<LowLevelType> NewValue = operandType(Function, Instruction, 4);
-    return Type && Success && Address && Compare && NewValue && isI1Type(*Success) &&
-                   *Compare == *Type && *NewValue == *Type && Type->isCategoryNeutralScalar() &&
-                   (Type->scalarBitWidth() == 8 || Type->scalarBitWidth() == 16 ||
-                    Type->scalarBitWidth() == 32 || Type->scalarBitWidth() == 64) &&
-                   Address->isPointer() && Address->addressSpace() == 0 &&
-                   Address->scalarBitWidth() == 64 &&
-                   isSupportedAtomicContainerAccess(Instruction, AtomicOperation::CompareExchange,
-                                                    (Type->scalarBitWidth() + 7U) / 8U)
-               ? LegalizationAction::Custom
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_FENCE: {
-    const MachineOperand *OrderingOperand = Instruction.operandAt(0);
-    const MachineOperand *ScopeOperand = Instruction.operandAt(1);
-    const auto Ordering = OrderingOperand == nullptr
-                              ? Expected<std::int64_t>(std::unexpected(
-                                    Error(ErrorCode::InvalidArgument, "missing fence ordering")))
-                              : signedImmediate(*OrderingOperand);
-    const auto Scope = ScopeOperand == nullptr
-                           ? Expected<std::int64_t>(std::unexpected(
-                                 Error(ErrorCode::InvalidArgument, "missing fence scope")))
-                           : signedImmediate(*ScopeOperand);
-    return Ordering && Scope && *Ordering >= 4 && *Ordering <= 7 && *Scope == 0
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_CONSTANT_POOL: {
-    const MachineOperand *PoolOperand = Instruction.operandAt(1);
-    const auto *Pool = PoolOperand == nullptr ? nullptr : PoolOperand->getIf<ConstantPoolOperand>();
-    return Type && Type->isPointer() && Type->addressSpace() == 0 && Type->scalarBitWidth() == 64 &&
-                   Pool != nullptr && Function.constantPool().entry(Pool->Index) != nullptr
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_GLOBAL_VALUE: {
-    const MachineOperand *GlobalOperand = Instruction.operandAt(1);
-    const auto *Global =
-        GlobalOperand == nullptr ? nullptr : GlobalOperand->getIf<GlobalSymbolOperand>();
-    return Type && Type->isPointer() && Type->addressSpace() == 0 && Type->scalarBitWidth() == 64 &&
-                   Global != nullptr && !Global->Name.empty()
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_BRINDIRECT:
-    return operandType(Function, Instruction, 0)
-                   .transform([](LowLevelType Value) {
-                     return Value.isPointer() && Value.addressSpace() == 0 &&
-                            Value.scalarBitWidth() == 64;
-                   })
-                   .value_or(false)
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  case GenericOpcode::G_TRAP:
-  case GenericOpcode::G_DEBUGTRAP:
-    return Instruction.operandCount() == 0 ? LegalizationAction::Legal
-                                           : LegalizationAction::Unsupported;
-  case GenericOpcode::G_UBSANTRAP:
-    return Instruction.operandCount() == 1 ? LegalizationAction::Legal
-                                           : LegalizationAction::Unsupported;
-  case GenericOpcode::G_CONSTANT: {
-    const MachineOperand *Immediate = Instruction.operandAt(1);
-    if (!Type || Immediate == nullptr ||
-        (Immediate->kind() != MachineOperand::Kind::Immediate &&
-         Immediate->kind() != MachineOperand::Kind::IntegerImmediate))
-      return LegalizationAction::Unsupported;
-    if (Type->scalarBitWidth() > 64)
-      return wideConstantUnmergePlan(Instruction, Function).has_value()
-                 ? LegalizationAction::Custom
-                 : LegalizationAction::Unsupported;
-    if (Type->isCategoryNeutralScalar() && Type->scalarBitWidth() == 64) {
-      auto Value = signedImmediate(*Immediate);
-      if (Value && shouldUseRV64IntegerConstantPool(*Value))
-        return LegalizationAction::Custom;
-    }
-    return isSupportedIntegerType(*Type) ? LegalizationAction::Legal
-                                         : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_ADD:
-  case GenericOpcode::G_SUB:
-  case GenericOpcode::G_MUL:
-  case GenericOpcode::G_SDIV:
-  case GenericOpcode::G_UDIV:
-  case GenericOpcode::G_SREM:
-  case GenericOpcode::G_UREM:
-  case GenericOpcode::G_AND:
-  case GenericOpcode::G_OR:
-  case GenericOpcode::G_XOR:
-  case GenericOpcode::G_SHL:
-  case GenericOpcode::G_LSHR:
-  case GenericOpcode::G_ASHR: {
-    const std::optional<LowLevelType> Left = operandType(Function, Instruction, 1);
-    const std::optional<LowLevelType> Right = operandType(Function, Instruction, 2);
-    if (!Type || !Left || !Right || *Left != *Type || *Right != *Type ||
-        !Type->isCategoryNeutralScalar())
-      return LegalizationAction::Unsupported;
-    if (Type->scalarBitWidth() > 64) {
-      return Generic == GenericOpcode::G_ADD || Generic == GenericOpcode::G_SUB ||
-                     Generic == GenericOpcode::G_AND || Generic == GenericOpcode::G_OR ||
-                     Generic == GenericOpcode::G_XOR || Generic == GenericOpcode::G_SHL
-                 ? LegalizationAction::Custom
-                 : LegalizationAction::Unsupported;
-    }
-    if (Generic == GenericOpcode::G_MUL && !Subtarget.features().has(RISCVExtension::M)) {
-      if (Type->scalarBitWidth() == 64)
-        return LegalizationAction::Libcall;
-      const MachineOperand *DestinationOperand = Instruction.operandAt(0);
-      const Register *Destination =
-          DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
-      MachineInstr *Extension =
-          Destination == nullptr ? nullptr : singleRegisterUse(Function, *Destination);
-      const std::optional<LowLevelType> ExtensionType =
-          Extension == nullptr ? std::nullopt : resultType(Function, *Extension);
-      if (Extension != nullptr && Extension->opcode() == opcode(GenericOpcode::G_ANYEXT) &&
-          ExtensionType && ExtensionType->scalarBitWidth() == 64) {
-        return LegalizationAction::Legal;
-      }
-      return LegalizationAction::Custom;
-    }
-    return isSupportedIntegerScalar(*Type) ? LegalizationAction::Legal
-                                           : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_PTR_ADD: {
-    const std::optional<LowLevelType> Pointer = operandType(Function, Instruction, 1);
-    const std::optional<LowLevelType> Offset = operandType(Function, Instruction, 2);
-    return Type && Pointer && Offset && Type->isPointer() && Type->addressSpace() == 0 &&
-                   Type->scalarBitWidth() == 64 && *Pointer == *Type &&
-                   isSupportedIntegerScalar(*Offset) && Offset->scalarBitWidth() == 64
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  default:
-    break;
-  }
-
-  // Keep the unsupported policy mechanically synchronized with the complete
-  // target-independent opcode inventory. Supported cases return above.
-  switch (Generic) {
-#define RCC_CORE_OPCODE(Name)
-#define RCC_GENERIC_OPCODE(Name) case GenericOpcode::Name:
-#include "rcc/CodeGen/GenericOpcode.def"
-#undef RCC_GENERIC_OPCODE
-#undef RCC_CORE_OPCODE
-    return LegalizationAction::Unsupported;
-  case GenericOpcode::Invalid:
-  case GenericOpcode::CORE_PSEUDO_END:
-  case GenericOpcode::G_GENERIC_END:
-  default:
-    break;
-  }
-  return makeUnexpected(ErrorCode::InternalError,
-                        "RISC-V legalizer received an invalid generic opcode value");
-}
-
-Expected<void> RISCVLegalizerInfo::lower(MachineInstr &Instruction, MachineFunction &Function,
-                                         const TargetDescription &Target,
-                                         LegalizationAction Action) const {
-  if (Action == LegalizationAction::Custom &&
-      Instruction.opcode() == opcode(GenericOpcode::G_INTRINSIC_W_SIDE_EFFECTS)) {
-    return lowerVACopy(Instruction, Function, Target);
-  }
-  if (Action == LegalizationAction::Custom &&
-      Instruction.opcode() == opcode(GenericOpcode::G_VAARG)) {
-    return lowerVAArg(Instruction, Function, Target);
-  }
-  if (Instruction.opcode() == opcode(GenericOpcode::G_MUL) &&
-      !Subtarget.features().has(RISCVExtension::M)) {
-    if (Action == LegalizationAction::Custom)
-      return widenNarrowMultiply(Instruction, Function, Target);
-    if (Action == LegalizationAction::Libcall)
-      return lowerMultiplyLibcall(Instruction, Function, Target,
-                                  Subtarget.features().has(RISCVExtension::D)
-                                      ? CallingConvention::LP64D
-                                      : CallingConvention::LP64,
-                                  *Subtarget.getCallLowering());
-  }
-  if (Action == LegalizationAction::Custom &&
-      resultType(Function, Instruction)
-          .transform([](LowLevelType Type) { return Type.scalarBitWidth() > 64; })
-          .value_or(false)) {
-    const Opcode Value = Instruction.opcode();
-    if (Value == opcode(GenericOpcode::G_ADD) || Value == opcode(GenericOpcode::G_SUB) ||
-        Value == opcode(GenericOpcode::G_AND) || Value == opcode(GenericOpcode::G_OR) ||
-        Value == opcode(GenericOpcode::G_XOR)) {
-      return lowerWideBinary(Instruction, Function, Target,
-                             static_cast<GenericOpcode>(Value.value()));
-    }
-    if (Value == opcode(GenericOpcode::G_SHL))
-      return lowerWideShiftLeft(Instruction, Function, Target);
-    if (Value == opcode(GenericOpcode::G_PHI))
-      return lowerWidePhi(Instruction, Function, Target);
-  }
-  if (Action == LegalizationAction::Custom &&
-      Instruction.opcode() == opcode(GenericOpcode::G_MERGE_VALUES))
-    return lowerMergeValues(Instruction, Function, Target);
-  if (Action == LegalizationAction::Custom &&
-      Instruction.opcode() == opcode(GenericOpcode::G_UNMERGE_VALUES))
-    return lowerUnmergeValues(Instruction, Function, Target);
-  if (Action == LegalizationAction::Custom &&
-      Instruction.opcode() == opcode(GenericOpcode::G_BRCOND))
-    return lowerNarrowBranchCondition(Instruction, Function, Target);
-  if (Action == LegalizationAction::Custom && Instruction.opcode() == opcode(GenericOpcode::G_PHI))
-    return lowerNarrowPhi(Instruction, Function, Target);
-  if (Action == LegalizationAction::Custom && Instruction.opcode() == opcode(GenericOpcode::G_ICMP))
+RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &Subtarget) {
+  const LowLevelType S8 = *LowLevelType::makeCategoryNeutralScalar(8);
+  const LowLevelType S16 = *LowLevelType::makeCategoryNeutralScalar(16);
+  const LowLevelType S32 = *LowLevelType::makeCategoryNeutralScalar(32);
+  const LowLevelType S64 = *LowLevelType::makeCategoryNeutralScalar(64);
+  const LowLevelType S128 = *LowLevelType::makeCategoryNeutralScalar(128);
+  const LowLevelType P0 = *LowLevelType::makePointer(0, 64);
+  const bool HasM = Subtarget.features().has(RISCVExtension::M);
+  const bool HasA = Subtarget.features().has(RISCVExtension::A);
+  const bool HasF = Subtarget.features().has(RISCVExtension::F);
+  const bool HasD = Subtarget.features().has(RISCVExtension::D);
+
+  getActionDefinitionsBuilder({opcode(GenericOpcode::G_ASSERT_SEXT),
+                               opcode(GenericOpcode::G_ASSERT_ZEXT),
+                               opcode(GenericOpcode::G_ASSERT_ALIGN)})
+      .alwaysLegal();
+
+  getActionDefinitionsBuilder({opcode(GenericOpcode::G_ADD), opcode(GenericOpcode::G_SUB)})
+      .legalFor({S64})
+      .customFor({S32})
+      .widenScalarToNextPow2(0)
+      .clampScalar(0, S64, S64);
+
+  getActionDefinitionsBuilder(
+      {opcode(GenericOpcode::G_AND), opcode(GenericOpcode::G_OR), opcode(GenericOpcode::G_XOR)})
+      .legalFor({S64})
+      .widenScalarToNextPow2(0)
+      .clampScalar(0, S64, S64);
+
+  getActionDefinitionsBuilder({opcode(GenericOpcode::G_UADDE), opcode(GenericOpcode::G_UADDO),
+                               opcode(GenericOpcode::G_USUBE), opcode(GenericOpcode::G_USUBO),
+                               opcode(GenericOpcode::G_READ_REGISTER),
+                               opcode(GenericOpcode::G_WRITE_REGISTER)})
+      .lower();
+  getActionDefinitionsBuilder({opcode(GenericOpcode::G_SADDE), opcode(GenericOpcode::G_SADDO),
+                               opcode(GenericOpcode::G_SSUBE), opcode(GenericOpcode::G_SSUBO)})
+      .minScalar(0, S64)
+      .lower();
+  getActionDefinitionsBuilder({opcode(GenericOpcode::G_UADDSAT), opcode(GenericOpcode::G_SADDSAT),
+                               opcode(GenericOpcode::G_USUBSAT), opcode(GenericOpcode::G_SSUBSAT),
+                               opcode(GenericOpcode::G_SSHLSAT), opcode(GenericOpcode::G_USHLSAT)})
+      .lower();
+
+  getActionDefinitionsBuilder(
+      {opcode(GenericOpcode::G_SHL), opcode(GenericOpcode::G_ASHR), opcode(GenericOpcode::G_LSHR)})
+      .legalFor({{S64, S64}})
+      .customFor({{S32, S32}})
+      .widenScalarToNextPow2(0)
+      .clampScalar(1, S64, S64)
+      .clampScalar(0, S64, S64);
+
+  getActionDefinitionsBuilder({opcode(GenericOpcode::G_ZEXT), opcode(GenericOpcode::G_SEXT),
+                               opcode(GenericOpcode::G_ANYEXT)})
+      .legalFor({{S32, S16}, {S64, S16}, {S64, S32}})
+      .maxScalar(0, S64);
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_TRUNC)).alwaysLegal();
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_SEXT_INREG))
+      .customFor({S64})
+      .clampScalar(0, S64, S64)
+      .lower();
+
+  for (const GenericOpcode Generic :
+       {GenericOpcode::G_MERGE_VALUES, GenericOpcode::G_UNMERGE_VALUES}) {
+    const std::size_t BigTypeIndex = Generic == GenericOpcode::G_MERGE_VALUES ? 0 : 1;
+    const std::size_t LittleTypeIndex = Generic == GenericOpcode::G_MERGE_VALUES ? 1 : 0;
+    getActionDefinitionsBuilder(opcode(Generic))
+        .widenScalarToNextPow2(LittleTypeIndex, 64)
+        .widenScalarToNextPow2(BigTypeIndex, 64)
+        .clampScalar(LittleTypeIndex, S64, S64)
+        .clampScalar(BigTypeIndex, S64, S64);
+  }
+  getActionDefinitionsBuilder({opcode(GenericOpcode::G_FSHL), opcode(GenericOpcode::G_FSHR)})
+      .lower();
+  getActionDefinitionsBuilder({opcode(GenericOpcode::G_ROTR), opcode(GenericOpcode::G_ROTL)})
+      .lower();
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_BITREVERSE)).maxScalar(0, S64).lower();
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_BSWAP)).maxScalar(0, S64).lower();
+
+  getActionDefinitionsBuilder(
+      {opcode(GenericOpcode::G_CTLZ), opcode(GenericOpcode::G_CTTZ), opcode(GenericOpcode::G_CTLS)})
+      .maxScalar(0, S64)
+      .scalarSameSizeAs(1, 0)
+      .lower();
+  getActionDefinitionsBuilder(
+      {opcode(GenericOpcode::G_CTLZ_ZERO_POISON), opcode(GenericOpcode::G_CTTZ_ZERO_POISON)})
+      .maxScalar(0, S64)
+      .scalarSameSizeAs(1, 0)
+      .lower();
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_CTPOP))
+      .widenScalarToNextPow2(0, 8)
+      .clampScalar(0, S8, S64)
+      .scalarSameSizeAs(1, 0)
+      .lower();
+
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_CONSTANT))
+      .legalFor({P0})
+      .customFor({S64})
+      .widenScalarToNextPow2(0)
+      .clampScalar(0, S64, S64);
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_FREEZE))
+      .legalFor({S16, S32, S64, P0})
+      .widenScalarToNextPow2(0)
+      .clampScalar(0, S16, S64);
+  getActionDefinitionsBuilder(
+      {opcode(GenericOpcode::G_IMPLICIT_DEF), opcode(GenericOpcode::G_CONSTANT_FOLD_BARRIER)})
+      .legalFor({S32, S64, P0})
+      .widenScalarToNextPow2(0)
+      .clampScalar(0, S32, S64);
+
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_ICMP))
+      .legalFor({{S64, S64}, {S64, P0}})
+      .customIf([](const LegalizationQuery &Query) {
+        return Query.Types.size() == 2 && Query.Types[0].isCategoryNeutralScalar() &&
+               (Query.Types[1].isCategoryNeutralScalar() || Query.Types[1].isPointer());
+      });
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_SELECT))
+      .legalFor({{S32, S64}, {S64, S64}, {P0, S64}})
+      .widenScalarToNextPow2(0)
+      .clampScalar(0, S32, S64)
+      .clampScalar(1, S64, S64);
+
+  auto &LoadRules = getActionDefinitionsBuilder(opcode(GenericOpcode::G_LOAD));
+  auto &StoreRules = getActionDefinitionsBuilder(opcode(GenericOpcode::G_STORE));
+  auto &ExtLoadRules = getActionDefinitionsBuilder(
+      {opcode(GenericOpcode::G_SEXTLOAD), opcode(GenericOpcode::G_ZEXTLOAD)});
+  const std::uint64_t Align8 =
+      Subtarget.features().has(RISCVSubtargetOption::UnalignedScalarMemory) ? 8 : 8;
+  const std::uint64_t Align16 =
+      Subtarget.features().has(RISCVSubtargetOption::UnalignedScalarMemory) ? 8 : 16;
+  const std::uint64_t Align32 =
+      Subtarget.features().has(RISCVSubtargetOption::UnalignedScalarMemory) ? 8 : 32;
+  const std::uint64_t Align64 =
+      Subtarget.features().has(RISCVSubtargetOption::UnalignedScalarMemory) ? 8 : 64;
+  LoadRules
+      .legalForTypesWithMemDesc({{S16, P0, S8, Align8},
+                                 {S32, P0, S8, Align8},
+                                 {S16, P0, S16, Align16},
+                                 {S32, P0, S16, Align16},
+                                 {S32, P0, S32, Align32},
+                                 {S64, P0, S8, Align8},
+                                 {S64, P0, S16, Align16},
+                                 {S64, P0, S32, Align32},
+                                 {S64, P0, S64, Align64},
+                                 {P0, P0, S64, Align64}})
+      .widenScalarToNextPow2(0, 8)
+      .lowerIfMemSizeNotByteSizePow2()
+      .clampScalar(0, S16, S64)
+      .lower();
+  StoreRules
+      .legalForTypesWithMemDesc({{S16, P0, S8, Align8},
+                                 {S32, P0, S8, Align8},
+                                 {S16, P0, S16, Align16},
+                                 {S32, P0, S16, Align16},
+                                 {S32, P0, S32, Align32},
+                                 {S64, P0, S8, Align8},
+                                 {S64, P0, S16, Align16},
+                                 {S64, P0, S32, Align32},
+                                 {S64, P0, S64, Align64},
+                                 {P0, P0, S64, Align64}})
+      .clampScalar(0, S16, S64)
+      .lowerIfMemSizeNotByteSizePow2()
+      .lower();
+  ExtLoadRules
+      .legalForTypesWithMemDesc(
+          {{S64, P0, S8, Align8}, {S64, P0, S16, Align16}, {S64, P0, S32, Align32}})
+      .widenScalarToNextPow2(0)
+      .clampScalar(0, S64, S64)
+      .lower();
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_FPEXTLOAD))
+      .legalForTypesWithMemDesc({{S64, P0, S32, Align32}})
+      .lower();
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_FPTRUNCSTORE))
+      .legalForTypesWithMemDesc({{S64, P0, S32, Align32}})
+      .lower();
+
+  getActionDefinitionsBuilder({opcode(GenericOpcode::G_PTR_ADD), opcode(GenericOpcode::G_PTRMASK)})
+      .legalFor({{P0, S64}});
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_PTRTOINT))
+      .legalFor({{S64, P0}})
+      .clampScalar(0, S64, S64);
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_INTTOPTR))
+      .legalFor({{P0, S64}})
+      .clampScalar(1, S64, S64);
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_BR)).alwaysLegal();
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_BRCOND)).legalFor({S64}).minScalar(0, S64);
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_BRJT)).customFor({{P0, S64}});
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_BRINDIRECT)).legalFor({P0});
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_PHI))
+      .legalFor({P0, S32, S64})
+      .widenScalarToNextPow2(0)
+      .clampScalar(0, S32, S64);
+  getActionDefinitionsBuilder(
+      {opcode(GenericOpcode::G_GLOBAL_VALUE), opcode(GenericOpcode::G_JUMP_TABLE),
+       opcode(GenericOpcode::G_CONSTANT_POOL), opcode(GenericOpcode::G_FRAME_INDEX)})
+      .legalFor({P0});
+
+  if (HasM) {
+    getActionDefinitionsBuilder(opcode(GenericOpcode::G_MUL))
+        .legalFor({S64})
+        .widenScalarToNextPow2(0)
+        .clampScalar(0, S64, S64);
+    getActionDefinitionsBuilder({opcode(GenericOpcode::G_SMULH), opcode(GenericOpcode::G_UMULH)})
+        .legalFor({S64})
+        .lower();
+    getActionDefinitionsBuilder({opcode(GenericOpcode::G_SMULO), opcode(GenericOpcode::G_UMULO)})
+        .minScalar(0, S64)
+        .lower();
+  } else {
+    getActionDefinitionsBuilder(opcode(GenericOpcode::G_MUL))
+        .libcallFor({S64, S128})
+        .widenScalarToNextPow2(0)
+        .clampScalar(0, S64, S128);
+    getActionDefinitionsBuilder({opcode(GenericOpcode::G_SMULH), opcode(GenericOpcode::G_UMULH)})
+        .lowerFor({S64});
+    getActionDefinitionsBuilder({opcode(GenericOpcode::G_SMULO), opcode(GenericOpcode::G_UMULO)})
+        .minScalar(0, S64)
+        .lower();
+  }
+  if (HasM) {
+    getActionDefinitionsBuilder({opcode(GenericOpcode::G_SDIV), opcode(GenericOpcode::G_UDIV),
+                                 opcode(GenericOpcode::G_UREM)})
+        .legalFor({S64})
+        .customFor({S32})
+        .libcallFor({S128})
+        .clampScalar(0, S32, S128)
+        .widenScalarToNextPow2(0);
+    getActionDefinitionsBuilder(opcode(GenericOpcode::G_SREM))
+        .legalFor({S64})
+        .libcallFor({S128})
+        .clampScalar(0, S64, S128)
+        .widenScalarToNextPow2(0);
+  } else {
+    getActionDefinitionsBuilder({opcode(GenericOpcode::G_SDIV), opcode(GenericOpcode::G_UDIV),
+                                 opcode(GenericOpcode::G_SREM), opcode(GenericOpcode::G_UREM)})
+        .libcallFor({S64, S128})
+        .clampScalar(0, S64, S128)
+        .widenScalarToNextPow2(0);
+  }
+  getActionDefinitionsBuilder({opcode(GenericOpcode::G_SDIVREM), opcode(GenericOpcode::G_UDIVREM)})
+      .lower();
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_ABS)).lower();
+  getActionDefinitionsBuilder({opcode(GenericOpcode::G_ABDS), opcode(GenericOpcode::G_ABDU)})
+      .lower();
+  getActionDefinitionsBuilder({opcode(GenericOpcode::G_UMAX), opcode(GenericOpcode::G_UMIN),
+                               opcode(GenericOpcode::G_SMAX), opcode(GenericOpcode::G_SMIN),
+                               opcode(GenericOpcode::G_SCMP), opcode(GenericOpcode::G_UCMP)})
+      .lower();
+  getActionDefinitionsBuilder({opcode(GenericOpcode::G_MEMCPY), opcode(GenericOpcode::G_MEMMOVE),
+                               opcode(GenericOpcode::G_MEMSET)})
+      .libcall();
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_MEMCPY_INLINE)).lower();
+  getActionDefinitionsBuilder({opcode(GenericOpcode::G_DYN_STACKALLOC),
+                               opcode(GenericOpcode::G_STACKSAVE),
+                               opcode(GenericOpcode::G_STACKRESTORE)})
+      .lower();
+
+  getActionDefinitionsBuilder(
+      {opcode(GenericOpcode::G_FADD), opcode(GenericOpcode::G_FSUB), opcode(GenericOpcode::G_FMUL),
+       opcode(GenericOpcode::G_FDIV), opcode(GenericOpcode::G_FMA), opcode(GenericOpcode::G_FSQRT),
+       opcode(GenericOpcode::G_FMAXNUM), opcode(GenericOpcode::G_FMINNUM),
+       opcode(GenericOpcode::G_FMAXIMUMNUM), opcode(GenericOpcode::G_FMINIMUMNUM)})
+      .legalFor(HasF, {S32})
+      .legalFor(HasD, {S64})
+      .libcallFor({S32, S64})
+      .libcallFor({S128});
+  getActionDefinitionsBuilder({opcode(GenericOpcode::G_FNEG), opcode(GenericOpcode::G_FABS)})
+      .legalFor(HasF, {S32})
+      .legalFor(HasD, {S64})
+      .lowerFor({S32, S64, S128});
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_FREM))
+      .libcallFor({S32, S64, S128})
+      .minScalar(0, S32);
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_FCOPYSIGN))
+      .legalFor(HasF, {{S32, S32}})
+      .legalFor(HasD, {{S64, S64}, {S32, S64}, {S64, S32}})
+      .lower();
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_FPTRUNC))
+      .legalFor(HasD, {{S32, S64}})
+      .libcallFor({{S32, S64}, {S32, S128}, {S64, S128}});
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_FPEXT))
+      .legalFor(HasD, {{S64, S32}})
+      .libcallFor({{S64, S32}, {S128, S32}, {S128, S64}});
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_FCMP))
+      .legalFor(HasF, {{S64, S32}})
+      .legalFor(HasD, {{S64, S64}})
+      .clampScalar(0, S64, S64)
+      .libcallFor({{S64, S32}, {S64, S64}, {S64, S128}});
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_IS_FPCLASS)).lower();
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_FCONSTANT))
+      .legalFor(HasF, {S32})
+      .legalFor(HasD, {S64})
+      .customFor({S32, S64})
+      .lowerFor({S64, S128});
+  getActionDefinitionsBuilder({opcode(GenericOpcode::G_FPTOSI), opcode(GenericOpcode::G_FPTOUI)})
+      .legalFor(HasF, {{S64, S32}})
+      .legalFor(HasD, {{S64, S64}})
+      .customFor(HasF, {{S32, S32}})
+      .customFor(HasD, {{S32, S64}})
+      .widenScalarToNextPow2(0)
+      .minScalar(0, S32)
+      .libcallFor({{S32, S32},
+                   {S64, S32},
+                   {S32, S64},
+                   {S64, S64},
+                   {S32, S128},
+                   {S64, S128},
+                   {S128, S32},
+                   {S128, S64},
+                   {S128, S128}});
+  getActionDefinitionsBuilder({opcode(GenericOpcode::G_SITOFP), opcode(GenericOpcode::G_UITOFP)})
+      .legalFor(HasF, {{S32, S64}})
+      .legalFor(HasD, {{S64, S64}})
+      .widenScalarToNextPow2(1)
+      .minScalar(1, S32)
+      .libcallFor({{S32, S32},
+                   {S64, S32},
+                   {S32, S64},
+                   {S64, S64},
+                   {S128, S32},
+                   {S128, S64},
+                   {S32, S128},
+                   {S64, S128},
+                   {S128, S128}});
+  getActionDefinitionsBuilder({opcode(GenericOpcode::G_FCEIL), opcode(GenericOpcode::G_FFLOOR),
+                               opcode(GenericOpcode::G_FRINT), opcode(GenericOpcode::G_FNEARBYINT),
+                               opcode(GenericOpcode::G_INTRINSIC_TRUNC),
+                               opcode(GenericOpcode::G_INTRINSIC_ROUND),
+                               opcode(GenericOpcode::G_INTRINSIC_ROUNDEVEN)})
+      .libcallFor({S32, S64, S128});
+  getActionDefinitionsBuilder({opcode(GenericOpcode::G_FCOS), opcode(GenericOpcode::G_FSIN),
+                               opcode(GenericOpcode::G_FTAN), opcode(GenericOpcode::G_FPOW),
+                               opcode(GenericOpcode::G_FLOG), opcode(GenericOpcode::G_FLOG2),
+                               opcode(GenericOpcode::G_FLOG10), opcode(GenericOpcode::G_FEXP),
+                               opcode(GenericOpcode::G_FEXP2), opcode(GenericOpcode::G_FEXP10),
+                               opcode(GenericOpcode::G_FACOS), opcode(GenericOpcode::G_FASIN),
+                               opcode(GenericOpcode::G_FATAN), opcode(GenericOpcode::G_FATAN2),
+                               opcode(GenericOpcode::G_FCOSH), opcode(GenericOpcode::G_FSINH),
+                               opcode(GenericOpcode::G_FTANH), opcode(GenericOpcode::G_FMODF)})
+      .libcallFor({S32, S64, S128});
+  getActionDefinitionsBuilder({opcode(GenericOpcode::G_FPOWI), opcode(GenericOpcode::G_FLDEXP)})
+      .libcallFor({{S32, S32}, {S64, S32}, {S128, S32}});
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_FCANONICALIZE))
+      .legalFor(HasF, {S32})
+      .legalFor(HasD, {S64});
+
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_VASTART)).customFor({P0});
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_VAARG)).clampScalar(0, S64, S64).lower();
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS))
+      .customIf([](const LegalizationQuery &Query) {
+        return Query.Types.size() >= 3 && Query.Types[0].isCategoryNeutralScalar() &&
+               Query.Types[2].isPointer();
+      });
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_ATOMIC_CMPXCHG))
+      .legalFor(HasA, {{S64, P0}})
+      .customIf([](const LegalizationQuery &Query) {
+        return Query.Types.size() >= 2 && Query.Types[0].isCategoryNeutralScalar() &&
+               Query.Types[1].isPointer();
+      })
+      .libcallFor(!HasA, {{S8, P0}, {S16, P0}, {S32, P0}, {S64, P0}})
+      .clampScalar(0, S64, S64);
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_ATOMICRMW_ADD))
+      .legalFor(HasA, {{S64, P0}})
+      .libcallFor(!HasA, {{S8, P0}, {S16, P0}, {S32, P0}, {S64, P0}})
+      .clampScalar(0, S64, S64);
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_ATOMICRMW_SUB))
+      .libcallFor(!HasA, {{S8, P0}, {S16, P0}, {S32, P0}, {S64, P0}})
+      .clampScalar(0, S64, S64)
+      .lower();
+  getActionDefinitionsBuilder(
+      {opcode(GenericOpcode::G_ATOMICRMW_XCHG), opcode(GenericOpcode::G_ATOMICRMW_AND),
+       opcode(GenericOpcode::G_ATOMICRMW_OR), opcode(GenericOpcode::G_ATOMICRMW_XOR),
+       opcode(GenericOpcode::G_ATOMICRMW_MAX), opcode(GenericOpcode::G_ATOMICRMW_MIN),
+       opcode(GenericOpcode::G_ATOMICRMW_UMAX), opcode(GenericOpcode::G_ATOMICRMW_UMIN)})
+      .legalFor(HasA, {{S64, P0}})
+      .libcallFor(!HasA, {{S8, P0}, {S16, P0}, {S32, P0}, {S64, P0}})
+      .clampScalar(0, S64, S64);
+  getActionDefinitionsBuilder(
+      {opcode(GenericOpcode::G_INTRINSIC), opcode(GenericOpcode::G_INTRINSIC_W_SIDE_EFFECTS)})
+      .alwaysLegal();
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_FENCE)).alwaysLegal();
+  getActionDefinitionsBuilder({opcode(GenericOpcode::G_TRAP), opcode(GenericOpcode::G_DEBUGTRAP),
+                               opcode(GenericOpcode::G_UBSANTRAP)})
+      .alwaysLegal();
+}
+
+Expected<void> RISCVLegalizerInfo::legalizeCustom(LegalizerHelper &Helper,
+                                                  MachineInstr &Instruction,
+                                                  const LegalizationDecision &Decision) const {
+  if (Decision.Action != LegalizationAction::Custom)
+    return makeUnexpected(ErrorCode::InvalidArgument,
+                          "RISC-V custom legalizer received a non-custom action");
+
+  MachineFunction &Function = Helper.MIRBuilder.function();
+  const TargetDescription &Target = Helper.MIRBuilder.target();
+  if (Instruction.opcode() == opcode(GenericOpcode::G_ICMP))
     return lowerIntegerCompare(Instruction, Function, Target);
-  if (Action == LegalizationAction::Custom &&
-      Instruction.opcode() == opcode(GenericOpcode::G_ATOMIC_CMPXCHG))
+  if (Instruction.opcode() == opcode(GenericOpcode::G_ATOMIC_CMPXCHG))
     return lowerAtomicCompareExchange(Instruction, Function, Target);
-  if (Action == LegalizationAction::Custom &&
-      Instruction.opcode() == opcode(GenericOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS))
+  if (Instruction.opcode() == opcode(GenericOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS))
     return lowerAtomicCompareExchangeWithSuccess(Instruction, Function, Target);
-  if (Action == LegalizationAction::Custom &&
-      (Instruction.opcode() == opcode(GenericOpcode::G_ANYEXT) ||
-       Instruction.opcode() == opcode(GenericOpcode::G_SEXT) ||
-       Instruction.opcode() == opcode(GenericOpcode::G_ZEXT))) {
-    if (Instruction.opcode() == opcode(GenericOpcode::G_ANYEXT) &&
-        narrowAnyExtBinaryPlan(Instruction, Function).has_value())
-      return foldNarrowAnyExtBinary(Instruction, Function, Target);
+  if (Instruction.opcode() == opcode(GenericOpcode::G_ANYEXT) ||
+      Instruction.opcode() == opcode(GenericOpcode::G_SEXT) ||
+      Instruction.opcode() == opcode(GenericOpcode::G_ZEXT)) {
     return foldConstantExtension(Instruction, Function,
                                  static_cast<GenericOpcode>(Instruction.opcode().value()));
   }
-  if (Action == LegalizationAction::Custom &&
-      Instruction.opcode() == opcode(GenericOpcode::G_CONSTANT)) {
+  if (Instruction.opcode() == opcode(GenericOpcode::G_CONSTANT)) {
     const MachineOperand *Immediate = Instruction.operandAt(1);
     const std::optional<LowLevelType> Type = resultType(Function, Instruction);
     if (Immediate != nullptr && Type && Type->isCategoryNeutralScalar() &&
@@ -3789,8 +1113,10 @@ Expected<void> RISCVLegalizerInfo::lower(MachineInstr &Instruction, MachineFunct
       auto Value = signedImmediate(*Immediate);
       if (Value && shouldUseRV64IntegerConstantPool(*Value))
         return lowerConstantToPool(Instruction, Function, Target);
+      return {};
     }
-    return foldWideConstant(Instruction, Function, Target);
+    return makeUnexpected(ErrorCode::Unsupported,
+                          "RISC-V constant custom legalization is unavailable");
   }
   return makeUnexpected(ErrorCode::Unsupported,
                         "RISC-V legalizer has no lowering rule for this instruction");
diff --git a/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.h b/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.h
index f6ba000..d8e5e3e 100644
--- a/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.h
+++ b/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.h
@@ -11,17 +11,11 @@ class RISCVSubtarget;
 
 class RISCVLegalizerInfo final : public LegalizerInfo {
 public:
-  explicit RISCVLegalizerInfo(const RISCVSubtarget &Subtarget) : Subtarget(Subtarget) {}
-
-  [[nodiscard]] Expected<LegalizationAction>
-  classify(const MachineInstr &Instruction, const MachineFunction &Function,
-           const TargetDescription &Target) const override;
-  [[nodiscard]] Expected<void> lower(MachineInstr &Instruction, MachineFunction &Function,
-                                     const TargetDescription &Target,
-                                     LegalizationAction Action) const override;
+  explicit RISCVLegalizerInfo(const RISCVSubtarget &Subtarget);
+  [[nodiscard]] Expected<void> legalizeCustom(LegalizerHelper &Helper, MachineInstr &Instruction,
+                                              const LegalizationDecision &Decision) const override;
 
 private:
-  const RISCVSubtarget &Subtarget;
 };
 
 } // namespace rcc::riscv