[CodeGen] Match LLVM Legalizer ownership

3 files changed, 410 insertions, 3090 deletions

diff --git a/backend/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp b/backend/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp
index e4c937f..59c992e 100644
--- a/backend/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp
+++ b/backend/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp
@@ -177,10 +177,10 @@ public:
     const std::optional<LowLevelType> Type = resultType(Function, Instruction);
     const auto Integer = [Type] { return Type && isSupportedIntegerType(*Type); };
     const auto Float = [Type] { return Type && isSupportedFloatingScalar(*Type); };
-    auto Classification = Subtarget.getLegalizerInfo()->classify(Instruction, Function, Target);
+    auto Classification = Subtarget.getLegalizerInfo()->getDecision(Instruction, Function, Target);
     if (!Classification)
       return std::unexpected(Classification.error());
-    if (*Classification != LegalizationAction::Legal) {
+    if (Classification->Action != LegalizationAction::Legal) {
       std::string Message = "RISC-V selector cannot select opcode '" +
                             std::string(getGenericOpcodeName(Generic)) + "'";
       if (Type)
diff --git a/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp b/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
index 1a29ebf..4d81404 100644
--- a/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
+++ b/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
@@ -2,9 +2,9 @@
 
 #include "RISCVLegalizerInfo.h"
 
-#include "RISCVCallLowering.h"
 #include "RISCVImmediateMaterialization.h"
 #include "rcc/CodeGen/GenericOpcodes.h"
+#include "rcc/CodeGen/GlobalISel/LegalizerHelper.h"
 #include "rcc/CodeGen/LowLevelType.h"
 #include "rcc/CodeGen/MachineBasicBlock.h"
 #include "rcc/CodeGen/MachineFunction.h"
@@ -59,45 +59,6 @@ operandType(const MachineFunction &Function, const MachineInstr &Instruction, st
   return Operand == nullptr ? std::nullopt : registerType(Function, *Operand);
 }
 
-[[nodiscard]] bool isSupportedFloatingScalar(LowLevelType Type) noexcept {
-  return Type.isCategoryNeutralScalar() &&
-         (Type.scalarBitWidth() == 32 || Type.scalarBitWidth() == 64);
-}
-
-[[nodiscard]] bool isSupportedIntegerType(LowLevelType Type) noexcept {
-  return (Type.isCategoryNeutralScalar() && Type.scalarBitWidth() != 0 &&
-          Type.scalarBitWidth() <= 64) ||
-         (Type.isPointer() && Type.scalarBitWidth() == 64);
-}
-
-[[nodiscard]] bool isSupportedIntegerScalar(LowLevelType Type) noexcept {
-  return Type.isCategoryNeutralScalar() && Type.scalarBitWidth() != 0 &&
-         Type.scalarBitWidth() <= 64;
-}
-
-[[nodiscard]] bool isNativeIntegerScalar(LowLevelType Type) noexcept {
-  return Type.isCategoryNeutralScalar() &&
-         (Type.scalarBitWidth() == 32 || Type.scalarBitWidth() == 64);
-}
-
-[[nodiscard]] bool isScalarGPRType(LowLevelType Type) noexcept {
-  return (Type.isCategoryNeutralScalar() && Type.scalarBitWidth() != 0 &&
-          Type.scalarBitWidth() <= 64) ||
-         (Type.isPointer() && Type.scalarBitWidth() == 64);
-}
-
-[[nodiscard]] bool isI1Type(LowLevelType Type) noexcept {
-  return Type.isCategoryNeutralScalar() && Type.scalarBitWidth() == 1;
-}
-
-[[nodiscard]] bool isScalarIntegerExtension(LowLevelType Destination,
-                                            LowLevelType Source) noexcept {
-  return Destination.isCategoryNeutralScalar() && Source.isCategoryNeutralScalar() &&
-         Destination.scalarBitWidth() != 0 && Source.scalarBitWidth() != 0 &&
-         Source.scalarBitWidth() < Destination.scalarBitWidth() &&
-         Destination.scalarBitWidth() <= 64;
-}
-
 [[nodiscard]] Expected<APInt>
 extendIntegerImmediate(const APInt &Value, std::uint32_t DestinationWidth, bool SignExtend) {
   if (DestinationWidth == 0 || DestinationWidth > 64)
@@ -193,13 +154,6 @@ foldConstantExtension(MachineInstr &Instruction, MachineFunction &Function, Gene
   return {};
 }
 
-struct WideConstantUnmergePlan final {
-  MachineInstr *Extension = nullptr;
-  MachineInstr *Unmerge = nullptr;
-  std::vector<Register> Outputs;
-  bool SignExtend = false;
-};
-
 [[nodiscard]] MachineInstr *singleRegisterUse(const MachineFunction &Function, Register Reg) {
   MachineInstr *UseInstruction = nullptr;
   auto &RegisterInfo = const_cast<MachineRegisterInfo &>(Function.registerInfo());
@@ -213,189 +167,6 @@ struct WideConstantUnmergePlan final {
   return UseInstruction;
 }
 
-[[nodiscard]] std::optional<WideConstantUnmergePlan>
-wideConstantUnmergePlan(const MachineInstr &Constant, const MachineFunction &Function) {
-  const MachineOperand *ResultOperand = Constant.operandAt(0);
-  const Register *Result = ResultOperand == nullptr ? nullptr : ResultOperand->getIf<Register>();
-  if (Result == nullptr || !Result->isVirtual())
-    return std::nullopt;
-
-  MachineInstr *FirstUse = singleRegisterUse(Function, *Result);
-  if (FirstUse == nullptr)
-    return std::nullopt;
-  MachineInstr *Extension = nullptr;
-  MachineInstr *Unmerge = FirstUse;
-  if (FirstUse->opcode() == opcode(GenericOpcode::G_ANYEXT) ||
-      FirstUse->opcode() == opcode(GenericOpcode::G_SEXT) ||
-      FirstUse->opcode() == opcode(GenericOpcode::G_ZEXT)) {
-    Extension = FirstUse;
-    const MachineOperand *ExtendedOperand = Extension->operandAt(0);
-    const Register *Extended =
-        ExtendedOperand == nullptr ? nullptr : ExtendedOperand->getIf<Register>();
-    if (Extended == nullptr || !Extended->isVirtual())
-      return std::nullopt;
-    Unmerge = singleRegisterUse(Function, *Extended);
-    if (Unmerge == nullptr)
-      return std::nullopt;
-  }
-  if (Unmerge->opcode() != opcode(GenericOpcode::G_UNMERGE_VALUES) || Unmerge->operandCount() < 3)
-    return std::nullopt;
-
-  WideConstantUnmergePlan Plan;
-  Plan.Extension = Extension;
-  Plan.Unmerge = Unmerge;
-  Plan.SignExtend = Extension != nullptr && Extension->opcode() != opcode(GenericOpcode::G_ZEXT);
-  for (std::size_t Index = 0; Index + 1U < Unmerge->operandCount(); ++Index) {
-    const MachineOperand *Operand = Unmerge->operandAt(Index);
-    const Register *Reg = Operand == nullptr ? nullptr : Operand->getIf<Register>();
-    if (Reg == nullptr || !Reg->isVirtual() || !Operand->isDef())
-      return std::nullopt;
-    const auto Type = registerType(Function, *Reg);
-    if (!Type || !isSupportedIntegerScalar(*Type))
-      return std::nullopt;
-    Plan.Outputs.push_back(*Reg);
-  }
-  const MachineOperand *SourceOperand = Unmerge->operandAt(Unmerge->operandCount() - 1U);
-  const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>();
-  const Register *ExpectedSource = Result;
-  if (Extension != nullptr) {
-    const MachineOperand *ExtendedOperand = Extension->operandAt(0);
-    ExpectedSource = ExtendedOperand == nullptr ? nullptr : ExtendedOperand->getIf<Register>();
-  }
-  if (Source == nullptr || ExpectedSource == nullptr || *Source != *ExpectedSource)
-    return std::nullopt;
-  return Plan;
-}
-
-[[nodiscard]] Expected<APInt> wideConstantWord(const APInt &Value, std::size_t WordIndex,
-                                               std::uint32_t Width, bool SignExtend) {
-  std::uint64_t Raw = 0;
-  if (WordIndex < Value.words().size())
-    Raw = Value.words()[WordIndex];
-  else if (SignExtend && Value.isNegative())
-    Raw = std::numeric_limits<std::uint64_t>::max();
-  const std::size_t WordBitOffset = WordIndex * 64U;
-  if (SignExtend && Value.isNegative() && WordBitOffset < Value.bitWidth()) {
-    const std::uint32_t Remaining = Value.bitWidth() - static_cast<std::uint32_t>(WordBitOffset);
-    if (Remaining < 64)
-      Raw |= ~std::uint64_t{0} << Remaining;
-  }
-  if (Width < 64)
-    Raw &= (std::uint64_t{1} << Width) - 1U;
-  return APInt::create(Width, std::span<const std::uint64_t>(&Raw, 1));
-}
-
-[[nodiscard]] Expected<void> foldWideConstant(MachineInstr &Instruction, MachineFunction &Function,
-                                              const TargetDescription &Target) {
-  const auto Plan = wideConstantUnmergePlan(Instruction, Function);
-  if (!Plan)
-    return makeUnexpected(ErrorCode::Unsupported,
-                          "RISC-V wide constant requires a G_UNMERGE_VALUES use");
-  const MachineOperand *ImmediateOperand = Instruction.operandAt(1);
-  const auto *Integer =
-      ImmediateOperand == nullptr ? nullptr : ImmediateOperand->getIf<IntegerImmediateOperand>();
-  if (Integer == nullptr)
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "RISC-V wide constant has no integer immediate");
-
-  std::vector<APInt> Words;
-  Words.reserve(Plan->Outputs.size());
-  for (std::size_t Index = 0; Index < Plan->Outputs.size(); ++Index) {
-    const auto Type = registerType(Function, Plan->Outputs[Index]);
-    if (!Type)
-      return makeUnexpected(ErrorCode::VerificationError,
-                            "RISC-V wide constant output has no virtual-register type");
-    auto Word = wideConstantWord(Integer->Value, Index, Type->scalarBitWidth(), Plan->SignExtend);
-    if (!Word)
-      return std::unexpected(Word.error());
-    Words.push_back(std::move(*Word));
-  }
-
-  std::vector<std::size_t> Representatives(Plan->Outputs.size());
-  for (std::size_t Index = 0; Index < Representatives.size(); ++Index) {
-    Representatives[Index] = Index;
-    for (std::size_t Previous = 0; Previous < Index; ++Previous) {
-      if (Words[Previous] == Words[Index]) {
-        Representatives[Index] = Previous;
-        break;
-      }
-    }
-  }
-
-  const Register FirstOutput = Plan->Outputs.front();
-  auto Retargeted = Instruction.operandAt(0)->setRegister(FirstOutput);
-  if (!Retargeted)
-    return std::unexpected(Retargeted.error());
-  auto SetOpcode = Instruction.setOpcode(opcode(GenericOpcode::G_CONSTANT));
-  if (!SetOpcode)
-    return std::unexpected(SetOpcode.error());
-  while (Instruction.operandCount() > 1) {
-    auto Erased = Instruction.eraseOperand(1);
-    if (!Erased)
-      return std::unexpected(Erased.error());
-  }
-  auto Added = Instruction.addOperand(MachineOperand::integerImmediate(std::move(Words.front())));
-  if (!Added)
-    return std::unexpected(Added.error());
-
-  MachineBasicBlock *Block = Instruction.parent();
-  if (Block == nullptr)
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "RISC-V wide constant is detached during legalization");
-  if (Plan->Unmerge != &Instruction) {
-    auto RemovedUnmerge = Plan->Unmerge->parent()->removeInstruction(*Plan->Unmerge);
-    if (!RemovedUnmerge)
-      return std::unexpected(RemovedUnmerge.error());
-  }
-  if (Plan->Extension != nullptr) {
-    auto RemovedExtension = Plan->Extension->parent()->removeInstruction(*Plan->Extension);
-    if (!RemovedExtension)
-      return std::unexpected(RemovedExtension.error());
-  }
-
-  for (std::size_t Index = 1; Index < Plan->Outputs.size(); ++Index) {
-    if (Representatives[Index] == Index)
-      continue;
-    auto Replaced = Function.registerInfo().replaceRegWith(Plan->Outputs[Index],
-                                                           Plan->Outputs[Representatives[Index]]);
-    if (!Replaced)
-      return std::unexpected(Replaced.error());
-  }
-
-  MachineInstr *Last = &Instruction;
-  for (std::size_t Index = 1; Index < Plan->Outputs.size(); ++Index) {
-    if (Representatives[Index] != Index)
-      continue;
-    MachineIRBuilder Builder(Function, Target);
-    auto Point = Builder.setInsertPointAfter(*Last);
-    if (!Point)
-      return std::unexpected(Point.error());
-    auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_CONSTANT));
-    if (!Pending)
-      return std::unexpected(Pending.error());
-    auto Def = Pending->addDef(Plan->Outputs[Index]);
-    if (!Def)
-      return std::unexpected(Def.error());
-    auto Value = Pending->addIntegerImmediate(std::move(Words[Index]));
-    if (!Value)
-      return std::unexpected(Value.error());
-    auto Built = Pending->build();
-    if (!Built)
-      return std::unexpected(Built.error());
-    Last = *Built;
-  }
-  return {};
-}
-
-[[nodiscard]] Expected<Register> buildIntegerConstantBefore(MachineIRBuilder &Builder,
-                                                            MachineInstr &Anchor,
-                                                            std::uint32_t Width,
-                                                            std::uint64_t Value);
-
-[[nodiscard]] Expected<Register> buildBinaryBefore(MachineIRBuilder &Builder, MachineInstr &Anchor,
-                                                   GenericOpcode Generic, LowLevelType Type,
-                                                   Register Left, Register Right);
-
 [[nodiscard]] Expected<Register> buildUnaryBefore(MachineIRBuilder &Builder, MachineInstr &Anchor,
                                                   GenericOpcode Generic, LowLevelType Type,
                                                   Register Source);
@@ -405,123 +176,6 @@ wideConstantUnmergePlan(const MachineInstr &Constant, const MachineFunction &Fun
 [[nodiscard]] Expected<void> rewriteUnary(MachineInstr &Instruction, GenericOpcode Generic,
                                           Register Destination, Register Source);
 
-struct NarrowAnyExtBinaryPlan final {
-  MachineInstr *Binary = nullptr;
-  GenericOpcode Generic = GenericOpcode::Invalid;
-  std::array<Register, 2> WideInputs{};
-  std::vector<MachineInstr *> Artifacts;
-  std::vector<MachineInstr *> Constants;
-  std::uint32_t SourceWidth = 0;
-};
-
-[[nodiscard]] std::optional<NarrowAnyExtBinaryPlan>
-narrowAnyExtBinaryPlan(const MachineInstr &Extension, const MachineFunction &Function) {
-  const MachineOperand *DestinationOperand = Extension.operandAt(0);
-  const MachineOperand *SourceOperand = Extension.operandAt(1);
-  const Register *Destination =
-      DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
-  const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>();
-  if (Destination == nullptr || Source == nullptr || !Destination->isVirtual() ||
-      !Source->isVirtual() || singleRegisterUse(Function, *Source) != &Extension)
-    return std::nullopt;
-  const auto DestinationType = registerType(Function, *Destination);
-  const auto SourceType = registerType(Function, *Source);
-  if (!DestinationType || !SourceType)
-    return std::nullopt;
-  if (!isScalarIntegerExtension(*DestinationType, *SourceType) ||
-      DestinationType->scalarBitWidth() != 64)
-    return std::nullopt;
-
-  const MachineOperand *SourceDefinition = Function.registerInfo().getUniqueVRegDef(*Source);
-  MachineInstr *Binary = SourceDefinition == nullptr
-                             ? nullptr
-                             : const_cast<MachineInstr *>(SourceDefinition->parent());
-  if (Binary == nullptr || Binary->operandCount() != 3)
-    return std::nullopt;
-  const GenericOpcode Generic = static_cast<GenericOpcode>(Binary->opcode().value());
-  if (Generic != GenericOpcode::G_ADD && Generic != GenericOpcode::G_SUB &&
-      Generic != GenericOpcode::G_MUL && Generic != GenericOpcode::G_AND &&
-      Generic != GenericOpcode::G_OR && Generic != GenericOpcode::G_XOR &&
-      Generic != GenericOpcode::G_SHL)
-    return std::nullopt;
-
-  NarrowAnyExtBinaryPlan Plan;
-  Plan.Binary = Binary;
-  Plan.Generic = Generic;
-  Plan.SourceWidth = SourceType->scalarBitWidth();
-  const auto AddArtifact = [&Plan](MachineInstr *Artifact) {
-    if (std::ranges::find(Plan.Artifacts, Artifact) == Plan.Artifacts.end())
-      Plan.Artifacts.push_back(Artifact);
-  };
-  const auto TruncatedWideSource = [&](MachineInstr *Truncation,
-                                       MachineInstr *OnlyUse) -> std::optional<Register> {
-    if (Truncation == nullptr || Truncation->opcode() != opcode(GenericOpcode::G_TRUNC) ||
-        Truncation->operandCount() != 2)
-      return std::nullopt;
-    const MachineOperand *NarrowOperand = Truncation->operandAt(0);
-    const Register *Narrow = NarrowOperand == nullptr ? nullptr : NarrowOperand->getIf<Register>();
-    const MachineOperand *WideOperand = Truncation->operandAt(1);
-    const Register *Wide = WideOperand == nullptr ? nullptr : WideOperand->getIf<Register>();
-    const std::optional<LowLevelType> WideType =
-        WideOperand == nullptr ? std::nullopt : registerType(Function, *WideOperand);
-    if (Narrow == nullptr || Wide == nullptr || !Wide->isVirtual() || !WideType ||
-        *WideType != *DestinationType || singleRegisterUse(Function, *Narrow) != OnlyUse)
-      return std::nullopt;
-    return *Wide;
-  };
-  for (std::size_t Index = 0; Index != Plan.WideInputs.size(); ++Index) {
-    const MachineOperand *Operand = Binary->operandAt(Index + 1U);
-    const Register *Reg = Operand == nullptr ? nullptr : Operand->getIf<Register>();
-    if (Reg == nullptr || !Reg->isVirtual())
-      return std::nullopt;
-    const auto Type = registerType(Function, *Reg);
-    if (!Type)
-      return std::nullopt;
-    if (*Type == *DestinationType) {
-      Plan.WideInputs[Index] = *Reg;
-      continue;
-    }
-    if (*Type != *SourceType)
-      return std::nullopt;
-
-    const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(*Reg);
-    MachineInstr *Def =
-        Definition == nullptr ? nullptr : const_cast<MachineInstr *>(Definition->parent());
-    if (auto Wide = TruncatedWideSource(Def, Binary)) {
-      Plan.WideInputs[Index] = *Wide;
-      AddArtifact(Def);
-      continue;
-    }
-    if (Def != nullptr &&
-        (Def->opcode() == opcode(GenericOpcode::COPY) ||
-         Def->opcode() == opcode(GenericOpcode::G_ANYEXT)) &&
-        Def->operandCount() == 2 && singleRegisterUse(Function, *Reg) == Binary) {
-      const MachineOperand *NarrowOperand = Def->operandAt(1);
-      const Register *Narrow =
-          NarrowOperand == nullptr ? nullptr : NarrowOperand->getIf<Register>();
-      const MachineOperand *NarrowDefinition =
-          Narrow == nullptr ? nullptr : Function.registerInfo().getUniqueVRegDef(*Narrow);
-      MachineInstr *Truncation = NarrowDefinition == nullptr
-                                     ? nullptr
-                                     : const_cast<MachineInstr *>(NarrowDefinition->parent());
-      if (auto Wide = TruncatedWideSource(Truncation, Def)) {
-        Plan.WideInputs[Index] = *Wide;
-        AddArtifact(Def);
-        AddArtifact(Truncation);
-        continue;
-      }
-    }
-    if (Def != nullptr && Def->opcode() == opcode(GenericOpcode::G_CONSTANT) &&
-        singleRegisterUse(Function, *Reg) == Binary) {
-      Plan.WideInputs[Index] = *Reg;
-      Plan.Constants.push_back(Def);
-      continue;
-    }
-    return std::nullopt;
-  }
-  return Plan;
-}
-
 [[nodiscard]] Expected<void> widenConstantToRV64(MachineInstr &Instruction,
                                                  MachineFunction &Function) {
   const MachineOperand *DestinationOperand = Instruction.operandAt(0);
@@ -893,127 +547,6 @@ lowerAtomicCompareExchangeWithSuccess(MachineInstr &Instruction, MachineFunction
   return {};
 }
 
-[[nodiscard]] Expected<void> foldNarrowAnyExtBinary(MachineInstr &Extension,
-                                                    MachineFunction &Function,
-                                                    const TargetDescription &Target) {
-  const auto Plan = narrowAnyExtBinaryPlan(Extension, Function);
-  if (!Plan)
-    return makeUnexpected(ErrorCode::Unsupported,
-                          "RISC-V narrow any-extension does not have a legal binary chain");
-  for (MachineInstr *Constant : Plan->Constants) {
-    auto Widened = widenConstantToRV64(*Constant, Function);
-    if (!Widened)
-      return std::unexpected(Widened.error());
-  }
-  for (std::size_t Index = 0; Index != Plan->WideInputs.size(); ++Index) {
-    auto Replaced = Plan->Binary->operandAt(Index + 1U)->setRegister(Plan->WideInputs[Index]);
-    if (!Replaced)
-      return std::unexpected(Replaced.error());
-  }
-  for (MachineInstr *Artifact : Plan->Artifacts) {
-    if (Artifact->parent() == nullptr)
-      continue;
-    auto Removed = Artifact->parent()->removeInstruction(*Artifact);
-    if (!Removed)
-      return std::unexpected(Removed.error());
-  }
-
-  if (Plan->Generic == GenericOpcode::G_SHL) {
-    Register Amount = Plan->WideInputs[1];
-    if (Plan->SourceWidth < 32 || Plan->SourceWidth > 32) {
-      const std::uint64_t Mask = Plan->SourceWidth == 64
-                                     ? std::numeric_limits<std::uint64_t>::max()
-                                     : (std::uint64_t{1} << Plan->SourceWidth) - 1U;
-      auto WideType = LowLevelType::makeCategoryNeutralScalar(64);
-      if (!WideType)
-        return std::unexpected(WideType.error());
-      MachineIRBuilder Builder(Function, Target);
-      auto MaskValue = buildIntegerConstantBefore(Builder, Extension, 64, Mask);
-      if (!MaskValue)
-        return std::unexpected(MaskValue.error());
-      auto MaskedAmount = buildBinaryBefore(Builder, Extension, GenericOpcode::G_AND, *WideType,
-                                            Amount, *MaskValue);
-      if (!MaskedAmount)
-        return std::unexpected(MaskedAmount.error());
-      Amount = *MaskedAmount;
-    }
-    auto SetOpcode =
-        Extension.setOpcode(Plan->SourceWidth == 32 ? instructionOpcode(InstructionOpcode::G_SLLW)
-                                                    : opcode(GenericOpcode::G_SHL));
-    if (!SetOpcode)
-      return std::unexpected(SetOpcode.error());
-    while (Extension.operandCount() > 1) {
-      auto Erased = Extension.eraseOperand(1);
-      if (!Erased)
-        return std::unexpected(Erased.error());
-    }
-    for (Register Input : {Plan->WideInputs[0], Amount}) {
-      auto Operand =
-          MachineOperand::registerOperand(Input, RegisterOperandFlags{RegisterOperandFlag::Use});
-      if (!Operand)
-        return std::unexpected(Operand.error());
-      auto Added = Extension.addOperand(std::move(*Operand));
-      if (!Added)
-        return std::unexpected(Added.error());
-    }
-    MachineBasicBlock *BinaryBlock = Plan->Binary->parent();
-    if (BinaryBlock == nullptr)
-      return makeUnexpected(ErrorCode::VerificationError,
-                            "RISC-V narrow shift is detached during legalization");
-    return BinaryBlock->removeInstruction(*Plan->Binary);
-  }
-
-  if (Plan->SourceWidth == 32 &&
-      (Plan->Generic == GenericOpcode::G_ADD || Plan->Generic == GenericOpcode::G_SUB)) {
-    auto WideType = LowLevelType::makeCategoryNeutralScalar(64);
-    if (!WideType)
-      return std::unexpected(WideType.error());
-    auto Temporary = Function.registerInfo().createVirtualRegister(*WideType);
-    if (!Temporary)
-      return std::unexpected(Temporary.error());
-    auto Retargeted = Plan->Binary->operandAt(0)->setRegister(*Temporary);
-    if (!Retargeted)
-      return std::unexpected(Retargeted.error());
-    auto SetOpcode = Extension.setOpcode(opcode(GenericOpcode::G_SEXT_INREG));
-    if (!SetOpcode)
-      return std::unexpected(SetOpcode.error());
-    auto ReplacedSource = Extension.operandAt(1)->setRegister(*Temporary);
-    if (!ReplacedSource)
-      return std::unexpected(ReplacedSource.error());
-    auto Width = Extension.addOperand(MachineOperand::immediate(32));
-    if (!Width)
-      return std::unexpected(Width.error());
-    return {};
-  }
-
-  auto SetOpcode = Extension.setOpcode(opcode(Plan->Generic));
-  if (!SetOpcode)
-    return std::unexpected(SetOpcode.error());
-  while (Extension.operandCount() > 1) {
-    auto Erased = Extension.eraseOperand(1);
-    if (!Erased)
-      return std::unexpected(Erased.error());
-  }
-  for (Register Input : Plan->WideInputs) {
-    auto Operand =
-        MachineOperand::registerOperand(Input, RegisterOperandFlags{RegisterOperandFlag::Use});
-    if (!Operand)
-      return std::unexpected(Operand.error());
-    auto Added = Extension.addOperand(std::move(*Operand));
-    if (!Added)
-      return std::unexpected(Added.error());
-  }
-  MachineBasicBlock *BinaryBlock = Plan->Binary->parent();
-  if (BinaryBlock == nullptr)
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "RISC-V narrow binary instruction is detached during legalization");
-  auto Removed = BinaryBlock->removeInstruction(*Plan->Binary);
-  if (!Removed)
-    return std::unexpected(Removed.error());
-  (void)Target;
-  return {};
-}
-
 [[nodiscard]] Expected<void> lowerConstantToPool(MachineInstr &Instruction,
                                                  MachineFunction &Function,
                                                  const TargetDescription &Target) {
@@ -1097,63 +630,6 @@ lowerAtomicCompareExchangeWithSuccess(MachineInstr &Instruction, MachineFunction
   return {};
 }
 
-[[nodiscard]] bool isSupportedNonAtomicMemoryAccess(const MachineInstr &Instruction) noexcept {
-  if (Instruction.memOperands().size() != 1)
-    return false;
-  const MachineMemOperand &Memory = *Instruction.memOperands().front();
-  const MachineMemOperandDesc &Description = Memory.description();
-  return !Memory.isAtomic() && Description.AddressSpace == 0 && Description.Size.isKnown() &&
-         !Description.Size.isScalable() && Description.Size.minimumBytes() != 0;
-}
-
-[[nodiscard]] bool isSupportedAtomicMemoryAccess(const MachineInstr &Instruction,
-                                                 AtomicOperation Operation,
-                                                 std::uint64_t Bytes) noexcept {
-  if (Instruction.memOperands().size() != 1 || (Bytes != 4 && Bytes != 8))
-    return false;
-  const MachineMemOperand &Memory = *Instruction.memOperands().front();
-  const MachineMemOperandDesc &Description = Memory.description();
-  return Description.AtomicKind == Operation && Description.AddressSpace == 0 &&
-         Description.Size.isKnown() && !Description.Size.isScalable() &&
-         Description.Size.minimumBytes() == Bytes && Description.Align.bytes() >= Bytes &&
-         Description.SyncScope.value() == 0;
-}
-
-[[nodiscard]] bool isSupportedAtomicContainerAccess(const MachineInstr &Instruction,
-                                                    AtomicOperation Operation,
-                                                    std::uint64_t ContainerBytes) noexcept {
-  if (Instruction.memOperands().size() != 1 || ContainerBytes == 0)
-    return false;
-  const MachineMemOperandDesc &Description = Instruction.memOperands().front()->description();
-  if (Description.AtomicKind != Operation || Description.AddressSpace != 0 ||
-      !Description.Size.isKnown() || Description.Size.isScalable() ||
-      Description.SyncScope.value() != 0)
-    return false;
-  const std::uint64_t MemoryBytes = Description.Size.minimumBytes();
-  return (MemoryBytes == 1 || MemoryBytes == 2 || MemoryBytes == 4 || MemoryBytes == 8) &&
-         MemoryBytes <= ContainerBytes && Description.Align.bytes() >= MemoryBytes;
-}
-
-[[nodiscard]] std::optional<std::uint64_t>
-memoryAccessBytes(const MachineInstr &Instruction) noexcept {
-  if (Instruction.memOperands().size() != 1)
-    return std::nullopt;
-  const MachineMemOperandDesc &Description = Instruction.memOperands().front()->description();
-  if (!Description.Size.isKnown() || Description.Size.isScalable())
-    return std::nullopt;
-  return Description.Size.minimumBytes();
-}
-
-[[nodiscard]] bool memoryMatchesType(const MachineInstr &Instruction, LowLevelType Type) noexcept {
-  const auto Bytes = memoryAccessBytes(Instruction);
-  if (!Bytes || (!isSupportedIntegerType(Type) && !isSupportedFloatingScalar(Type)))
-    return false;
-  if (isSupportedFloatingScalar(Type))
-    return Type.scalarBitWidth() == 32 ? *Bytes == 4 : *Bytes == 8;
-  const std::uint64_t ValueBytes = (Type.scalarBitWidth() + 7U) / 8U;
-  return (*Bytes == 1 || *Bytes == 2 || *Bytes == 4 || *Bytes == 8) && *Bytes <= ValueBytes;
-}
-
 [[nodiscard]] Expected<std::int64_t> signedImmediate(const MachineOperand &Operand) {
   if (const std::int64_t *Value = Operand.getIf<std::int64_t>())
     return *Value;
@@ -1168,19 +644,6 @@ memoryAccessBytes(const MachineInstr &Instruction) noexcept {
   return std::bit_cast<std::int64_t>(Raw);
 }
 
-[[nodiscard]] Expected<void> replaceRegisterUses(MachineFunction &Function, Register OldReg,
-                                                 Register NewReg) {
-  std::vector<MachineOperand *> Uses;
-  for (MachineOperand &Operand : Function.registerInfo().useOperands(OldReg))
-    Uses.push_back(&Operand);
-  for (MachineOperand *Operand : Uses) {
-    auto Replaced = Operand->setRegister(NewReg);
-    if (!Replaced)
-      return std::unexpected(Replaced.error());
-  }
-  return {};
-}
-
 [[nodiscard]] Expected<void> eraseInstruction(MachineInstr &Instruction) {
   MachineBasicBlock *Block = Instruction.parent();
   if (Block == nullptr)
@@ -1189,67 +652,6 @@ memoryAccessBytes(const MachineInstr &Instruction) noexcept {
   return Block->removeInstruction(Instruction);
 }
 
-[[nodiscard]] bool allDefsUnused(const MachineInstr &Instruction, const MachineFunction &Function) {
-  for (const MachineOperand &Operand : Instruction.operands()) {
-    const Register *Reg = Operand.getIf<Register>();
-    if (Reg != nullptr && Operand.isDef() && Reg->isVirtual() &&
-        !Function.registerInfo().useEmpty(*Reg))
-      return false;
-  }
-  return true;
-}
-
-void eraseDeadArtifacts(std::span<MachineInstr *const> Candidates,
-                        const MachineFunction &Function) {
-  bool Changed = false;
-  do {
-    Changed = false;
-    for (auto It = Candidates.rbegin(); It != Candidates.rend(); ++It) {
-      MachineInstr *Instruction = *It;
-      if (Instruction == nullptr || Instruction->parent() == nullptr ||
-          !allDefsUnused(*Instruction, Function))
-        continue;
-      const Opcode Value = Instruction->opcode();
-      if (Value != opcode(GenericOpcode::G_TRUNC) && Value != opcode(GenericOpcode::G_ANYEXT) &&
-          Value != opcode(GenericOpcode::G_SEXT) && Value != opcode(GenericOpcode::G_ZEXT) &&
-          Value != opcode(GenericOpcode::G_MERGE_VALUES) &&
-          Value != opcode(GenericOpcode::G_UNMERGE_VALUES) && Value != opcode(GenericOpcode::COPY))
-        continue;
-      if (eraseInstruction(*Instruction))
-        Changed = true;
-    }
-  } while (Changed);
-}
-
-[[nodiscard]] Expected<Register> buildIntegerConstantBefore(MachineIRBuilder &Builder,
-                                                            MachineInstr &Anchor,
-                                                            std::uint32_t Width,
-                                                            std::uint64_t Value) {
-  auto Type = LowLevelType::makeCategoryNeutralScalar(Width);
-  if (!Type)
-    return std::unexpected(Type.error());
-  auto Result = Builder.function().registerInfo().createVirtualRegister(*Type);
-  if (!Result)
-    return std::unexpected(Result.error());
-  auto Immediate = APInt::fromUnsigned(Width, Value);
-  if (!Immediate)
-    return std::unexpected(Immediate.error());
-  auto Point = Builder.setInsertPointBefore(Anchor);
-  if (!Point)
-    return std::unexpected(Point.error());
-  auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_CONSTANT));
-  if (!Pending)
-    return std::unexpected(Pending.error());
-  if (auto Added = Pending->addDef(*Result); !Added)
-    return std::unexpected(Added.error());
-  if (auto Added = Pending->addIntegerImmediate(std::move(*Immediate)); !Added)
-    return std::unexpected(Added.error());
-  auto Built = Pending->build();
-  if (!Built)
-    return std::unexpected(Built.error());
-  return *Result;
-}
-
 [[nodiscard]] Expected<Register> buildUnaryBefore(MachineIRBuilder &Builder, MachineInstr &Anchor,
                                                   GenericOpcode Generic, LowLevelType Type,
                                                   Register Source) {
@@ -1272,30 +674,6 @@ void eraseDeadArtifacts(std::span<MachineInstr *const> Candidates,
   return *Result;
 }
 
-[[nodiscard]] Expected<Register> buildBinaryBefore(MachineIRBuilder &Builder, MachineInstr &Anchor,
-                                                   GenericOpcode Generic, LowLevelType Type,
-                                                   Register Left, Register Right) {
-  auto Result = Builder.function().registerInfo().createVirtualRegister(Type);
-  if (!Result)
-    return std::unexpected(Result.error());
-  auto Point = Builder.setInsertPointBefore(Anchor);
-  if (!Point)
-    return std::unexpected(Point.error());
-  auto Pending = Builder.buildInstr(opcode(Generic));
-  if (!Pending)
-    return std::unexpected(Pending.error());
-  if (auto Added = Pending->addDef(*Result); !Added)
-    return std::unexpected(Added.error());
-  if (auto Added = Pending->addUse(Left); !Added)
-    return std::unexpected(Added.error());
-  if (auto Added = Pending->addUse(Right); !Added)
-    return std::unexpected(Added.error());
-  auto Built = Pending->build();
-  if (!Built)
-    return std::unexpected(Built.error());
-  return *Result;
-}
-
 [[nodiscard]] Expected<void> rewriteUnary(MachineInstr &Instruction, GenericOpcode Generic,
                                           Register Destination, Register Source) {
   auto SetOpcode = Instruction.setOpcode(opcode(Generic));
@@ -1319,2469 +697,415 @@ void eraseDeadArtifacts(std::span<MachineInstr *const> Candidates,
   return Instruction.addOperand(std::move(*Use));
 }
 
-[[nodiscard]] Expected<void> widenNarrowMultiply(MachineInstr &Instruction,
-                                                 MachineFunction &Function,
-                                                 const TargetDescription &Target) {
-  const std::optional<LowLevelType> Type = resultType(Function, Instruction);
-  const MachineOperand *DestinationOperand = Instruction.operandAt(0);
-  const MachineOperand *LeftOperand = Instruction.operandAt(1);
-  const MachineOperand *RightOperand = Instruction.operandAt(2);
-  const Register *Destination =
-      DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
-  const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf<Register>();
-  const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf<Register>();
-  if (!Type || Destination == nullptr || Left == nullptr || Right == nullptr ||
-      Type->scalarBitWidth() >= 64) {
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "RISC-V narrow multiply has invalid operands");
-  }
-
-  MachineInstr *Extension = singleRegisterUse(Function, *Destination);
-  if (Extension != nullptr && Extension->opcode() == opcode(GenericOpcode::G_ANYEXT)) {
-    const auto Plan = narrowAnyExtBinaryPlan(*Extension, Function);
-    if (Plan && Plan->Binary == &Instruction)
-      return foldNarrowAnyExtBinary(*Extension, Function, Target);
-  }
-
-  auto WideType = LowLevelType::makeCategoryNeutralScalar(64);
-  if (!WideType)
-    return std::unexpected(WideType.error());
-  MachineIRBuilder Builder(Function, Target);
-  auto WideLeft = buildUnaryBefore(Builder, Instruction, GenericOpcode::G_ANYEXT, *WideType, *Left);
-  if (!WideLeft)
-    return std::unexpected(WideLeft.error());
-  auto WideRight =
-      buildUnaryBefore(Builder, Instruction, GenericOpcode::G_ANYEXT, *WideType, *Right);
-  if (!WideRight)
-    return std::unexpected(WideRight.error());
-  auto Product = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_MUL, *WideType, *WideLeft,
-                                   *WideRight);
-  if (!Product)
-    return std::unexpected(Product.error());
-  return rewriteUnary(Instruction, GenericOpcode::G_TRUNC, *Destination, *Product);
-}
-
-[[nodiscard]] Expected<void> lowerMultiplyLibcall(MachineInstr &Instruction,
-                                                  MachineFunction &Function,
-                                                  const TargetDescription &Target,
-                                                  CallingConvention Convention,
-                                                  const TargetCallLowering &CallLowering) {
-  const std::optional<LowLevelType> Type = resultType(Function, Instruction);
-  const MachineOperand *DestinationOperand = Instruction.operandAt(0);
-  const MachineOperand *LeftOperand = Instruction.operandAt(1);
-  const MachineOperand *RightOperand = Instruction.operandAt(2);
-  const Register *Destination =
-      DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
-  const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf<Register>();
-  const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf<Register>();
-  if (!Type || Destination == nullptr || Left == nullptr || Right == nullptr ||
-      Type->scalarBitWidth() != 64) {
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "RISC-V multiply libcall requires three s64 registers");
-  }
-  auto ABIAlignment = Alignment::create(8);
-  auto Callee = MachineCallTarget::externalSymbol("__muldi3");
-  if (!ABIAlignment || !Callee)
-    return makeUnexpected(ErrorCode::InternalError,
-                          "RISC-V multiply libcall ABI facts are unavailable");
-  const auto Value = [Type = *Type, Alignment = *ABIAlignment](Register Reg, std::uint32_t Number) {
-    return ABIValue{
-        .Value = Reg,
-        .Type = Type,
-        .Category = ABIValueCategory::Integer,
-        .OriginalValue = Number,
-        .FragmentIndex = 0,
-        .FragmentCount = 1,
-        .FragmentOffsetBytes = 0,
-        .OriginalAlignment = Alignment,
-        .Extension = ABIExtensionKind::None,
-        .Flags = {},
-        .ByValSizeBytes = 0,
-        .ByValAlignment = Alignment,
-    };
-  };
-
-  CallLoweringRequest Request;
-  Request.Convention = callingConventionID(Convention);
-  Request.Callee = std::move(*Callee);
-  Request.Arguments = {Value(*Left, 0), Value(*Right, 1)};
-  Request.Results = {Value(*Destination, 0)};
-  Request.FixedArgumentCount = Request.Arguments.size();
-
-  MachineIRBuilder Builder(Function, Target);
-  auto Point = Builder.setInsertPointBefore(Instruction);
-  if (!Point)
-    return std::unexpected(Point.error());
-  auto Lowered = CallLowering.lowerCall(Builder, Request);
-  if (!Lowered)
-    return std::unexpected(Lowered.error());
-  return eraseInstruction(Instruction);
-}
-
-[[nodiscard]] Expected<void> rewriteBinary(MachineInstr &Instruction, GenericOpcode Generic,
-                                           Register Destination, Register Left, Register Right) {
-  auto Rewritten = rewriteUnary(Instruction, Generic, Destination, Left);
-  if (!Rewritten)
-    return std::unexpected(Rewritten.error());
-  auto Use = MachineOperand::registerOperand(Right, RegisterOperandFlags{RegisterOperandFlag::Use});
-  if (!Use)
-    return std::unexpected(Use.error());
-  return Instruction.addOperand(std::move(*Use));
-}
-
-[[nodiscard]] bool isKnownZeroExtended(const MachineFunction &Function, Register Reg,
-                                       std::uint32_t ActiveBits, unsigned Depth = 0) {
-  if (Depth > 8 || !Reg.isVirtual())
-    return false;
-  const auto Type = registerType(Function, Reg);
-  if (!Type)
-    return false;
-  const std::uint32_t Width = Type->scalarBitWidth();
-  if (Width <= ActiveBits)
-    return true;
-  const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(Reg);
-  const MachineInstr *Def = Definition == nullptr ? nullptr : Definition->parent();
-  if (Def == nullptr)
-    return false;
-  if (Def->opcode() == opcode(GenericOpcode::G_ASSERT_ZEXT)) {
-    const MachineOperand *BitsOperand = Def->operandAt(2);
-    auto Bits = BitsOperand == nullptr
-                    ? Expected<std::int64_t>(std::unexpected(
-                          Error(ErrorCode::InvalidArgument, "missing asserted extension width")))
-                    : signedImmediate(*BitsOperand);
-    return Bits && *Bits >= 0 && static_cast<std::uint64_t>(*Bits) <= ActiveBits;
-  }
-  if (Def->opcode() == opcode(GenericOpcode::G_ZEXT)) {
-    const MachineOperand *SourceOperand = Def->operandAt(1);
-    const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>();
-    if (Source == nullptr)
-      return false;
-    const auto SourceType = registerType(Function, *Source);
-    return SourceType && SourceType->scalarBitWidth() <= ActiveBits;
-  }
-  if (Def->opcode() == opcode(GenericOpcode::G_TRUNC)) {
-    const MachineOperand *SourceOperand = Def->operandAt(1);
-    const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>();
-    return Source != nullptr && isKnownZeroExtended(Function, *Source, ActiveBits, Depth + 1U);
-  }
-  return false;
-}
-
-[[nodiscard]] Expected<Register> zeroExtendToRV64(MachineIRBuilder &Builder, MachineInstr &Anchor,
-                                                  MachineFunction &Function, Register Source,
-                                                  std::vector<MachineInstr *> &DeadCandidates) {
-  const auto SourceType = registerType(Function, Source);
-  if (!SourceType)
-    return makeUnexpected(ErrorCode::VerificationError, "RISC-V merge source has no scalar type");
-  if (SourceType->scalarBitWidth() == 64)
-    return Source;
-  const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(Source);
-  MachineInstr *Def =
-      Definition == nullptr ? nullptr : const_cast<MachineInstr *>(Definition->parent());
-  if (Def != nullptr && Def->opcode() == opcode(GenericOpcode::G_TRUNC)) {
-    const MachineOperand *WideOperand = Def->operandAt(1);
-    const Register *Wide = WideOperand == nullptr ? nullptr : WideOperand->getIf<Register>();
-    if (Wide != nullptr) {
-      const auto WideType = registerType(Function, *Wide);
-      if (WideType && WideType->scalarBitWidth() == 64 &&
-          isKnownZeroExtended(Function, *Wide, SourceType->scalarBitWidth())) {
-        DeadCandidates.push_back(Def);
-        return *Wide;
-      }
-    }
-  }
-  auto WideType = LowLevelType::makeCategoryNeutralScalar(64);
-  if (!WideType)
-    return std::unexpected(WideType.error());
-  return buildUnaryBefore(Builder, Anchor, GenericOpcode::G_ZEXT, *WideType, Source);
-}
-
-[[nodiscard]] MachineInstr *singleUseInstruction(const MachineFunction &Function, Register Reg) {
-  MachineInstr *Result = nullptr;
-  auto &RegisterInfo = const_cast<MachineRegisterInfo &>(Function.registerInfo());
-  for (MachineOperand &Operand : RegisterInfo.useOperands(Reg)) {
-    MachineInstr *Instruction = Operand.parent();
-    if (Instruction == nullptr || Instruction->parent() == nullptr)
-      continue;
-    if (Result != nullptr && Result != Instruction)
-      return nullptr;
-    Result = Instruction;
-  }
-  return Result;
-}
-
-[[nodiscard]] Expected<void> lowerWideMergeValues(MachineInstr &Instruction,
-                                                  MachineFunction &Function) {
-  const MachineOperand *DestinationOperand = Instruction.operandAt(0);
-  const Register *Destination =
-      DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
-  const MachineOperand *LowOperand = Instruction.operandAt(1);
-  const Register *Low = LowOperand == nullptr ? nullptr : LowOperand->getIf<Register>();
-  if (Destination == nullptr || Low == nullptr)
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "RISC-V wide merge has invalid register operands");
-
-  std::vector<MachineInstr *> Truncations;
-  auto &RegisterInfo = Function.registerInfo();
-  std::vector<MachineOperand *> Uses;
-  for (MachineOperand &Use : RegisterInfo.useOperands(*Destination))
-    Uses.push_back(&Use);
-  for (MachineOperand *Use : Uses) {
-    MachineInstr *Trunc = Use->parent();
-    if (Trunc == nullptr || Trunc->parent() == nullptr ||
-        Trunc->opcode() != opcode(GenericOpcode::G_TRUNC) || Trunc->operandAt(1) != Use)
-      return makeUnexpected(ErrorCode::Unsupported,
-                            "RISC-V wide merge is only representable through low-part truncation");
-    const MachineOperand *TruncResultOperand = Trunc->operandAt(0);
-    const Register *TruncResult =
-        TruncResultOperand == nullptr ? nullptr : TruncResultOperand->getIf<Register>();
-    if (TruncResult == nullptr)
-      return makeUnexpected(ErrorCode::VerificationError,
-                            "RISC-V wide merge truncation has no result");
-    const auto LowType = registerType(Function, *Low);
-    const auto ResultType = registerType(Function, *TruncResult);
-    if (!LowType || !ResultType || ResultType->scalarBitWidth() > LowType->scalarBitWidth())
-      return makeUnexpected(ErrorCode::Unsupported,
-                            "RISC-V wide merge truncation spans more than its low source");
-    if (*ResultType == *LowType) {
-      auto Replaced = replaceRegisterUses(Function, *TruncResult, *Low);
-      if (!Replaced)
-        return std::unexpected(Replaced.error());
-      Truncations.push_back(Trunc);
-    } else {
-      auto Replaced = Use->setRegister(*Low);
-      if (!Replaced)
-        return std::unexpected(Replaced.error());
-    }
-  }
-  for (MachineInstr *Trunc : Truncations) {
-    auto Erased = eraseInstruction(*Trunc);
-    if (!Erased)
-      return std::unexpected(Erased.error());
-  }
-  return eraseInstruction(Instruction);
-}
-
-[[nodiscard]] Expected<void> lowerMergeValues(MachineInstr &Instruction, MachineFunction &Function,
-                                              const TargetDescription &Target) {
-  const std::optional<LowLevelType> DestinationType = resultType(Function, Instruction);
-  if (!DestinationType || DestinationType->scalarBitWidth() == 0)
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "RISC-V merge destination has no scalar type");
-  if (DestinationType->scalarBitWidth() > 64)
-    return lowerWideMergeValues(Instruction, Function);
-
-  const MachineOperand *DestinationOperand = Instruction.operandAt(0);
-  const Register *Destination =
-      DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
-  if (Destination == nullptr || Instruction.operandCount() < 3)
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "RISC-V merge has invalid register operands");
-
-  Register Result = *Destination;
-  MachineInstr *FoldedExtension = nullptr;
-  if (DestinationType->scalarBitWidth() < 64) {
-    MachineInstr *Use = singleUseInstruction(Function, *Destination);
-    if (Use != nullptr && Use->opcode() == opcode(GenericOpcode::G_ZEXT) &&
-        operandType(Function, *Use, 0)
-            .transform([](LowLevelType Type) { return Type.scalarBitWidth() == 64; })
-            .value_or(false)) {
-      const MachineOperand *ExtendedOperand = Use->operandAt(0);
-      const Register *Extended =
-          ExtendedOperand == nullptr ? nullptr : ExtendedOperand->getIf<Register>();
-      if (Extended != nullptr) {
-        Result = *Extended;
-        FoldedExtension = Use;
-      }
-    }
-  }
-
-  auto WideType = LowLevelType::makeCategoryNeutralScalar(64);
-  if (!WideType)
-    return std::unexpected(WideType.error());
-  MachineIRBuilder Builder(Function, Target);
-  std::vector<MachineInstr *> DeadCandidates;
-  std::vector<Register> Terms;
-  Terms.reserve(Instruction.operandCount() - 1U);
-  for (std::size_t Index = 1; Index < Instruction.operandCount(); ++Index) {
-    const MachineOperand *Operand = Instruction.operandAt(Index);
-    const Register *Source = Operand == nullptr ? nullptr : Operand->getIf<Register>();
-    if (Source == nullptr)
-      return makeUnexpected(ErrorCode::VerificationError, "RISC-V merge source is not a register");
-    auto WideSource = zeroExtendToRV64(Builder, Instruction, Function, *Source, DeadCandidates);
-    if (!WideSource)
-      return std::unexpected(WideSource.error());
-    if (Index == 1) {
-      Terms.push_back(*WideSource);
-      continue;
-    }
-    const std::optional<LowLevelType> SourceType = registerType(Function, *Operand);
-    if (!SourceType)
-      return makeUnexpected(ErrorCode::VerificationError, "RISC-V merge source has no scalar type");
-    const std::uint64_t Shift =
-        static_cast<std::uint64_t>(SourceType->scalarBitWidth()) * (Index - 1U);
-    auto Amount = buildIntegerConstantBefore(Builder, Instruction, 64, Shift);
-    if (!Amount)
-      return std::unexpected(Amount.error());
-    auto Shifted = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_SHL, *WideType,
-                                     *WideSource, *Amount);
-    if (!Shifted)
-      return std::unexpected(Shifted.error());
-    Terms.push_back(*Shifted);
-  }
-
-  Register Accumulator = Terms.front();
-  for (std::size_t Index = 1; Index + 1U < Terms.size(); ++Index) {
-    auto Combined = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_OR, *WideType,
-                                      Accumulator, Terms[Index]);
-    if (!Combined)
-      return std::unexpected(Combined.error());
-    Accumulator = *Combined;
-  }
-  if (DestinationType->scalarBitWidth() == 64 || FoldedExtension != nullptr) {
-    auto Rewritten =
-        rewriteBinary(Instruction, GenericOpcode::G_OR, Result, Accumulator, Terms.back());
-    if (!Rewritten)
-      return std::unexpected(Rewritten.error());
-  } else {
-    auto Combined = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_OR, *WideType,
-                                      Accumulator, Terms.back());
-    if (!Combined)
-      return std::unexpected(Combined.error());
-    auto Rewritten = rewriteUnary(Instruction, GenericOpcode::G_TRUNC, *Destination, *Combined);
-    if (!Rewritten)
-      return std::unexpected(Rewritten.error());
-  }
-  if (FoldedExtension != nullptr) {
-    auto Erased = eraseInstruction(*FoldedExtension);
-    if (!Erased)
-      return std::unexpected(Erased.error());
-  }
-  eraseDeadArtifacts(DeadCandidates, Function);
-  return {};
-}
-
-[[nodiscard]] Expected<void> foldWideExtendedUnmerge(MachineInstr &Instruction,
-                                                     MachineFunction &Function) {
-  const std::size_t OutputCount = Instruction.operandCount() - 1U;
-  const MachineOperand *SourceOperand = Instruction.operandAt(OutputCount);
-  const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>();
-  const MachineOperand *Definition =
-      Source == nullptr ? nullptr : Function.registerInfo().getUniqueVRegDef(*Source);
-  MachineInstr *Extension =
-      Definition == nullptr ? nullptr : const_cast<MachineInstr *>(Definition->parent());
-  if (Extension == nullptr || (Extension->opcode() != opcode(GenericOpcode::G_ZEXT) &&
-                               Extension->opcode() != opcode(GenericOpcode::G_ANYEXT) &&
-                               Extension->opcode() != opcode(GenericOpcode::G_SEXT)))
-    return makeUnexpected(ErrorCode::Unsupported,
-                          "RISC-V wide unmerge requires an extension-defined source");
-  const MachineOperand *NarrowOperand = Extension->operandAt(1);
-  const Register *Narrow = NarrowOperand == nullptr ? nullptr : NarrowOperand->getIf<Register>();
-  if (Narrow == nullptr)
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "RISC-V wide unmerge extension has no register source");
-  const auto NarrowType = registerType(Function, *Narrow);
-  if (!NarrowType)
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "RISC-V wide unmerge extension source has no type");
-  const std::uint32_t NarrowWidth = NarrowType->scalarBitWidth();
-
-  std::vector<MachineInstr *> DeadUsers;
-  for (std::size_t Index = 0; Index < OutputCount; ++Index) {
-    const MachineOperand *OutputOperand = Instruction.operandAt(Index);
-    const Register *Output = OutputOperand == nullptr ? nullptr : OutputOperand->getIf<Register>();
-    if (Output == nullptr || Function.registerInfo().useEmpty(*Output))
-      continue;
-    const auto OutputType = registerType(Function, *Output);
-    if (!OutputType)
-      return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide unmerge output has no type");
-    const std::uint64_t Offset = static_cast<std::uint64_t>(Index) * OutputType->scalarBitWidth();
-    if (Offset != 0)
-      return makeUnexpected(ErrorCode::Unsupported,
-                            "RISC-V wide unmerge cannot materialize a used high wide part");
-    if (OutputType->scalarBitWidth() == NarrowWidth) {
-      auto Replaced = replaceRegisterUses(Function, *Output, *Narrow);
-      if (!Replaced)
-        return std::unexpected(Replaced.error());
-      continue;
-    }
-    if (OutputType->scalarBitWidth() < NarrowWidth)
-      return makeUnexpected(ErrorCode::Unsupported,
-                            "RISC-V wide unmerge output unexpectedly narrows its source");
-    std::vector<MachineOperand *> Uses;
-    for (MachineOperand &Use : Function.registerInfo().useOperands(*Output))
-      Uses.push_back(&Use);
-    for (MachineOperand *Use : Uses) {
-      MachineInstr *Trunc = Use->parent();
-      if (Trunc == nullptr || Trunc->parent() == nullptr ||
-          Trunc->opcode() != opcode(GenericOpcode::G_TRUNC) || Trunc->operandAt(1) != Use)
-        return makeUnexpected(
-            ErrorCode::Unsupported,
-            "RISC-V wide unmerge result is only representable through truncation");
-      const MachineOperand *ResultOperand = Trunc->operandAt(0);
-      const Register *Result =
-          ResultOperand == nullptr ? nullptr : ResultOperand->getIf<Register>();
-      if (Result == nullptr)
-        return makeUnexpected(ErrorCode::VerificationError,
-                              "RISC-V wide unmerge truncation has no result");
-      const auto ResultType = registerType(Function, *Result);
-      if (!ResultType || ResultType->scalarBitWidth() > NarrowWidth)
-        return makeUnexpected(ErrorCode::Unsupported,
-                              "RISC-V wide unmerge truncation exceeds its source width");
-      if (ResultType->scalarBitWidth() == NarrowWidth) {
-        auto Replaced = replaceRegisterUses(Function, *Result, *Narrow);
-        if (!Replaced)
-          return std::unexpected(Replaced.error());
-        DeadUsers.push_back(Trunc);
-      } else {
-        auto Replaced = Use->setRegister(*Narrow);
-        if (!Replaced)
-          return std::unexpected(Replaced.error());
-      }
-    }
-  }
-  for (MachineInstr *User : DeadUsers) {
-    auto Erased = eraseInstruction(*User);
-    if (!Erased)
-      return std::unexpected(Erased.error());
-  }
-  auto Erased = eraseInstruction(Instruction);
-  if (!Erased)
-    return std::unexpected(Erased.error());
-  if (Function.registerInfo().useEmpty(*Source))
-    return eraseInstruction(*Extension);
-  return {};
-}
-
-[[nodiscard]] Expected<void> lowerUnmergeValues(MachineInstr &Instruction,
-                                                MachineFunction &Function,
-                                                const TargetDescription &Target) {
-  if (Instruction.operandCount() < 3)
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "RISC-V unmerge has invalid register operands");
-  const std::size_t OutputCount = Instruction.operandCount() - 1U;
-  const std::optional<LowLevelType> OutputType = operandType(Function, Instruction, 0);
-  const std::optional<LowLevelType> SourceType = operandType(Function, Instruction, OutputCount);
-  if (!OutputType || !SourceType)
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "RISC-V unmerge operands have no scalar types");
-  if (SourceType->scalarBitWidth() > 64 || OutputType->scalarBitWidth() > 64)
-    return foldWideExtendedUnmerge(Instruction, Function);
-
-  const MachineOperand *SourceOperand = Instruction.operandAt(OutputCount);
-  const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>();
-  if (Source == nullptr)
-    return makeUnexpected(ErrorCode::VerificationError, "RISC-V unmerge source is not a register");
-  auto WideType = LowLevelType::makeCategoryNeutralScalar(64);
-  if (!WideType)
-    return std::unexpected(WideType.error());
-  MachineIRBuilder Builder(Function, Target);
-  std::vector<MachineInstr *> FoldedExtensions;
-  for (std::size_t Index = 0; Index < OutputCount; ++Index) {
-    const MachineOperand *OutputOperand = Instruction.operandAt(Index);
-    const Register *Output = OutputOperand == nullptr ? nullptr : OutputOperand->getIf<Register>();
-    if (Output == nullptr || Function.registerInfo().useEmpty(*Output))
-      continue;
-    Register Extracted = *Source;
-    const std::uint64_t Offset = static_cast<std::uint64_t>(Index) * OutputType->scalarBitWidth();
-    if (Offset != 0) {
-      auto Amount = buildIntegerConstantBefore(Builder, Instruction, 64, Offset);
-      if (!Amount)
-        return std::unexpected(Amount.error());
-      auto Shifted = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_LSHR, *WideType,
-                                       *Source, *Amount);
-      if (!Shifted)
-        return std::unexpected(Shifted.error());
-      Extracted = *Shifted;
-    }
-
-    MachineInstr *Extension = singleUseInstruction(Function, *Output);
-    const bool FoldZExt =
-        Extension != nullptr && Extension->opcode() == opcode(GenericOpcode::G_ZEXT) &&
-        operandType(Function, *Extension, 0)
-            .transform([](LowLevelType Type) { return Type.scalarBitWidth() == 64; })
-            .value_or(false);
-    if (FoldZExt) {
-      const MachineOperand *ExtendedOperand = Extension->operandAt(0);
-      const Register *Extended =
-          ExtendedOperand == nullptr ? nullptr : ExtendedOperand->getIf<Register>();
-      if (Extended == nullptr)
-        return makeUnexpected(ErrorCode::VerificationError,
-                              "RISC-V unmerge extension has no destination");
-      const std::uint64_t Mask = OutputType->scalarBitWidth() == 64
-                                     ? std::numeric_limits<std::uint64_t>::max()
-                                     : (std::uint64_t{1} << OutputType->scalarBitWidth()) - 1U;
-      auto MaskReg = buildIntegerConstantBefore(Builder, Instruction, 64, Mask);
-      if (!MaskReg)
-        return std::unexpected(MaskReg.error());
-      auto Masked = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_AND, *WideType,
-                                      Extracted, *MaskReg);
-      if (!Masked)
-        return std::unexpected(Masked.error());
-      auto Replaced = replaceRegisterUses(Function, *Extended, *Masked);
-      if (!Replaced)
-        return std::unexpected(Replaced.error());
-      FoldedExtensions.push_back(Extension);
-      continue;
-    }
-    auto Truncated =
-        buildUnaryBefore(Builder, Instruction, GenericOpcode::G_TRUNC, *OutputType, Extracted);
-    if (!Truncated)
-      return std::unexpected(Truncated.error());
-    auto Replaced = replaceRegisterUses(Function, *Output, *Truncated);
-    if (!Replaced)
-      return std::unexpected(Replaced.error());
-  }
-  for (MachineInstr *Extension : FoldedExtensions) {
-    auto Erased = eraseInstruction(*Extension);
-    if (!Erased)
-      return std::unexpected(Erased.error());
-  }
-  return eraseInstruction(Instruction);
-}
-
-struct WideValueParts final {
-  std::uint32_t BitWidth = 0;
-  std::vector<Register> Parts;
-  std::vector<MachineInstr *> Artifacts;
-};
-
-[[nodiscard]] Expected<WideValueParts>
-collectWideValueParts(Register Value, MachineFunction &Function, unsigned Depth = 0) {
-  if (Depth > 16)
-    return makeUnexpected(ErrorCode::Unsupported, "RISC-V wide-value artifact chain is too deep");
-  const auto Type = registerType(Function, Value);
-  if (!Type)
-    return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide value has no scalar type");
-  if (!Type->isCategoryNeutralScalar() || Type->scalarBitWidth() == 0)
-    return makeUnexpected(ErrorCode::Unsupported,
-                          "RISC-V wide-value decomposition requires an integer scalar");
-  if (Type->scalarBitWidth() <= 64)
-    return WideValueParts{.BitWidth = Type->scalarBitWidth(), .Parts = {Value}, .Artifacts = {}};
-
-  const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(Value);
-  MachineInstr *Def =
-      Definition == nullptr ? nullptr : const_cast<MachineInstr *>(Definition->parent());
-  if (Def == nullptr)
-    return makeUnexpected(ErrorCode::Unsupported,
-                          "RISC-V wide value has no decomposable definition");
-  if (Def->opcode() == opcode(GenericOpcode::G_MERGE_VALUES)) {
-    WideValueParts Result;
-    Result.BitWidth = Type->scalarBitWidth();
-    for (std::size_t Index = 1; Index < Def->operandCount(); ++Index) {
-      const MachineOperand *Operand = Def->operandAt(Index);
-      const Register *Part = Operand == nullptr ? nullptr : Operand->getIf<Register>();
-      const std::optional<LowLevelType> PartType =
-          Operand == nullptr ? std::nullopt : registerType(Function, *Operand);
-      if (Part == nullptr || !PartType || PartType->scalarBitWidth() != 64)
-        return makeUnexpected(ErrorCode::Unsupported,
-                              "RISC-V wide merge must already use 64-bit limbs");
-      Result.Parts.push_back(*Part);
-    }
-    Result.Artifacts.push_back(Def);
-    return Result;
-  }
-  if (Def->opcode() == opcode(GenericOpcode::G_TRUNC)) {
-    const MachineOperand *SourceOperand = Def->operandAt(1);
-    const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>();
-    if (Source == nullptr)
-      return makeUnexpected(ErrorCode::VerificationError,
-                            "RISC-V wide truncation has no source register");
-    auto Result = collectWideValueParts(*Source, Function, Depth + 1U);
-    if (!Result)
-      return std::unexpected(Result.error());
-    Result->BitWidth = Type->scalarBitWidth();
-    const std::size_t RequiredParts = (Type->scalarBitWidth() + 63U) / 64U;
-    if (Result->Parts.size() < RequiredParts)
-      return makeUnexpected(ErrorCode::VerificationError,
-                            "RISC-V wide truncation source has too few limbs");
-    Result->Parts.resize(RequiredParts);
-    Result->Artifacts.push_back(Def);
-    return Result;
-  }
-  return makeUnexpected(ErrorCode::Unsupported,
-                        "RISC-V wide value is not defined by merge/trunc artifacts");
-}
-
-struct WideResultPlan final {
-  std::vector<Register> Outputs;
-  MachineInstr *Extension = nullptr;
-  MachineInstr *Unmerge = nullptr;
-};
-
-[[nodiscard]] Expected<WideResultPlan> wideResultPlan(Register Result, MachineFunction &Function,
-                                                      std::size_t RequiredParts) {
-  MachineInstr *Use = singleUseInstruction(Function, Result);
-  WideResultPlan Plan;
-  if (Use != nullptr && (Use->opcode() == opcode(GenericOpcode::G_ANYEXT) ||
-                         Use->opcode() == opcode(GenericOpcode::G_SEXT) ||
-                         Use->opcode() == opcode(GenericOpcode::G_ZEXT))) {
-    Plan.Extension = Use;
-    const MachineOperand *ExtendedOperand = Use->operandAt(0);
-    const Register *Extended =
-        ExtendedOperand == nullptr ? nullptr : ExtendedOperand->getIf<Register>();
-    Use = Extended == nullptr ? nullptr : singleUseInstruction(Function, *Extended);
-  }
-  if (Use == nullptr || Use->opcode() != opcode(GenericOpcode::G_UNMERGE_VALUES))
-    return makeUnexpected(ErrorCode::Unsupported,
-                          "RISC-V wide result must feed an unmerge artifact");
-  Plan.Unmerge = Use;
-  const std::size_t OutputCount = Use->operandCount() - 1U;
-  if (OutputCount < RequiredParts)
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "RISC-V wide result unmerge has too few outputs");
-  Plan.Outputs.reserve(RequiredParts);
-  for (std::size_t Index = 0; Index < RequiredParts; ++Index) {
-    const MachineOperand *Operand = Use->operandAt(Index);
-    const Register *Output = Operand == nullptr ? nullptr : Operand->getIf<Register>();
-    const std::optional<LowLevelType> Type =
-        Operand == nullptr ? std::nullopt : registerType(Function, *Operand);
-    if (Output == nullptr || !Type || Type->scalarBitWidth() != 64)
-      return makeUnexpected(ErrorCode::Unsupported,
-                            "RISC-V wide result must unmerge to 64-bit limbs");
-    Plan.Outputs.push_back(*Output);
-  }
-  return Plan;
-}
-
-[[nodiscard]] Expected<Register> buildIntegerCompareBefore(MachineIRBuilder &Builder,
-                                                           MachineInstr &Anchor,
-                                                           IntegerPredicate Predicate,
-                                                           Register Left, Register Right) {
-  auto Type = LowLevelType::makeCategoryNeutralScalar(64);
-  if (!Type)
-    return std::unexpected(Type.error());
-  auto Result = Builder.function().registerInfo().createVirtualRegister(*Type);
-  if (!Result)
-    return std::unexpected(Result.error());
-  auto Point = Builder.setInsertPointBefore(Anchor);
-  if (!Point)
-    return std::unexpected(Point.error());
-  auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_ICMP));
-  if (!Pending)
-    return std::unexpected(Pending.error());
-  if (auto Added = Pending->addDef(*Result); !Added)
-    return std::unexpected(Added.error());
-  if (auto Added = Pending->addIntegerPredicate(Predicate); !Added)
-    return std::unexpected(Added.error());
-  if (auto Added = Pending->addUse(Left); !Added)
-    return std::unexpected(Added.error());
-  if (auto Added = Pending->addUse(Right); !Added)
-    return std::unexpected(Added.error());
-  auto Built = Pending->build();
-  if (!Built)
-    return std::unexpected(Built.error());
-  return *Result;
-}
-
-[[nodiscard]] Expected<Register> buildSelectBefore(MachineIRBuilder &Builder, MachineInstr &Anchor,
-                                                   LowLevelType Type, Register Condition,
-                                                   Register TrueValue, Register FalseValue) {
-  auto Result = Builder.function().registerInfo().createVirtualRegister(Type);
-  if (!Result)
-    return std::unexpected(Result.error());
-  auto Point = Builder.setInsertPointBefore(Anchor);
-  if (!Point)
-    return std::unexpected(Point.error());
-  auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_SELECT));
-  if (!Pending)
-    return std::unexpected(Pending.error());
-  if (auto Added = Pending->addDef(*Result); !Added)
-    return std::unexpected(Added.error());
-  if (auto Added = Pending->addUse(Condition); !Added)
-    return std::unexpected(Added.error());
-  if (auto Added = Pending->addUse(TrueValue); !Added)
-    return std::unexpected(Added.error());
-  if (auto Added = Pending->addUse(FalseValue); !Added)
-    return std::unexpected(Added.error());
-  auto Built = Pending->build();
-  if (!Built)
-    return std::unexpected(Built.error());
-  return *Result;
-}
-
-struct TwoLimbShiftResult final {
-  Register Low;
-  Register High;
-};
-
-[[nodiscard]] Expected<TwoLimbShiftResult>
-emitTwoLimbLeftShiftBody(MachineIRBuilder &Builder, MachineInstr &Anchor, Register Low,
-                         Register High, Register Amount, Register AmountExcess, Register AmountLack,
-                         Register Zero, Register IsShort, Register IsZero) {
-  auto Type = LowLevelType::makeCategoryNeutralScalar(64);
-  if (!Type)
-    return std::unexpected(Type.error());
-  auto LowShort = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SHL, *Type, Low, Amount);
-  if (!LowShort)
-    return std::unexpected(LowShort.error());
-  auto CrossLow = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_LSHR, *Type, Low, AmountLack);
-  if (!CrossLow)
-    return std::unexpected(CrossLow.error());
-  auto HighShort = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SHL, *Type, High, Amount);
-  if (!HighShort)
-    return std::unexpected(HighShort.error());
-  auto ShortHigh =
-      buildBinaryBefore(Builder, Anchor, GenericOpcode::G_OR, *Type, *CrossLow, *HighShort);
-  if (!ShortHigh)
-    return std::unexpected(ShortHigh.error());
-  auto LongHigh =
-      buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SHL, *Type, Low, AmountExcess);
-  if (!LongHigh)
-    return std::unexpected(LongHigh.error());
-  auto ResultLow = buildSelectBefore(Builder, Anchor, *Type, IsShort, *LowShort, Zero);
-  if (!ResultLow)
-    return std::unexpected(ResultLow.error());
-  auto ShiftedHigh = buildSelectBefore(Builder, Anchor, *Type, IsShort, *ShortHigh, *LongHigh);
-  if (!ShiftedHigh)
-    return std::unexpected(ShiftedHigh.error());
-  auto ResultHigh = buildSelectBefore(Builder, Anchor, *Type, IsZero, High, *ShiftedHigh);
-  if (!ResultHigh)
-    return std::unexpected(ResultHigh.error());
-  return TwoLimbShiftResult{.Low = *ResultLow, .High = *ResultHigh};
-}
-
-[[nodiscard]] Expected<TwoLimbShiftResult> emitTwoLimbLeftShift(MachineIRBuilder &Builder,
-                                                                MachineInstr &Anchor, Register Low,
-                                                                Register High, Register Amount) {
-  auto Type = LowLevelType::makeCategoryNeutralScalar(64);
-  if (!Type)
-    return std::unexpected(Type.error());
-  auto Boundary = buildIntegerConstantBefore(Builder, Anchor, 64, 64);
-  if (!Boundary)
-    return std::unexpected(Boundary.error());
-  auto AmountExcess =
-      buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, Amount, *Boundary);
-  if (!AmountExcess)
-    return std::unexpected(AmountExcess.error());
-  auto AmountLack =
-      buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, *Boundary, Amount);
-  if (!AmountLack)
-    return std::unexpected(AmountLack.error());
-  auto Zero = buildIntegerConstantBefore(Builder, Anchor, 64, 0);
-  if (!Zero)
-    return std::unexpected(Zero.error());
-  auto IsShort = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan,
-                                           Amount, *Boundary);
-  if (!IsShort)
-    return std::unexpected(IsShort.error());
-  auto IsZero = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::Equal, Amount, *Zero);
-  if (!IsZero)
-    return std::unexpected(IsZero.error());
-  return emitTwoLimbLeftShiftBody(Builder, Anchor, Low, High, Amount, *AmountExcess, *AmountLack,
-                                  *Zero, *IsShort, *IsZero);
-}
-
-[[nodiscard]] Expected<Register>
-emitLowLimbOfTwoLimbLogicalRightShift(MachineIRBuilder &Builder, MachineInstr &Anchor, Register Low,
-                                      Register High, Register Amount, Register Boundary,
-                                      Register Zero) {
-  auto Type = LowLevelType::makeCategoryNeutralScalar(64);
-  if (!Type)
-    return std::unexpected(Type.error());
-  auto AmountExcess =
-      buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, Amount, Boundary);
-  if (!AmountExcess)
-    return std::unexpected(AmountExcess.error());
-  auto AmountLack =
-      buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, Boundary, Amount);
-  if (!AmountLack)
-    return std::unexpected(AmountLack.error());
-  auto IsShort = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan,
-                                           Amount, Boundary);
-  if (!IsShort)
-    return std::unexpected(IsShort.error());
-  auto IsZero = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::Equal, Amount, Zero);
-  if (!IsZero)
-    return std::unexpected(IsZero.error());
-  auto LowShort = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_LSHR, *Type, Low, Amount);
-  if (!LowShort)
-    return std::unexpected(LowShort.error());
-  auto HighShort =
-      buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SHL, *Type, High, *AmountLack);
-  if (!HighShort)
-    return std::unexpected(HighShort.error());
-  auto ShortValue =
-      buildBinaryBefore(Builder, Anchor, GenericOpcode::G_OR, *Type, *LowShort, *HighShort);
-  if (!ShortValue)
-    return std::unexpected(ShortValue.error());
-  auto LongValue =
-      buildBinaryBefore(Builder, Anchor, GenericOpcode::G_LSHR, *Type, High, *AmountExcess);
-  if (!LongValue)
-    return std::unexpected(LongValue.error());
-  auto Shifted = buildSelectBefore(Builder, Anchor, *Type, *IsShort, *ShortValue, *LongValue);
-  if (!Shifted)
-    return std::unexpected(Shifted.error());
-  return buildSelectBefore(Builder, Anchor, *Type, *IsZero, Low, *Shifted);
-}
-
-[[nodiscard]] Expected<std::vector<Register>>
-emitThreeLimbLeftShift(MachineIRBuilder &Builder, MachineInstr &Anchor,
-                       std::span<const Register> Parts, Register Amount) {
-  if (Parts.size() != 3)
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "RISC-V three-limb shift requires exactly three limbs");
-  auto Type = LowLevelType::makeCategoryNeutralScalar(64);
-  if (!Type)
-    return std::unexpected(Type.error());
-
-  auto OuterBoundary = buildIntegerConstantBefore(Builder, Anchor, 64, 128);
-  if (!OuterBoundary)
-    return std::unexpected(OuterBoundary.error());
-  auto OuterExcess =
-      buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, Amount, *OuterBoundary);
-  if (!OuterExcess)
-    return std::unexpected(OuterExcess.error());
-  auto OuterLack =
-      buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, *OuterBoundary, Amount);
-  if (!OuterLack)
-    return std::unexpected(OuterLack.error());
-  auto Zero = buildIntegerConstantBefore(Builder, Anchor, 64, 0);
-  if (!Zero)
-    return std::unexpected(Zero.error());
-  auto OuterIsShort = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan,
-                                                Amount, *OuterBoundary);
-  if (!OuterIsShort)
-    return std::unexpected(OuterIsShort.error());
-  auto OuterIsZero =
-      buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::Equal, Amount, *Zero);
-  if (!OuterIsZero)
-    return std::unexpected(OuterIsZero.error());
-
-  auto InnerBoundary = buildIntegerConstantBefore(Builder, Anchor, 64, 64);
-  if (!InnerBoundary)
-    return std::unexpected(InnerBoundary.error());
-  auto InnerExcess =
-      buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, Amount, *InnerBoundary);
-  if (!InnerExcess)
-    return std::unexpected(InnerExcess.error());
-  auto InnerLack =
-      buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, *InnerBoundary, Amount);
-  if (!InnerLack)
-    return std::unexpected(InnerLack.error());
-  auto InnerIsShort = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan,
-                                                Amount, *InnerBoundary);
-  if (!InnerIsShort)
-    return std::unexpected(InnerIsShort.error());
-  auto InnerIsZero =
-      buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::Equal, Amount, *Zero);
-  if (!InnerIsZero)
-    return std::unexpected(InnerIsZero.error());
-  auto Lower = emitTwoLimbLeftShiftBody(Builder, Anchor, Parts[0], Parts[1], Amount, *InnerExcess,
-                                        *InnerLack, *Zero, *InnerIsShort, *InnerIsZero);
-  if (!Lower)
-    return std::unexpected(Lower.error());
-
-  auto Cross = emitLowLimbOfTwoLimbLogicalRightShift(Builder, Anchor, Parts[0], Parts[1],
-                                                     *OuterLack, *InnerBoundary, *Zero);
-  if (!Cross)
-    return std::unexpected(Cross.error());
-  auto HighIsShort = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan,
-                                               Amount, *InnerBoundary);
-  if (!HighIsShort)
-    return std::unexpected(HighIsShort.error());
-  auto HighShort =
-      buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SHL, *Type, Parts[2], Amount);
-  if (!HighShort)
-    return std::unexpected(HighShort.error());
-  auto BoundedHigh = buildSelectBefore(Builder, Anchor, *Type, *HighIsShort, *HighShort, *Zero);
-  if (!BoundedHigh)
-    return std::unexpected(BoundedHigh.error());
-  auto ShortHigh =
-      buildBinaryBefore(Builder, Anchor, GenericOpcode::G_OR, *Type, *Cross, *BoundedHigh);
-  if (!ShortHigh)
-    return std::unexpected(ShortHigh.error());
-
-  auto LongIsShort = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan,
-                                               *OuterExcess, *InnerBoundary);
-  if (!LongIsShort)
-    return std::unexpected(LongIsShort.error());
-  auto LongShift =
-      buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SHL, *Type, Parts[0], *OuterExcess);
-  if (!LongShift)
-    return std::unexpected(LongShift.error());
-  auto LongHigh = buildSelectBefore(Builder, Anchor, *Type, *LongIsShort, *LongShift, *Zero);
-  if (!LongHigh)
-    return std::unexpected(LongHigh.error());
-
-  auto ResultLow = buildSelectBefore(Builder, Anchor, *Type, *OuterIsShort, Lower->Low, *Zero);
-  if (!ResultLow)
-    return std::unexpected(ResultLow.error());
-  auto ResultMiddle = buildSelectBefore(Builder, Anchor, *Type, *OuterIsShort, Lower->High, *Zero);
-  if (!ResultMiddle)
-    return std::unexpected(ResultMiddle.error());
-  auto ShiftedHigh =
-      buildSelectBefore(Builder, Anchor, *Type, *OuterIsShort, *ShortHigh, *LongHigh);
-  if (!ShiftedHigh)
-    return std::unexpected(ShiftedHigh.error());
-  auto ResultHigh = buildSelectBefore(Builder, Anchor, *Type, *OuterIsZero, Parts[2], *ShiftedHigh);
-  if (!ResultHigh)
-    return std::unexpected(ResultHigh.error());
-  return std::vector<Register>{*ResultLow, *ResultMiddle, *ResultHigh};
-}
-
-[[nodiscard]] Expected<void> lowerWideShiftLeft(MachineInstr &Instruction,
-                                                MachineFunction &Function,
-                                                const TargetDescription &Target) {
-  const MachineOperand *DestinationOperand = Instruction.operandAt(0);
-  const MachineOperand *ValueOperand = Instruction.operandAt(1);
-  const MachineOperand *AmountOperand = Instruction.operandAt(2);
-  const Register *Destination =
-      DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
-  const Register *Value = ValueOperand == nullptr ? nullptr : ValueOperand->getIf<Register>();
-  const Register *Amount = AmountOperand == nullptr ? nullptr : AmountOperand->getIf<Register>();
-  if (Destination == nullptr || Value == nullptr || Amount == nullptr)
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "RISC-V wide shift has invalid register operands");
-  auto ValueParts = collectWideValueParts(*Value, Function);
-  if (!ValueParts)
-    return std::unexpected(ValueParts.error());
-  auto AmountParts = collectWideValueParts(*Amount, Function);
-  if (!AmountParts)
-    return std::unexpected(AmountParts.error());
-  if (ValueParts->Parts.size() < 2 || AmountParts->Parts.empty())
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "RISC-V wide shift has an invalid limb decomposition");
-  auto ResultPlan = wideResultPlan(*Destination, Function, ValueParts->Parts.size());
-  if (!ResultPlan)
-    return std::unexpected(ResultPlan.error());
-
-  MachineIRBuilder Builder(Function, Target);
-  Expected<std::vector<Register>> Results =
-      ValueParts->Parts.size() == 2
-          ? emitTwoLimbLeftShift(Builder, Instruction, ValueParts->Parts[0], ValueParts->Parts[1],
-                                 AmountParts->Parts[0])
-                .transform([](TwoLimbShiftResult Result) {
-                  return std::vector<Register>{Result.Low, Result.High};
-                })
-      : ValueParts->Parts.size() == 3
-          ? emitThreeLimbLeftShift(Builder, Instruction, ValueParts->Parts, AmountParts->Parts[0])
-          : Expected<std::vector<Register>>(std::unexpected(
-                Error(ErrorCode::Unsupported,
-                      "RISC-V wide shift currently requires two or three 64-bit limbs")));
-  if (!Results)
-    return std::unexpected(Results.error());
-  for (std::size_t Index = 0; Index < Results->size(); ++Index) {
-    auto Replaced = replaceRegisterUses(Function, ResultPlan->Outputs[Index], (*Results)[Index]);
-    if (!Replaced)
-      return std::unexpected(Replaced.error());
-  }
-  if (auto Erased = eraseInstruction(*ResultPlan->Unmerge); !Erased)
-    return std::unexpected(Erased.error());
-  if (ResultPlan->Extension != nullptr) {
-    if (auto Erased = eraseInstruction(*ResultPlan->Extension); !Erased)
-      return std::unexpected(Erased.error());
-  }
-  if (auto Erased = eraseInstruction(Instruction); !Erased)
-    return std::unexpected(Erased.error());
-  for (std::size_t Index = 1; Index < AmountParts->Parts.size(); ++Index) {
-    const MachineOperand *Definition =
-        Function.registerInfo().getUniqueVRegDef(AmountParts->Parts[Index]);
-    if (Definition != nullptr && Definition->parent() != nullptr)
-      AmountParts->Artifacts.push_back(const_cast<MachineInstr *>(Definition->parent()));
-  }
-  ValueParts->Artifacts.insert(ValueParts->Artifacts.end(), AmountParts->Artifacts.begin(),
-                               AmountParts->Artifacts.end());
-  eraseDeadArtifacts(ValueParts->Artifacts, Function);
-  return {};
-}
-
-[[nodiscard]] Expected<std::vector<Register>> emitWideAdd(MachineIRBuilder &Builder,
-                                                          MachineInstr &Anchor,
-                                                          std::span<const Register> Left,
-                                                          std::span<const Register> Right) {
-  if (Left.size() != Right.size() || Left.size() < 2)
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "RISC-V wide add requires matching multi-limb operands");
-  auto Type = LowLevelType::makeCategoryNeutralScalar(64);
-  if (!Type)
-    return std::unexpected(Type.error());
-  std::vector<Register> Results;
-  Results.reserve(Left.size());
-  auto Low = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_ADD, *Type, Left[0], Right[0]);
-  if (!Low)
-    return std::unexpected(Low.error());
-  auto Carry = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan, *Low,
-                                         Right[0]);
-  if (!Carry)
-    return std::unexpected(Carry.error());
-  auto LowCopy = buildUnaryBefore(Builder, Anchor, GenericOpcode::COPY, *Type, *Low);
-  if (!LowCopy)
-    return std::unexpected(LowCopy.error());
-  Results.push_back(*LowCopy);
-  for (std::size_t Index = 1; Index < Left.size(); ++Index) {
-    auto Base =
-        buildBinaryBefore(Builder, Anchor, GenericOpcode::G_ADD, *Type, Left[Index], Right[Index]);
-    if (!Base)
-      return std::unexpected(Base.error());
-    Register BaseCarry;
-    if (Index + 1U < Left.size()) {
-      auto Compared = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan,
-                                                *Base, Left[Index]);
-      if (!Compared)
-        return std::unexpected(Compared.error());
-      BaseCarry = *Compared;
-    }
-    auto Sum = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_ADD, *Type, *Base, *Carry);
-    if (!Sum)
-      return std::unexpected(Sum.error());
-    if (Index + 1U < Left.size()) {
-      auto Zero = buildIntegerConstantBefore(Builder, Anchor, 64, 0);
-      if (!Zero)
-        return std::unexpected(Zero.error());
-      auto IsZero =
-          buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::Equal, *Sum, *Zero);
-      if (!IsZero)
-        return std::unexpected(IsZero.error());
-      auto CarryFromIncoming =
-          buildBinaryBefore(Builder, Anchor, GenericOpcode::G_AND, *Type, *IsZero, *Carry);
-      if (!CarryFromIncoming)
-        return std::unexpected(CarryFromIncoming.error());
-      auto NextCarry = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_OR, *Type, BaseCarry,
-                                         *CarryFromIncoming);
-      if (!NextCarry)
-        return std::unexpected(NextCarry.error());
-      Carry = *NextCarry;
-    }
-    auto Copy = buildUnaryBefore(Builder, Anchor, GenericOpcode::COPY, *Type, *Sum);
-    if (!Copy)
-      return std::unexpected(Copy.error());
-    Results.push_back(*Copy);
-  }
-  return Results;
-}
-
-[[nodiscard]] Expected<std::vector<Register>> emitWideSub(MachineIRBuilder &Builder,
-                                                          MachineInstr &Anchor,
-                                                          std::span<const Register> Left,
-                                                          std::span<const Register> Right) {
-  if (Left.size() != Right.size() || Left.size() < 2)
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "RISC-V wide sub requires matching multi-limb operands");
-  auto Type = LowLevelType::makeCategoryNeutralScalar(64);
-  if (!Type)
-    return std::unexpected(Type.error());
-  std::vector<Register> Results;
-  Results.reserve(Left.size());
-  auto Low = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, Left[0], Right[0]);
-  if (!Low)
-    return std::unexpected(Low.error());
-  Results.push_back(*Low);
-  auto Borrow = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan,
-                                          Left[0], Right[0]);
-  if (!Borrow)
-    return std::unexpected(Borrow.error());
-  for (std::size_t Index = 1; Index < Left.size(); ++Index) {
-    auto Base =
-        buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, Left[Index], Right[Index]);
-    if (!Base)
-      return std::unexpected(Base.error());
-    Register BaseBorrow;
-    if (Index + 1U < Left.size()) {
-      auto Compared = buildIntegerCompareBefore(
-          Builder, Anchor, IntegerPredicate::UnsignedGreaterThan, *Base, Left[Index]);
-      if (!Compared)
-        return std::unexpected(Compared.error());
-      BaseBorrow = *Compared;
-    }
-    auto Difference =
-        buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, *Base, *Borrow);
-    if (!Difference)
-      return std::unexpected(Difference.error());
-    Results.push_back(*Difference);
-    if (Index + 1U < Left.size()) {
-      auto Zero = buildIntegerConstantBefore(Builder, Anchor, 64, 0);
-      if (!Zero)
-        return std::unexpected(Zero.error());
-      auto IsZero =
-          buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::Equal, *Base, *Zero);
-      if (!IsZero)
-        return std::unexpected(IsZero.error());
-      auto BorrowFromIncoming =
-          buildBinaryBefore(Builder, Anchor, GenericOpcode::G_AND, *Type, *IsZero, *Borrow);
-      if (!BorrowFromIncoming)
-        return std::unexpected(BorrowFromIncoming.error());
-      auto NextBorrow = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_OR, *Type, BaseBorrow,
-                                          *BorrowFromIncoming);
-      if (!NextBorrow)
-        return std::unexpected(NextBorrow.error());
-      Borrow = *NextBorrow;
-    }
-  }
-  return Results;
-}
-
-[[nodiscard]] Expected<std::vector<Register>>
-emitWideBitwise(MachineIRBuilder &Builder, MachineInstr &Anchor, GenericOpcode Generic,
-                std::span<const Register> Left, std::span<const Register> Right) {
-  if (Left.size() != Right.size() || Left.size() < 2)
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "RISC-V wide bitwise operation requires matching multi-limb operands");
-  auto Type = LowLevelType::makeCategoryNeutralScalar(64);
-  if (!Type)
-    return std::unexpected(Type.error());
-  std::vector<Register> Results;
-  Results.reserve(Left.size());
-  for (std::size_t Index = 0; Index < Left.size(); ++Index) {
-    auto Part = buildBinaryBefore(Builder, Anchor, Generic, *Type, Left[Index], Right[Index]);
-    if (!Part)
-      return std::unexpected(Part.error());
-    Results.push_back(*Part);
-  }
-  return Results;
-}
-
-[[nodiscard]] Expected<void> lowerWideBinary(MachineInstr &Instruction, MachineFunction &Function,
-                                             const TargetDescription &Target,
-                                             GenericOpcode Generic) {
-  const MachineOperand *DestinationOperand = Instruction.operandAt(0);
-  const MachineOperand *LeftOperand = Instruction.operandAt(1);
-  const MachineOperand *RightOperand = Instruction.operandAt(2);
-  const Register *Destination =
-      DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
-  const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf<Register>();
-  const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf<Register>();
-  if (Destination == nullptr || Left == nullptr || Right == nullptr)
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "RISC-V wide binary instruction has invalid operands");
-  auto LeftParts = collectWideValueParts(*Left, Function);
-  if (!LeftParts)
-    return std::unexpected(LeftParts.error());
-  auto RightParts = collectWideValueParts(*Right, Function);
-  if (!RightParts)
-    return std::unexpected(RightParts.error());
-  if (LeftParts->Parts.size() != RightParts->Parts.size())
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "RISC-V wide binary operands have different limb counts");
-  auto ResultPlan = wideResultPlan(*Destination, Function, LeftParts->Parts.size());
-  if (!ResultPlan)
-    return std::unexpected(ResultPlan.error());
-
-  MachineIRBuilder Builder(Function, Target);
-  Expected<std::vector<Register>> Results =
-      Generic == GenericOpcode::G_ADD
-          ? emitWideAdd(Builder, Instruction, LeftParts->Parts, RightParts->Parts)
-      : Generic == GenericOpcode::G_SUB
-          ? emitWideSub(Builder, Instruction, LeftParts->Parts, RightParts->Parts)
-          : emitWideBitwise(Builder, Instruction, Generic, LeftParts->Parts, RightParts->Parts);
-  if (!Results)
-    return std::unexpected(Results.error());
-  for (std::size_t Index = 0; Index < Results->size(); ++Index) {
-    auto Replaced = replaceRegisterUses(Function, ResultPlan->Outputs[Index], (*Results)[Index]);
-    if (!Replaced)
-      return std::unexpected(Replaced.error());
-  }
-  if (auto Erased = eraseInstruction(*ResultPlan->Unmerge); !Erased)
-    return std::unexpected(Erased.error());
-  if (ResultPlan->Extension != nullptr) {
-    if (auto Erased = eraseInstruction(*ResultPlan->Extension); !Erased)
-      return std::unexpected(Erased.error());
-  }
-  if (auto Erased = eraseInstruction(Instruction); !Erased)
-    return std::unexpected(Erased.error());
-  LeftParts->Artifacts.insert(LeftParts->Artifacts.end(), RightParts->Artifacts.begin(),
-                              RightParts->Artifacts.end());
-  eraseDeadArtifacts(LeftParts->Artifacts, Function);
-  return {};
-}
-
-[[nodiscard]] bool canLowerNarrowBranchCondition(const MachineInstr &Instruction,
-                                                 const MachineFunction &Function) {
-  if (Instruction.opcode() != opcode(GenericOpcode::G_BRCOND) || Instruction.operandCount() < 2)
-    return false;
-  const MachineOperand *ConditionOperand = Instruction.operandAt(0);
-  const Register *Condition =
-      ConditionOperand == nullptr ? nullptr : ConditionOperand->getIf<Register>();
-  if (Condition == nullptr || !Condition->isVirtual() ||
-      singleRegisterUse(Function, *Condition) != &Instruction)
-    return false;
-  const auto ConditionType = registerType(Function, *Condition);
-  const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(*Condition);
-  const MachineInstr *Truncation = Definition == nullptr ? nullptr : Definition->parent();
-  if (!ConditionType || ConditionType->scalarBitWidth() != 1 || Truncation == nullptr ||
-      Truncation->opcode() != opcode(GenericOpcode::G_TRUNC) || Truncation->operandCount() != 2)
-    return false;
-  return operandType(Function, *Truncation, 1)
-      .transform(
-          [](LowLevelType Type) { return isScalarGPRType(Type) && Type.scalarBitWidth() == 64; })
-      .value_or(false);
-}
-
-[[nodiscard]] Expected<void> lowerNarrowBranchCondition(MachineInstr &Instruction,
-                                                        MachineFunction &Function,
-                                                        const TargetDescription &Target) {
-  if (!canLowerNarrowBranchCondition(Instruction, Function))
-    return makeUnexpected(ErrorCode::Unsupported,
-                          "RISC-V branch condition is not a narrow truncation");
-  const MachineOperand *ConditionOperand = Instruction.operandAt(0);
-  const Register Condition = *ConditionOperand->getIf<Register>();
-  const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(Condition);
-  MachineInstr *Truncation = const_cast<MachineInstr *>(Definition->parent());
-  const Register Source = *Truncation->operandAt(1)->getIf<Register>();
-  auto Type = LowLevelType::makeCategoryNeutralScalar(64);
-  if (!Type)
-    return std::unexpected(Type.error());
-  MachineIRBuilder Builder(Function, Target);
-  auto One = buildIntegerConstantBefore(Builder, Instruction, 64, 1);
-  if (!One)
-    return std::unexpected(One.error());
-  auto Masked = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_AND, *Type, Source, *One);
-  if (!Masked)
-    return std::unexpected(Masked.error());
-  auto Replaced = Instruction.operandAt(0)->setRegister(*Masked);
-  if (!Replaced)
-    return std::unexpected(Replaced.error());
-  if (Function.registerInfo().useEmpty(Condition))
-    return eraseInstruction(*Truncation);
-  return {};
-}
-
-struct NarrowPhiIncoming final {
-  Register Value;
-  MachineBasicBlock *Block = nullptr;
-  MachineInstr *ReusableTruncation = nullptr;
-};
-
-[[nodiscard]] MachineInstr *firstInstructionAfterLeadingCopies(MachineBasicBlock &Block) {
-  MachineInstr *Instruction = Block.firstInstruction();
-  while (Instruction != nullptr && Instruction->opcode() == opcode(GenericOpcode::COPY))
-    Instruction = Instruction->next();
-  return Instruction;
-}
-
-[[nodiscard]] Expected<Register>
-buildIncomingConversion(MachineFunction &Function, const TargetDescription &Target,
-                        MachineBasicBlock &Block, MachineInstr *Anchor, GenericOpcode Generic,
-                        LowLevelType Type, Register Source) {
-  auto Result = Function.registerInfo().createVirtualRegister(Type);
-  if (!Result)
-    return std::unexpected(Result.error());
-  MachineIRBuilder Builder(Function, Target);
-  auto Point =
-      Anchor == nullptr ? Builder.setInsertPoint(Block) : Builder.setInsertPointBefore(*Anchor);
-  if (!Point)
-    return std::unexpected(Point.error());
-  auto Pending = Builder.buildInstr(opcode(Generic));
-  if (!Pending)
-    return std::unexpected(Pending.error());
-  if (auto Added = Pending->addDef(*Result); !Added)
-    return std::unexpected(Added.error());
-  if (auto Added = Pending->addUse(Source); !Added)
-    return std::unexpected(Added.error());
-  auto Built = Pending->build();
-  if (!Built)
-    return std::unexpected(Built.error());
-  return *Result;
-}
-
-[[nodiscard]] Expected<void> lowerNarrowPhi(MachineInstr &Instruction, MachineFunction &Function,
-                                            const TargetDescription &Target) {
-  const std::optional<LowLevelType> OriginalType = resultType(Function, Instruction);
-  const MachineOperand *DestinationOperand = Instruction.operandAt(0);
-  const Register *Destination =
-      DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
-  if (!OriginalType || Destination == nullptr || OriginalType->scalarBitWidth() >= 32 ||
-      Instruction.operandCount() < 3 || (Instruction.operandCount() % 2U) == 0)
-    return makeUnexpected(ErrorCode::VerificationError, "RISC-V narrow PHI has invalid operands");
-  auto LegalType = LowLevelType::makeCategoryNeutralScalar(32);
-  if (!LegalType)
-    return std::unexpected(LegalType.error());
-
-  std::vector<NarrowPhiIncoming> Incoming;
-  Incoming.reserve((Instruction.operandCount() - 1U) / 2U);
-  for (std::size_t Index = 1; Index < Instruction.operandCount(); Index += 2) {
-    const MachineOperand *ValueOperand = Instruction.operandAt(Index);
-    const Register *Value = ValueOperand == nullptr ? nullptr : ValueOperand->getIf<Register>();
-    const MachineOperand *BlockOperand = Instruction.operandAt(Index + 1U);
-    const auto *Block =
-        BlockOperand == nullptr ? nullptr : BlockOperand->getIf<BasicBlockOperand>();
-    if (Value == nullptr || Block == nullptr || Block->Block == nullptr)
-      return makeUnexpected(ErrorCode::VerificationError,
-                            "RISC-V narrow PHI has an invalid incoming edge");
-    MachineInstr *Reusable = nullptr;
-    const MachineOperand *Definition =
-        Value->isVirtual() ? Function.registerInfo().getUniqueVRegDef(*Value) : nullptr;
-    MachineInstr *Def =
-        Definition == nullptr ? nullptr : const_cast<MachineInstr *>(Definition->parent());
-    if (Def != nullptr && Def->opcode() == opcode(GenericOpcode::G_TRUNC) &&
-        Def->operandCount() == 2 && singleRegisterUse(Function, *Value) == &Instruction &&
-        operandType(Function, *Def, 1)
-            .transform([](LowLevelType Type) { return Type.scalarBitWidth() >= 32; })
-            .value_or(false)) {
-      Reusable = Def;
-    }
-    Incoming.push_back(
-        NarrowPhiIncoming{.Value = *Value, .Block = Block->Block, .ReusableTruncation = Reusable});
-  }
-  for (NarrowPhiIncoming &Entry : Incoming) {
-    if (Entry.ReusableTruncation == nullptr)
-      continue;
-    const std::size_t Count = static_cast<std::size_t>(std::ranges::count(
-        Incoming, Entry.ReusableTruncation, &NarrowPhiIncoming::ReusableTruncation));
-    if (Count != 1)
-      Entry.ReusableTruncation = nullptr;
-  }
-  for (NarrowPhiIncoming &Entry : Incoming) {
-    if (Entry.ReusableTruncation == nullptr)
-      continue;
-    MachineBasicBlock *OriginalBlock = Entry.ReusableTruncation->parent();
-    if (OriginalBlock == nullptr)
-      return makeUnexpected(ErrorCode::VerificationError,
-                            "RISC-V narrow PHI input truncation is detached");
-    auto Removed = OriginalBlock->removeInstruction(*Entry.ReusableTruncation);
-    if (!Removed)
-      return std::unexpected(Removed.error());
-  }
-
-  std::vector<std::pair<MachineBasicBlock *, MachineInstr *>> Anchors;
-  for (const NarrowPhiIncoming &Entry : Incoming) {
-    if (std::ranges::find(Anchors, Entry.Block, &decltype(Anchors)::value_type::first) ==
-        Anchors.end()) {
-      Anchors.emplace_back(Entry.Block, firstInstructionAfterLeadingCopies(*Entry.Block));
-    }
-  }
-  std::vector<Register> LegalIncoming;
-  LegalIncoming.reserve(Incoming.size());
-  for (NarrowPhiIncoming &Entry : Incoming) {
-    const auto AnchorPosition =
-        std::ranges::find(Anchors, Entry.Block, &decltype(Anchors)::value_type::first);
-    MachineInstr *Anchor = AnchorPosition->second;
-    if (Entry.ReusableTruncation != nullptr) {
-      auto SetType = Function.registerInfo().setType(Entry.Value, *LegalType);
-      if (!SetType)
-        return std::unexpected(SetType.error());
-      auto Inserted =
-          Anchor == nullptr
-              ? Entry.Block->appendInstruction(*Entry.ReusableTruncation)
-              : Entry.Block->insertInstructionBefore(*Anchor, *Entry.ReusableTruncation);
-      if (!Inserted)
-        return std::unexpected(Inserted.error());
-      LegalIncoming.push_back(Entry.Value);
-      continue;
-    }
-    const auto ValueType = registerType(Function, Entry.Value);
-    if (!ValueType)
-      return makeUnexpected(ErrorCode::VerificationError,
-                            "RISC-V narrow PHI input has no scalar type");
-    const std::uint32_t Width = ValueType->scalarBitWidth();
-    const GenericOpcode Conversion = Width < 32   ? GenericOpcode::G_ANYEXT
-                                     : Width > 32 ? GenericOpcode::G_TRUNC
-                                                  : GenericOpcode::COPY;
-    auto Converted = buildIncomingConversion(Function, Target, *Entry.Block, Anchor, Conversion,
-                                             *LegalType, Entry.Value);
-    if (!Converted)
-      return std::unexpected(Converted.error());
-    LegalIncoming.push_back(*Converted);
-  }
-
-  auto LegalResult = Function.registerInfo().createVirtualRegister(*LegalType);
-  if (!LegalResult)
-    return std::unexpected(LegalResult.error());
-  MachineIRBuilder Builder(Function, Target);
-  auto Point = Builder.setInsertPointBefore(Instruction);
-  if (!Point)
-    return std::unexpected(Point.error());
-  auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_PHI));
-  if (!Pending)
-    return std::unexpected(Pending.error());
-  if (auto Added = Pending->addDef(*LegalResult); !Added)
-    return std::unexpected(Added.error());
-  for (std::size_t Index = 0; Index < Incoming.size(); ++Index) {
-    if (auto Added = Pending->addUse(LegalIncoming[Index]); !Added)
-      return std::unexpected(Added.error());
-    if (auto Added = Pending->addBasicBlock(*Incoming[Index].Block); !Added)
-      return std::unexpected(Added.error());
-  }
-  auto Built = Pending->build();
-  if (!Built)
-    return std::unexpected(Built.error());
-  auto Replaced = replaceRegisterUses(Function, *Destination, *LegalResult);
-  if (!Replaced)
-    return std::unexpected(Replaced.error());
-  return eraseInstruction(Instruction);
-}
-
-[[nodiscard]] Expected<void> lowerWidePhi(MachineInstr &Instruction, MachineFunction &Function,
-                                          const TargetDescription &Target) {
-  const MachineOperand *DestinationOperand = Instruction.operandAt(0);
-  const Register *Destination =
-      DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
-  if (Destination == nullptr || Instruction.operandCount() < 5 ||
-      (Instruction.operandCount() % 2U) == 0)
-    return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide PHI has invalid operands");
-
-  std::vector<WideValueParts> IncomingParts;
-  std::vector<MachineBasicBlock *> IncomingBlocks;
-  for (std::size_t Index = 1; Index < Instruction.operandCount(); Index += 2) {
-    const MachineOperand *ValueOperand = Instruction.operandAt(Index);
-    const Register *Value = ValueOperand == nullptr ? nullptr : ValueOperand->getIf<Register>();
-    const MachineOperand *BlockOperand = Instruction.operandAt(Index + 1U);
-    const auto *Block =
-        BlockOperand == nullptr ? nullptr : BlockOperand->getIf<BasicBlockOperand>();
-    if (Value == nullptr || Block == nullptr || Block->Block == nullptr)
-      return makeUnexpected(ErrorCode::VerificationError,
-                            "RISC-V wide PHI has an invalid incoming edge");
-    auto Parts = collectWideValueParts(*Value, Function);
-    if (!Parts)
-      return std::unexpected(Parts.error());
-    if (!IncomingParts.empty() && Parts->Parts.size() != IncomingParts.front().Parts.size())
-      return makeUnexpected(ErrorCode::VerificationError,
-                            "RISC-V wide PHI incoming values have different limb counts");
-    IncomingParts.push_back(std::move(*Parts));
-    IncomingBlocks.push_back(Block->Block);
-  }
-  const std::size_t PartCount = IncomingParts.front().Parts.size();
-  auto ResultPlan = wideResultPlan(*Destination, Function, PartCount);
-  if (!ResultPlan)
-    return std::unexpected(ResultPlan.error());
-  auto Type = LowLevelType::makeCategoryNeutralScalar(64);
-  if (!Type)
-    return std::unexpected(Type.error());
-  MachineIRBuilder Builder(Function, Target);
-  std::vector<Register> Results;
-  Results.reserve(PartCount);
-  for (std::size_t PartIndex = 0; PartIndex < PartCount; ++PartIndex) {
-    auto Result = Function.registerInfo().createVirtualRegister(*Type);
-    if (!Result)
-      return std::unexpected(Result.error());
-    auto Point = Builder.setInsertPointBefore(Instruction);
-    if (!Point)
-      return std::unexpected(Point.error());
-    auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_PHI));
-    if (!Pending)
-      return std::unexpected(Pending.error());
-    if (auto Added = Pending->addDef(*Result); !Added)
-      return std::unexpected(Added.error());
-    for (std::size_t IncomingIndex = 0; IncomingIndex < IncomingParts.size(); ++IncomingIndex) {
-      if (auto Added = Pending->addUse(IncomingParts[IncomingIndex].Parts[PartIndex]); !Added)
-        return std::unexpected(Added.error());
-      if (auto Added = Pending->addBasicBlock(*IncomingBlocks[IncomingIndex]); !Added)
-        return std::unexpected(Added.error());
-    }
-    auto Built = Pending->build();
-    if (!Built)
-      return std::unexpected(Built.error());
-    Results.push_back(*Result);
-  }
-  for (std::size_t Index = 0; Index < PartCount; ++Index) {
-    auto Replaced = replaceRegisterUses(Function, ResultPlan->Outputs[Index], Results[Index]);
-    if (!Replaced)
-      return std::unexpected(Replaced.error());
-  }
-  if (auto Erased = eraseInstruction(*ResultPlan->Unmerge); !Erased)
-    return std::unexpected(Erased.error());
-  if (ResultPlan->Extension != nullptr) {
-    if (auto Erased = eraseInstruction(*ResultPlan->Extension); !Erased)
-      return std::unexpected(Erased.error());
-  }
-  if (auto Erased = eraseInstruction(Instruction); !Erased)
-    return std::unexpected(Erased.error());
-  std::vector<MachineInstr *> Artifacts;
-  for (WideValueParts &Parts : IncomingParts)
-    Artifacts.insert(Artifacts.end(), Parts.Artifacts.begin(), Parts.Artifacts.end());
-  eraseDeadArtifacts(Artifacts, Function);
-  return {};
-}
-
-[[nodiscard]] bool canFoldWideMergeImmediately(const MachineInstr &Instruction,
-                                               const MachineFunction &Function) {
-  const MachineOperand *LowOperand = Instruction.operandAt(1);
-  const Register *Low = LowOperand == nullptr ? nullptr : LowOperand->getIf<Register>();
-  if (Low == nullptr)
-    return false;
-  const auto LowType = registerType(Function, *Low);
-  if (!LowType)
-    return false;
-  const MachineOperand *DestinationOperand = Instruction.operandAt(0);
-  const Register *Destination =
-      DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
-  if (Destination == nullptr)
-    return false;
-  auto &RegisterInfo = const_cast<MachineRegisterInfo &>(Function.registerInfo());
-  for (MachineOperand &Use : RegisterInfo.useOperands(*Destination)) {
-    MachineInstr *Trunc = Use.parent();
-    if (Trunc == nullptr || Trunc->opcode() != opcode(GenericOpcode::G_TRUNC) ||
-        Trunc->operandAt(1) != &Use)
-      return false;
-    const std::optional<LowLevelType> Result = resultType(Function, *Trunc);
-    if (!Result || Result->scalarBitWidth() > LowType->scalarBitWidth())
-      return false;
-  }
-  return true;
-}
-
-struct RV64ScalarLayout final {
-  std::uint64_t StoreSize = 0;
-  std::uint64_t AllocationSize = 0;
-  std::uint64_t ABIAlignment = 0;
-};
-
-[[nodiscard]] std::optional<RV64ScalarLayout> rv64ScalarLayout(LowLevelType Type) noexcept {
-  if (Type.isPointer()) {
-    return Type.addressSpace() == 0 && Type.scalarBitWidth() == 64
-               ? std::optional<RV64ScalarLayout>{{8, 8, 8}}
-               : std::nullopt;
-  }
-  if ((!isSupportedIntegerType(Type) && !isSupportedFloatingScalar(Type)) ||
-      Type.scalarBitWidth() == 0) {
-    return std::nullopt;
-  }
-  const std::uint64_t StoreBytes = (Type.scalarBitWidth() + 7U) / 8U;
-  const std::uint64_t AllocationBytes = std::bit_ceil(StoreBytes);
-  return RV64ScalarLayout{StoreBytes, AllocationBytes, std::min<std::uint64_t>(AllocationBytes, 8)};
-}
-
-[[nodiscard]] Expected<const MachineMemOperand *>
-createMemoryAccess(MachineFunction &Function, MemoryAccessFlag Access, LowLevelType Type,
-                   std::uint64_t SizeBytes, std::uint64_t AlignmentBytes) {
-  auto Size = MemorySize::fixed(SizeBytes);
-  auto Align = Alignment::create(AlignmentBytes);
-  if (!Size || !Align)
-    return makeUnexpected(ErrorCode::InternalError, "RISC-V memory facts are invalid");
-  MachineMemOperandDesc Description;
-  Description.Accesses = {Access};
-  Description.Size = *Size;
-  Description.MemoryType = Type;
-  Description.Align = *Align;
-  return Function.createMemOperand(std::move(Description));
-}
-
-struct VAArgPlan final {
-  Register Destination;
-  Register ListPointer;
-  LowLevelType DestinationType;
-  LowLevelType PointerType;
-  std::uint64_t Alignment = 0;
-  RV64ScalarLayout ValueLayout;
-};
-
-[[nodiscard]] std::optional<VAArgPlan> vaargPlan(const MachineInstr &Instruction,
-                                                 const MachineFunction &Function) {
-  if (Instruction.opcode() != opcode(GenericOpcode::G_VAARG) || Instruction.operandCount() != 3 ||
-      !Instruction.memOperands().empty()) {
-    return std::nullopt;
-  }
-  const MachineOperand *DestinationOperand = Instruction.operandAt(0);
-  const MachineOperand *ListOperand = Instruction.operandAt(1);
-  const MachineOperand *AlignmentOperand = Instruction.operandAt(2);
-  const Register *Destination =
-      DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
-  const Register *ListPointer = ListOperand == nullptr ? nullptr : ListOperand->getIf<Register>();
-  const std::optional<LowLevelType> DestinationType = resultType(Function, Instruction);
-  const std::optional<LowLevelType> PointerType = operandType(Function, Instruction, 1);
-  const auto AlignmentValue = AlignmentOperand == nullptr
-                                  ? Expected<std::int64_t>(std::unexpected(
-                                        Error(ErrorCode::InvalidArgument, "missing alignment")))
-                                  : signedImmediate(*AlignmentOperand);
-  const std::optional<RV64ScalarLayout> ValueLayout =
-      DestinationType ? rv64ScalarLayout(*DestinationType) : std::nullopt;
-  if (Destination == nullptr || ListPointer == nullptr || !Destination->isVirtual() ||
-      !ListPointer->isVirtual() || !DestinationOperand->isDef() || !ListOperand->isUse() ||
-      !DestinationType || !PointerType || !PointerType->isPointer() ||
-      PointerType->addressSpace() != 0 || PointerType->scalarBitWidth() != 64 || !AlignmentValue ||
-      *AlignmentValue <= 0 || !std::has_single_bit(static_cast<std::uint64_t>(*AlignmentValue)) ||
-      !ValueLayout) {
-    return std::nullopt;
-  }
-
-  return VAArgPlan{*Destination,
-                   *ListPointer,
-                   *DestinationType,
-                   *PointerType,
-                   static_cast<std::uint64_t>(*AlignmentValue),
-                   *ValueLayout};
-}
-
-[[nodiscard]] Expected<void> lowerVAArg(MachineInstr &Instruction, MachineFunction &Function,
-                                        const TargetDescription &Target) {
-  const auto Plan = vaargPlan(Instruction, Function);
-  if (!Plan) {
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "RISC-V va_arg legalization requires a scalar result, p0 list, and "
-                          "power-of-two alignment");
-  }
-
-  auto ListLoadMemory =
-      createMemoryAccess(Function, MemoryAccessFlag::Load, Plan->PointerType, 8, 8);
-  if (!ListLoadMemory)
-    return std::unexpected(ListLoadMemory.error());
-  auto ListStoreMemory =
-      createMemoryAccess(Function, MemoryAccessFlag::Store, Plan->PointerType, 8, 8);
-  if (!ListStoreMemory)
-    return std::unexpected(ListStoreMemory.error());
-
-  auto CurrentList = Function.registerInfo().createVirtualRegister(Plan->PointerType);
-  if (!CurrentList)
-    return std::unexpected(CurrentList.error());
-  MachineIRBuilder Builder(Function, Target);
-  if (auto Point = Builder.setInsertPointBefore(Instruction); !Point)
-    return std::unexpected(Point.error());
-  auto ListLoad = Builder.buildInstr(opcode(GenericOpcode::G_LOAD));
-  if (!ListLoad)
-    return std::unexpected(ListLoad.error());
-  if (auto Added = ListLoad->addDef(*CurrentList); !Added)
-    return std::unexpected(Added.error());
-  if (auto Added = ListLoad->addUse(Plan->ListPointer); !Added)
-    return std::unexpected(Added.error());
-  if (auto Added = ListLoad->addMemOperand(**ListLoadMemory); !Added)
-    return std::unexpected(Added.error());
-  if (auto Built = ListLoad->build(); !Built)
-    return std::unexpected(Built.error());
-
-  Register ValueAddress = *CurrentList;
-  if (Plan->Alignment > 1) {
-    auto AlignmentOffset =
-        buildIntegerConstantBefore(Builder, Instruction, 64, Plan->Alignment - 1U);
-    if (!AlignmentOffset)
-      return std::unexpected(AlignmentOffset.error());
-    auto BiasedAddress = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_PTR_ADD,
-                                           Plan->PointerType, ValueAddress, *AlignmentOffset);
-    if (!BiasedAddress)
-      return std::unexpected(BiasedAddress.error());
-    auto AlignmentMask =
-        buildIntegerConstantBefore(Builder, Instruction, 64, ~(Plan->Alignment - std::uint64_t{1}));
-    if (!AlignmentMask)
-      return std::unexpected(AlignmentMask.error());
-    auto AlignedAddress = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_PTRMASK,
-                                            Plan->PointerType, *BiasedAddress, *AlignmentMask);
-    if (!AlignedAddress)
-      return std::unexpected(AlignedAddress.error());
-    ValueAddress = *AlignedAddress;
-  }
-
-  auto Increment =
-      buildIntegerConstantBefore(Builder, Instruction, 64, Plan->ValueLayout.AllocationSize);
-  if (!Increment)
-    return std::unexpected(Increment.error());
-  auto NextList = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_PTR_ADD,
-                                    Plan->PointerType, ValueAddress, *Increment);
-  if (!NextList)
-    return std::unexpected(NextList.error());
-  auto ListStore = Builder.buildInstr(opcode(GenericOpcode::G_STORE));
-  if (!ListStore)
-    return std::unexpected(ListStore.error());
-  if (auto Added = ListStore->addUse(*NextList); !Added)
-    return std::unexpected(Added.error());
-  if (auto Added = ListStore->addUse(Plan->ListPointer); !Added)
-    return std::unexpected(Added.error());
-  if (auto Added = ListStore->addMemOperand(**ListStoreMemory); !Added)
-    return std::unexpected(Added.error());
-  if (auto Built = ListStore->build(); !Built)
-    return std::unexpected(Built.error());
-
-  if (!Function.registerInfo().useEmpty(Plan->Destination)) {
-    auto ValueLoadMemory =
-        createMemoryAccess(Function, MemoryAccessFlag::Load, Plan->DestinationType,
-                           Plan->ValueLayout.StoreSize, Plan->ValueLayout.ABIAlignment);
-    if (!ValueLoadMemory)
-      return std::unexpected(ValueLoadMemory.error());
-    auto ValueLoad = Builder.buildInstr(opcode(GenericOpcode::G_LOAD));
-    if (!ValueLoad)
-      return std::unexpected(ValueLoad.error());
-    if (auto Added = ValueLoad->addDef(Plan->Destination); !Added)
-      return std::unexpected(Added.error());
-    if (auto Added = ValueLoad->addUse(ValueAddress); !Added)
-      return std::unexpected(Added.error());
-    if (auto Added = ValueLoad->addMemOperand(**ValueLoadMemory); !Added)
-      return std::unexpected(Added.error());
-    if (auto Built = ValueLoad->build(); !Built)
-      return std::unexpected(Built.error());
-  }
-  return eraseInstruction(Instruction);
-}
-
-struct VACopyPlan final {
-  Register Destination;
-  Register Source;
-  LowLevelType PointerType;
-};
-
-[[nodiscard]] std::optional<VACopyPlan> vacopyPlan(const MachineInstr &Instruction,
-                                                   const MachineFunction &Function) {
-  if (Instruction.opcode() != opcode(GenericOpcode::G_INTRINSIC_W_SIDE_EFFECTS) ||
-      Instruction.operandCount() != 3 || !Instruction.memOperands().empty()) {
-    return std::nullopt;
-  }
-  const MachineOperand *IntrinsicIDOperand = Instruction.operandAt(0);
-  const MachineOperand *DestinationOperand = Instruction.operandAt(1);
-  const MachineOperand *SourceOperand = Instruction.operandAt(2);
-  const auto *Intrinsic =
-      IntrinsicIDOperand == nullptr ? nullptr : IntrinsicIDOperand->getIf<IntrinsicOperand>();
-  const Register *Destination =
-      DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
-  const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>();
-  const std::optional<LowLevelType> DestinationType = operandType(Function, Instruction, 1);
-  const std::optional<LowLevelType> SourceType = operandType(Function, Instruction, 2);
-  if (Intrinsic == nullptr || Intrinsic->ID != intrinsicID(GenericIntrinsic::VACopy) ||
-      Destination == nullptr || Source == nullptr || !Destination->isVirtual() ||
-      !Source->isVirtual() || !DestinationOperand->isUse() || !SourceOperand->isUse() ||
-      !DestinationType || !SourceType || *DestinationType != *SourceType ||
-      !DestinationType->isPointer() || DestinationType->addressSpace() != 0 ||
-      DestinationType->scalarBitWidth() != 64) {
-    return std::nullopt;
-  }
-  return VACopyPlan{*Destination, *Source, *DestinationType};
-}
-
-[[nodiscard]] Expected<void> lowerVACopy(MachineInstr &Instruction, MachineFunction &Function,
-                                         const TargetDescription &Target) {
-  const auto Plan = vacopyPlan(Instruction, Function);
-  if (!Plan) {
-    return makeUnexpected(ErrorCode::VerificationError,
-                          "RISC-V va_copy legalization requires two p0 operands");
-  }
-  auto Size = MemorySize::fixed(8);
-  auto Align = Alignment::create(8);
-  if (!Size || !Align)
-    return makeUnexpected(ErrorCode::InternalError, "RISC-V pointer memory facts are invalid");
-
-  MachineMemOperandDesc LoadDescription;
-  LoadDescription.Accesses = {MemoryAccessFlag::Load};
-  LoadDescription.Size = *Size;
-  LoadDescription.MemoryType = Plan->PointerType;
-  LoadDescription.Align = *Align;
-  auto LoadMemory = Function.createMemOperand(std::move(LoadDescription));
-  if (!LoadMemory)
-    return std::unexpected(LoadMemory.error());
-
-  MachineMemOperandDesc StoreDescription;
-  StoreDescription.Accesses = {MemoryAccessFlag::Store};
-  StoreDescription.Size = *Size;
-  StoreDescription.MemoryType = Plan->PointerType;
-  StoreDescription.Align = *Align;
-  auto StoreMemory = Function.createMemOperand(std::move(StoreDescription));
-  if (!StoreMemory)
-    return std::unexpected(StoreMemory.error());
-
-  auto Value = Function.registerInfo().createVirtualRegister(Plan->PointerType);
-  if (!Value)
-    return std::unexpected(Value.error());
-  MachineIRBuilder Builder(Function, Target);
-  if (auto Point = Builder.setInsertPointBefore(Instruction); !Point)
-    return std::unexpected(Point.error());
-
-  auto Load = Builder.buildInstr(opcode(GenericOpcode::G_LOAD));
-  if (!Load)
-    return std::unexpected(Load.error());
-  if (auto Added = Load->addDef(*Value); !Added)
-    return std::unexpected(Added.error());
-  if (auto Added = Load->addUse(Plan->Source); !Added)
-    return std::unexpected(Added.error());
-  if (auto Added = Load->addMemOperand(**LoadMemory); !Added)
-    return std::unexpected(Added.error());
-  if (auto Built = Load->build(); !Built)
-    return std::unexpected(Built.error());
-
-  auto Store = Builder.buildInstr(opcode(GenericOpcode::G_STORE));
-  if (!Store)
-    return std::unexpected(Store.error());
-  if (auto Added = Store->addUse(*Value); !Added)
-    return std::unexpected(Added.error());
-  if (auto Added = Store->addUse(Plan->Destination); !Added)
-    return std::unexpected(Added.error());
-  if (auto Added = Store->addMemOperand(**StoreMemory); !Added)
-    return std::unexpected(Added.error());
-  if (auto Built = Store->build(); !Built)
-    return std::unexpected(Built.error());
-  return eraseInstruction(Instruction);
-}
-
 } // namespace
 
-Expected<LegalizationAction> RISCVLegalizerInfo::classify(const MachineInstr &Instruction,
-                                                          const MachineFunction &Function,
-                                                          const TargetDescription &) const {
-  if (!Subtarget.features().has(RISCVExtension::I)) {
-    return makeUnexpected(ErrorCode::InternalError,
-                          "RISC-V legalizer requires the base I extension");
-  }
-  if (!isGenericOpcode(Instruction.opcode()))
-    return LegalizationAction::Legal;
-
-  const GenericOpcode Generic = static_cast<GenericOpcode>(Instruction.opcode().value());
-  const std::optional<LowLevelType> Type = resultType(Function, Instruction);
-  switch (Generic) {
-  case GenericOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
-    return vacopyPlan(Instruction, Function) ? LegalizationAction::Custom
-                                             : LegalizationAction::Unsupported;
-  case GenericOpcode::G_VAARG:
-    return vaargPlan(Instruction, Function) ? LegalizationAction::Custom
-                                            : LegalizationAction::Unsupported;
-  case GenericOpcode::G_FRAME_INDEX:
-    return Type && Type->isPointer() && Type->addressSpace() == 0 && Type->scalarBitWidth() == 64
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  case GenericOpcode::G_BR:
-    return LegalizationAction::Legal;
-  case GenericOpcode::G_BRCOND: {
-    const std::optional<LowLevelType> Condition = operandType(Function, Instruction, 0);
-    if (!Condition || !isScalarGPRType(*Condition))
-      return LegalizationAction::Unsupported;
-    return canLowerNarrowBranchCondition(Instruction, Function) ? LegalizationAction::Custom
-                                                                : LegalizationAction::Legal;
-  }
-  case GenericOpcode::G_PHI: {
-    if (!Type)
-      return LegalizationAction::Unsupported;
-    for (std::size_t Index = 1; Index < Instruction.operandCount(); Index += 2) {
-      const std::optional<LowLevelType> Incoming = operandType(Function, Instruction, Index);
-      if (!Incoming || *Incoming != *Type)
-        return LegalizationAction::Unsupported;
-    }
-    if (Type->isCategoryNeutralScalar() && Type->scalarBitWidth() > 64)
-      return LegalizationAction::Custom;
-    if (Type->isCategoryNeutralScalar() && Type->scalarBitWidth() < 32)
-      return LegalizationAction::Custom;
-    if (!isSupportedIntegerType(*Type))
-      return LegalizationAction::Unsupported;
-    return LegalizationAction::Legal;
-  }
-  case GenericOpcode::G_IMPLICIT_DEF:
-    return Type && isSupportedIntegerType(*Type) ? LegalizationAction::Legal
-                                                 : LegalizationAction::Unsupported;
-  case GenericOpcode::G_STORE:
-    if (Instruction.operandCount() == 0)
-      return LegalizationAction::Unsupported;
-    if (const std::optional<LowLevelType> ValueType =
-            registerType(Function, *Instruction.operandAt(0));
-        ValueType) {
-      const std::optional<LowLevelType> AddressType = operandType(Function, Instruction, 1);
-      if (!AddressType || !AddressType->isPointer() || AddressType->addressSpace() != 0 ||
-          AddressType->scalarBitWidth() != 64) {
-        return LegalizationAction::Unsupported;
-      }
-      if (Instruction.memOperands().size() == 1 && Instruction.memOperands().front()->isAtomic()) {
-        const std::uint64_t Bytes = ValueType->scalarBitWidth() / 8U;
-        return isSupportedIntegerType(*ValueType) &&
-                       isSupportedAtomicMemoryAccess(Instruction, AtomicOperation::Store, Bytes)
-                   ? LegalizationAction::Legal
-                   : LegalizationAction::Unsupported;
-      }
-      if (isSupportedIntegerType(*ValueType) && isSupportedNonAtomicMemoryAccess(Instruction) &&
-          memoryMatchesType(Instruction, *ValueType)) {
-        return LegalizationAction::Legal;
-      }
-    }
-    return LegalizationAction::Unsupported;
-  case GenericOpcode::G_SEXTLOAD:
-  case GenericOpcode::G_ZEXTLOAD: {
-    const auto Bytes = memoryAccessBytes(Instruction);
-    const std::optional<LowLevelType> AddressType = operandType(Function, Instruction, 1);
-    return Type && Bytes && AddressType && isSupportedIntegerType(*Type) &&
-                   AddressType->isPointer() && AddressType->addressSpace() == 0 &&
-                   AddressType->scalarBitWidth() == 64 &&
-                   *Bytes < ((Type->scalarBitWidth() + 7U) / 8U) &&
-                   (*Bytes == 1 || *Bytes == 2 || *Bytes == 4) &&
-                   isSupportedNonAtomicMemoryAccess(Instruction)
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_FPEXTLOAD: {
-    const auto Bytes = memoryAccessBytes(Instruction);
-    const std::optional<LowLevelType> AddressType = operandType(Function, Instruction, 1);
-    return Type && Bytes && AddressType && Type->isCategoryNeutralScalar() &&
-                   Type->scalarBitWidth() == 64 && AddressType->isPointer() &&
-                   AddressType->addressSpace() == 0 && AddressType->scalarBitWidth() == 64 &&
-                   *Bytes == 4 && isSupportedNonAtomicMemoryAccess(Instruction)
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_FPTRUNCSTORE: {
-    const std::optional<LowLevelType> ValueType = operandType(Function, Instruction, 0);
-    const std::optional<LowLevelType> AddressType = operandType(Function, Instruction, 1);
-    const auto Bytes = memoryAccessBytes(Instruction);
-    return ValueType && AddressType && Bytes && ValueType->isCategoryNeutralScalar() &&
-                   ValueType->scalarBitWidth() == 64 && AddressType->isPointer() &&
-                   AddressType->addressSpace() == 0 && AddressType->scalarBitWidth() == 64 &&
-                   *Bytes == 4 && isSupportedNonAtomicMemoryAccess(Instruction)
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_LOAD:
-    if (Type) {
-      const std::optional<LowLevelType> AddressType = operandType(Function, Instruction, 1);
-      if (!AddressType || !AddressType->isPointer() || AddressType->addressSpace() != 0 ||
-          AddressType->scalarBitWidth() != 64) {
-        return LegalizationAction::Unsupported;
-      }
-      if (Instruction.memOperands().size() == 1 && Instruction.memOperands().front()->isAtomic()) {
-        const std::uint64_t Bytes = Type->scalarBitWidth() / 8U;
-        return isSupportedIntegerType(*Type) &&
-                       isSupportedAtomicMemoryAccess(Instruction, AtomicOperation::Load, Bytes)
-                   ? LegalizationAction::Legal
-                   : LegalizationAction::Unsupported;
-      }
-      if (isSupportedIntegerType(*Type) && isSupportedNonAtomicMemoryAccess(Instruction) &&
-          memoryMatchesType(Instruction, *Type)) {
-        return LegalizationAction::Legal;
-      }
-    }
-    return LegalizationAction::Unsupported;
-  case GenericOpcode::G_FADD:
-  case GenericOpcode::G_FSUB:
-  case GenericOpcode::G_FMUL:
-  case GenericOpcode::G_FDIV: {
-    const std::optional<LowLevelType> Left = operandType(Function, Instruction, 1);
-    const std::optional<LowLevelType> Right = operandType(Function, Instruction, 2);
-    return Type && Left && Right && isSupportedFloatingScalar(*Type) && *Left == *Type &&
-                   *Right == *Type
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_FMA: {
-    const std::optional<LowLevelType> First = operandType(Function, Instruction, 1);
-    const std::optional<LowLevelType> Second = operandType(Function, Instruction, 2);
-    const std::optional<LowLevelType> Third = operandType(Function, Instruction, 3);
-    return Type && First && Second && Third && isSupportedFloatingScalar(*Type) &&
-                   *First == *Type && *Second == *Type && *Third == *Type
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_FNEG:
-  case GenericOpcode::G_FABS:
-  case GenericOpcode::G_FSQRT: {
-    const std::optional<LowLevelType> Source = operandType(Function, Instruction, 1);
-    return Type && Source && isSupportedFloatingScalar(*Type) && *Source == *Type
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_FCOPYSIGN: {
-    const std::optional<LowLevelType> Left = operandType(Function, Instruction, 1);
-    const std::optional<LowLevelType> Right = operandType(Function, Instruction, 2);
-    return Type && Left && Right && isSupportedFloatingScalar(*Type) && *Left == *Type &&
-                   *Right == *Type
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_FCONSTANT: {
-    const MachineOperand *Immediate = Instruction.operandAt(1);
-    const auto *Floating =
-        Immediate == nullptr ? nullptr : Immediate->getIf<FloatingImmediateOperand>();
-    return Type && isSupportedFloatingScalar(*Type) && Floating != nullptr &&
-                   Floating->Bits.bitWidth() == Type->scalarBitWidth()
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_ASSERT_SEXT:
-  case GenericOpcode::G_ASSERT_ZEXT:
-  case GenericOpcode::G_ASSERT_ALIGN: {
-    const std::optional<LowLevelType> Source = operandType(Function, Instruction, 1);
-    return Type && Source && *Type == *Source && isScalarGPRType(*Type) &&
-                   Instruction.operandCount() == 3
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_FREEZE:
-  case GenericOpcode::G_CONSTANT_FOLD_BARRIER: {
-    const std::optional<LowLevelType> Source = operandType(Function, Instruction, 1);
-    return Type && Source && *Type == *Source && isSupportedIntegerType(*Type)
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_PTRTOINT: {
-    const std::optional<LowLevelType> Source = operandType(Function, Instruction, 1);
-    return Type && Source && Type->isCategoryNeutralScalar() && Type->scalarBitWidth() == 64 &&
-                   Source->isPointer() && Source->scalarBitWidth() == 64
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_INTTOPTR: {
-    const std::optional<LowLevelType> Source = operandType(Function, Instruction, 1);
-    return Type && Source && Type->isPointer() && Type->scalarBitWidth() == 64 &&
-                   Source->isCategoryNeutralScalar() && Source->scalarBitWidth() == 64
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_BITCAST: {
-    const std::optional<LowLevelType> Source = operandType(Function, Instruction, 1);
-    return Type && Source && *Type != *Source && Type->scalarBitWidth() == Source->scalarBitWidth()
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_ANYEXT:
-  case GenericOpcode::G_SEXT:
-  case GenericOpcode::G_ZEXT: {
-    const std::optional<LowLevelType> Source = operandType(Function, Instruction, 1);
-    if (!Type || !Source)
-      return LegalizationAction::Unsupported;
-    if (!isScalarIntegerExtension(*Type, *Source)) {
-      const MachineOperand *DestinationOperand = Instruction.operandAt(0);
-      const Register *Destination =
-          DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
-      MachineInstr *OnlyUse =
-          Destination == nullptr ? nullptr : singleUseInstruction(Function, *Destination);
-      return Type->isCategoryNeutralScalar() && Source->isCategoryNeutralScalar() &&
-                     Type->scalarBitWidth() > 64 &&
-                     Source->scalarBitWidth() < Type->scalarBitWidth() && OnlyUse != nullptr &&
-                     OnlyUse->opcode() == opcode(GenericOpcode::G_UNMERGE_VALUES)
-                 ? LegalizationAction::Legal
-                 : LegalizationAction::Unsupported;
-    }
-    const MachineOperand *SourceOperand = Instruction.operandAt(1);
-    const Register *SourceRegister =
-        SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>();
-    const MachineOperand *SourceDefinition =
-        SourceRegister == nullptr || !SourceRegister->isVirtual()
-            ? nullptr
-            : Function.registerInfo().getUniqueVRegDef(*SourceRegister);
-    const MachineInstr *SourceDef =
-        SourceDefinition == nullptr ? nullptr : SourceDefinition->parent();
-    if (SourceDef != nullptr && SourceDef->opcode() == opcode(GenericOpcode::G_CONSTANT))
-      return LegalizationAction::Custom;
-    if (Generic == GenericOpcode::G_ANYEXT &&
-        narrowAnyExtBinaryPlan(Instruction, Function).has_value())
-      return LegalizationAction::Custom;
-    return LegalizationAction::Legal;
-  }
-  case GenericOpcode::G_TRUNC: {
-    const std::optional<LowLevelType> Source = operandType(Function, Instruction, 1);
-    if (Type && Source && Type->isCategoryNeutralScalar() && Source->isCategoryNeutralScalar() &&
-        Type->scalarBitWidth() > 64 && Source->scalarBitWidth() > Type->scalarBitWidth())
-      return LegalizationAction::Legal;
-    return Type && Source && isSupportedIntegerScalar(*Type) && isSupportedIntegerScalar(*Source) &&
-                   Type->scalarBitWidth() != 0 && Source->scalarBitWidth() >= Type->scalarBitWidth()
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_MERGE_VALUES:
-  case GenericOpcode::G_UNMERGE_VALUES: {
-    if (Instruction.operandCount() < 3)
-      return LegalizationAction::Unsupported;
-    const std::size_t SmallTypeIndex = Generic == GenericOpcode::G_MERGE_VALUES ? 1 : 0;
-    const std::size_t BigTypeIndex =
-        Generic == GenericOpcode::G_MERGE_VALUES ? 0 : Instruction.operandCount() - 1U;
-    const std::optional<LowLevelType> SmallType =
-        operandType(Function, Instruction, SmallTypeIndex);
-    const std::optional<LowLevelType> BigType = operandType(Function, Instruction, BigTypeIndex);
-    if (SmallType && BigType && Generic == GenericOpcode::G_MERGE_VALUES &&
-        BigType->scalarBitWidth() > 64 && !canFoldWideMergeImmediately(Instruction, Function))
-      return LegalizationAction::Legal;
-    return SmallType && BigType && SmallType->isCategoryNeutralScalar() &&
-                   BigType->isCategoryNeutralScalar() && SmallType->scalarBitWidth() != 0 &&
-                   BigType->scalarBitWidth() != 0
-               ? LegalizationAction::Custom
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_SEXT_INREG: {
-    const std::optional<LowLevelType> Source = operandType(Function, Instruction, 1);
-    const MachineOperand *WidthOperand = Instruction.operandAt(2);
-    const auto Width =
-        WidthOperand == nullptr
-            ? Expected<std::int64_t>(std::unexpected(
-                  Error(ErrorCode::InvalidArgument, "missing in-register extension width")))
-            : signedImmediate(*WidthOperand);
-    return Type && Source && *Type == *Source && isScalarGPRType(*Type) && Width && *Width > 0 &&
-                   static_cast<std::uint64_t>(*Width) <= Type->scalarBitWidth()
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_ICMP: {
-    const MachineOperand *Predicate = Instruction.operandAt(1);
-    const auto *IntegerPredicate =
-        Predicate == nullptr ? nullptr : Predicate->getIf<PredicateOperand>();
-    const std::optional<LowLevelType> Left = operandType(Function, Instruction, 2);
-    const std::optional<LowLevelType> Right = operandType(Function, Instruction, 3);
-    if (!Type || !isSupportedIntegerScalar(*Type) || IntegerPredicate == nullptr ||
-        IntegerPredicate->integerPredicate() == nullptr || !Left || !Right || *Left != *Right ||
-        !isSupportedIntegerType(*Left)) {
-      return LegalizationAction::Unsupported;
-    }
-    return Type->scalarBitWidth() == 64 && Left->scalarBitWidth() == 64
-               ? LegalizationAction::Legal
-               : LegalizationAction::Custom;
-  }
-  case GenericOpcode::G_FCMP: {
-    const MachineOperand *Predicate = Instruction.operandAt(1);
-    const auto *FloatingPredicate =
-        Predicate == nullptr ? nullptr : Predicate->getIf<PredicateOperand>();
-    const std::optional<LowLevelType> Left = operandType(Function, Instruction, 2);
-    const std::optional<LowLevelType> Right = operandType(Function, Instruction, 3);
-    return Type && isI1Type(*Type) && FloatingPredicate != nullptr &&
-                   FloatingPredicate->floatingPredicate() != nullptr && Left && Right &&
-                   *Left == *Right && isSupportedFloatingScalar(*Left)
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_FPEXT:
-  case GenericOpcode::G_FPTRUNC: {
-    const std::optional<LowLevelType> Source = operandType(Function, Instruction, 1);
-    const bool IsExtension = Generic == GenericOpcode::G_FPEXT;
-    return Type && Source && isSupportedFloatingScalar(*Type) &&
-                   isSupportedFloatingScalar(*Source) &&
-                   ((IsExtension && Source->scalarBitWidth() == 32 &&
-                     Type->scalarBitWidth() == 64) ||
-                    (!IsExtension && Source->scalarBitWidth() == 64 &&
-                     Type->scalarBitWidth() == 32))
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_FPTOSI:
-  case GenericOpcode::G_FPTOUI: {
-    const std::optional<LowLevelType> Source = operandType(Function, Instruction, 1);
-    return Type && Source && isSupportedIntegerType(*Type) && isSupportedFloatingScalar(*Source)
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_SITOFP:
-  case GenericOpcode::G_UITOFP: {
-    const std::optional<LowLevelType> Source = operandType(Function, Instruction, 1);
-    return Type && Source && isSupportedFloatingScalar(*Type) && isSupportedIntegerType(*Source)
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_FMINNUM:
-  case GenericOpcode::G_FMAXNUM: {
-    const std::optional<LowLevelType> Left = operandType(Function, Instruction, 1);
-    const std::optional<LowLevelType> Right = operandType(Function, Instruction, 2);
-    return Type && Left && Right && isSupportedFloatingScalar(*Type) && *Left == *Type &&
-                   *Right == *Type
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_SELECT: {
-    const std::optional<LowLevelType> Condition = operandType(Function, Instruction, 1);
-    const std::optional<LowLevelType> TrueValue = operandType(Function, Instruction, 2);
-    const std::optional<LowLevelType> FalseValue = operandType(Function, Instruction, 3);
-    return Type && Condition && TrueValue && FalseValue && isScalarGPRType(*Condition) &&
-                   *TrueValue == *Type && *FalseValue == *Type && isSupportedIntegerType(*Type)
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_PTRMASK: {
-    const std::optional<LowLevelType> Pointer = operandType(Function, Instruction, 1);
-    const std::optional<LowLevelType> Mask = operandType(Function, Instruction, 2);
-    return Type && Pointer && Mask && Type->isPointer() && Type->addressSpace() == 0 &&
-                   Type->scalarBitWidth() == 64 && *Pointer == *Type &&
-                   isSupportedIntegerScalar(*Mask) && Mask->scalarBitWidth() == 64
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_ABS: {
-    const std::optional<LowLevelType> Source = operandType(Function, Instruction, 1);
-    return Type && Source && *Source == *Type && isSupportedIntegerScalar(*Type)
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_SDIVREM:
-  case GenericOpcode::G_UDIVREM: {
-    const std::optional<LowLevelType> Remainder = operandType(Function, Instruction, 1);
-    const std::optional<LowLevelType> Left = operandType(Function, Instruction, 2);
-    const std::optional<LowLevelType> Right = operandType(Function, Instruction, 3);
-    return Type && Remainder && Left && Right && isSupportedIntegerScalar(*Type) &&
-                   *Remainder == *Type && *Left == *Type && *Right == *Type
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_SMULH:
-  case GenericOpcode::G_UMULH: {
-    const std::optional<LowLevelType> Left = operandType(Function, Instruction, 1);
-    const std::optional<LowLevelType> Right = operandType(Function, Instruction, 2);
-    return Type && Left && Right && isNativeIntegerScalar(*Type) && *Left == *Type &&
-                   *Right == *Type
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_SMIN:
-  case GenericOpcode::G_SMAX:
-  case GenericOpcode::G_UMIN:
-  case GenericOpcode::G_UMAX: {
-    const std::optional<LowLevelType> Left = operandType(Function, Instruction, 1);
-    const std::optional<LowLevelType> Right = operandType(Function, Instruction, 2);
-    return Type && Left && Right && isSupportedIntegerScalar(*Type) && *Left == *Type &&
-                   *Right == *Type
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_UADDO:
-  case GenericOpcode::G_USUBO:
-  case GenericOpcode::G_SADDO:
-  case GenericOpcode::G_SSUBO:
-  case GenericOpcode::G_UMULO:
-  case GenericOpcode::G_SMULO: {
-    const std::optional<LowLevelType> Overflow = operandType(Function, Instruction, 1);
-    const std::optional<LowLevelType> Left = operandType(Function, Instruction, 2);
-    const std::optional<LowLevelType> Right = operandType(Function, Instruction, 3);
-    return Type && Overflow && Left && Right && isNativeIntegerScalar(*Type) &&
-                   isI1Type(*Overflow) && *Left == *Type && *Right == *Type
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_UADDE:
-  case GenericOpcode::G_USUBE:
-  case GenericOpcode::G_SADDE:
-  case GenericOpcode::G_SSUBE: {
-    const std::optional<LowLevelType> CarryOut = operandType(Function, Instruction, 1);
-    const std::optional<LowLevelType> Left = operandType(Function, Instruction, 2);
-    const std::optional<LowLevelType> Right = operandType(Function, Instruction, 3);
-    const std::optional<LowLevelType> CarryIn = operandType(Function, Instruction, 4);
-    return Type && CarryOut && Left && Right && CarryIn && isNativeIntegerScalar(*Type) &&
-                   isI1Type(*CarryOut) && isI1Type(*CarryIn) && *Left == *Type && *Right == *Type
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_ATOMICRMW_XCHG:
-  case GenericOpcode::G_ATOMICRMW_ADD:
-  case GenericOpcode::G_ATOMICRMW_SUB:
-  case GenericOpcode::G_ATOMICRMW_AND:
-  case GenericOpcode::G_ATOMICRMW_OR:
-  case GenericOpcode::G_ATOMICRMW_XOR:
-  case GenericOpcode::G_ATOMICRMW_MAX:
-  case GenericOpcode::G_ATOMICRMW_MIN:
-  case GenericOpcode::G_ATOMICRMW_UMAX:
-  case GenericOpcode::G_ATOMICRMW_UMIN: {
-    const std::optional<LowLevelType> Address = operandType(Function, Instruction, 1);
-    const std::optional<LowLevelType> Value = operandType(Function, Instruction, 2);
-    const std::uint64_t Bytes = Type ? Type->scalarBitWidth() / 8U : 0;
-    return Type && Address && Value && isNativeIntegerScalar(*Type) && *Value == *Type &&
-                   Address->isPointer() && Address->addressSpace() == 0 &&
-                   Address->scalarBitWidth() == 64 &&
-                   isSupportedAtomicMemoryAccess(Instruction, AtomicOperation::ReadModifyWrite,
-                                                 Bytes)
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_ATOMIC_CMPXCHG: {
-    const std::optional<LowLevelType> Address = operandType(Function, Instruction, 1);
-    const std::optional<LowLevelType> Compare = operandType(Function, Instruction, 2);
-    const std::optional<LowLevelType> NewValue = operandType(Function, Instruction, 3);
-    if (!Type || !Address || !Compare || !NewValue || *Compare != *Type || *NewValue != *Type ||
-        !Type->isCategoryNeutralScalar() ||
-        (Type->scalarBitWidth() != 8 && Type->scalarBitWidth() != 16 &&
-         Type->scalarBitWidth() != 32 && Type->scalarBitWidth() != 64) ||
-        !Address->isPointer() || Address->addressSpace() != 0 || Address->scalarBitWidth() != 64 ||
-        !isSupportedAtomicContainerAccess(Instruction, AtomicOperation::CompareExchange,
-                                          (Type->scalarBitWidth() + 7U) / 8U)) {
-      return LegalizationAction::Unsupported;
-    }
-    return Type->scalarBitWidth() == 64 ? LegalizationAction::Legal : LegalizationAction::Custom;
-  }
-  case GenericOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
-    const std::optional<LowLevelType> Success = operandType(Function, Instruction, 1);
-    const std::optional<LowLevelType> Address = operandType(Function, Instruction, 2);
-    const std::optional<LowLevelType> Compare = operandType(Function, Instruction, 3);
-    const std::optional<LowLevelType> NewValue = operandType(Function, Instruction, 4);
-    return Type && Success && Address && Compare && NewValue && isI1Type(*Success) &&
-                   *Compare == *Type && *NewValue == *Type && Type->isCategoryNeutralScalar() &&
-                   (Type->scalarBitWidth() == 8 || Type->scalarBitWidth() == 16 ||
-                    Type->scalarBitWidth() == 32 || Type->scalarBitWidth() == 64) &&
-                   Address->isPointer() && Address->addressSpace() == 0 &&
-                   Address->scalarBitWidth() == 64 &&
-                   isSupportedAtomicContainerAccess(Instruction, AtomicOperation::CompareExchange,
-                                                    (Type->scalarBitWidth() + 7U) / 8U)
-               ? LegalizationAction::Custom
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_FENCE: {
-    const MachineOperand *OrderingOperand = Instruction.operandAt(0);
-    const MachineOperand *ScopeOperand = Instruction.operandAt(1);
-    const auto Ordering = OrderingOperand == nullptr
-                              ? Expected<std::int64_t>(std::unexpected(
-                                    Error(ErrorCode::InvalidArgument, "missing fence ordering")))
-                              : signedImmediate(*OrderingOperand);
-    const auto Scope = ScopeOperand == nullptr
-                           ? Expected<std::int64_t>(std::unexpected(
-                                 Error(ErrorCode::InvalidArgument, "missing fence scope")))
-                           : signedImmediate(*ScopeOperand);
-    return Ordering && Scope && *Ordering >= 4 && *Ordering <= 7 && *Scope == 0
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_CONSTANT_POOL: {
-    const MachineOperand *PoolOperand = Instruction.operandAt(1);
-    const auto *Pool = PoolOperand == nullptr ? nullptr : PoolOperand->getIf<ConstantPoolOperand>();
-    return Type && Type->isPointer() && Type->addressSpace() == 0 && Type->scalarBitWidth() == 64 &&
-                   Pool != nullptr && Function.constantPool().entry(Pool->Index) != nullptr
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_GLOBAL_VALUE: {
-    const MachineOperand *GlobalOperand = Instruction.operandAt(1);
-    const auto *Global =
-        GlobalOperand == nullptr ? nullptr : GlobalOperand->getIf<GlobalSymbolOperand>();
-    return Type && Type->isPointer() && Type->addressSpace() == 0 && Type->scalarBitWidth() == 64 &&
-                   Global != nullptr && !Global->Name.empty()
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_BRINDIRECT:
-    return operandType(Function, Instruction, 0)
-                   .transform([](LowLevelType Value) {
-                     return Value.isPointer() && Value.addressSpace() == 0 &&
-                            Value.scalarBitWidth() == 64;
-                   })
-                   .value_or(false)
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  case GenericOpcode::G_TRAP:
-  case GenericOpcode::G_DEBUGTRAP:
-    return Instruction.operandCount() == 0 ? LegalizationAction::Legal
-                                           : LegalizationAction::Unsupported;
-  case GenericOpcode::G_UBSANTRAP:
-    return Instruction.operandCount() == 1 ? LegalizationAction::Legal
-                                           : LegalizationAction::Unsupported;
-  case GenericOpcode::G_CONSTANT: {
-    const MachineOperand *Immediate = Instruction.operandAt(1);
-    if (!Type || Immediate == nullptr ||
-        (Immediate->kind() != MachineOperand::Kind::Immediate &&
-         Immediate->kind() != MachineOperand::Kind::IntegerImmediate))
-      return LegalizationAction::Unsupported;
-    if (Type->scalarBitWidth() > 64)
-      return wideConstantUnmergePlan(Instruction, Function).has_value()
-                 ? LegalizationAction::Custom
-                 : LegalizationAction::Unsupported;
-    if (Type->isCategoryNeutralScalar() && Type->scalarBitWidth() == 64) {
-      auto Value = signedImmediate(*Immediate);
-      if (Value && shouldUseRV64IntegerConstantPool(*Value))
-        return LegalizationAction::Custom;
-    }
-    return isSupportedIntegerType(*Type) ? LegalizationAction::Legal
-                                         : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_ADD:
-  case GenericOpcode::G_SUB:
-  case GenericOpcode::G_MUL:
-  case GenericOpcode::G_SDIV:
-  case GenericOpcode::G_UDIV:
-  case GenericOpcode::G_SREM:
-  case GenericOpcode::G_UREM:
-  case GenericOpcode::G_AND:
-  case GenericOpcode::G_OR:
-  case GenericOpcode::G_XOR:
-  case GenericOpcode::G_SHL:
-  case GenericOpcode::G_LSHR:
-  case GenericOpcode::G_ASHR: {
-    const std::optional<LowLevelType> Left = operandType(Function, Instruction, 1);
-    const std::optional<LowLevelType> Right = operandType(Function, Instruction, 2);
-    if (!Type || !Left || !Right || *Left != *Type || *Right != *Type ||
-        !Type->isCategoryNeutralScalar())
-      return LegalizationAction::Unsupported;
-    if (Type->scalarBitWidth() > 64) {
-      return Generic == GenericOpcode::G_ADD || Generic == GenericOpcode::G_SUB ||
-                     Generic == GenericOpcode::G_AND || Generic == GenericOpcode::G_OR ||
-                     Generic == GenericOpcode::G_XOR || Generic == GenericOpcode::G_SHL
-                 ? LegalizationAction::Custom
-                 : LegalizationAction::Unsupported;
-    }
-    if (Generic == GenericOpcode::G_MUL && !Subtarget.features().has(RISCVExtension::M)) {
-      if (Type->scalarBitWidth() == 64)
-        return LegalizationAction::Libcall;
-      const MachineOperand *DestinationOperand = Instruction.operandAt(0);
-      const Register *Destination =
-          DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
-      MachineInstr *Extension =
-          Destination == nullptr ? nullptr : singleRegisterUse(Function, *Destination);
-      const std::optional<LowLevelType> ExtensionType =
-          Extension == nullptr ? std::nullopt : resultType(Function, *Extension);
-      if (Extension != nullptr && Extension->opcode() == opcode(GenericOpcode::G_ANYEXT) &&
-          ExtensionType && ExtensionType->scalarBitWidth() == 64) {
-        return LegalizationAction::Legal;
-      }
-      return LegalizationAction::Custom;
-    }
-    return isSupportedIntegerScalar(*Type) ? LegalizationAction::Legal
-                                           : LegalizationAction::Unsupported;
-  }
-  case GenericOpcode::G_PTR_ADD: {
-    const std::optional<LowLevelType> Pointer = operandType(Function, Instruction, 1);
-    const std::optional<LowLevelType> Offset = operandType(Function, Instruction, 2);
-    return Type && Pointer && Offset && Type->isPointer() && Type->addressSpace() == 0 &&
-                   Type->scalarBitWidth() == 64 && *Pointer == *Type &&
-                   isSupportedIntegerScalar(*Offset) && Offset->scalarBitWidth() == 64
-               ? LegalizationAction::Legal
-               : LegalizationAction::Unsupported;
-  }
-  default:
-    break;
-  }
-
-  // Keep the unsupported policy mechanically synchronized with the complete
-  // target-independent opcode inventory. Supported cases return above.
-  switch (Generic) {
-#define RCC_CORE_OPCODE(Name)
-#define RCC_GENERIC_OPCODE(Name) case GenericOpcode::Name:
-#include "rcc/CodeGen/GenericOpcode.def"
-#undef RCC_GENERIC_OPCODE
-#undef RCC_CORE_OPCODE
-    return LegalizationAction::Unsupported;
-  case GenericOpcode::Invalid:
-  case GenericOpcode::CORE_PSEUDO_END:
-  case GenericOpcode::G_GENERIC_END:
-  default:
-    break;
-  }
-  return makeUnexpected(ErrorCode::InternalError,
-                        "RISC-V legalizer received an invalid generic opcode value");
-}
-
-Expected<void> RISCVLegalizerInfo::lower(MachineInstr &Instruction, MachineFunction &Function,
-                                         const TargetDescription &Target,
-                                         LegalizationAction Action) const {
-  if (Action == LegalizationAction::Custom &&
-      Instruction.opcode() == opcode(GenericOpcode::G_INTRINSIC_W_SIDE_EFFECTS)) {
-    return lowerVACopy(Instruction, Function, Target);
-  }
-  if (Action == LegalizationAction::Custom &&
-      Instruction.opcode() == opcode(GenericOpcode::G_VAARG)) {
-    return lowerVAArg(Instruction, Function, Target);
-  }
-  if (Instruction.opcode() == opcode(GenericOpcode::G_MUL) &&
-      !Subtarget.features().has(RISCVExtension::M)) {
-    if (Action == LegalizationAction::Custom)
-      return widenNarrowMultiply(Instruction, Function, Target);
-    if (Action == LegalizationAction::Libcall)
-      return lowerMultiplyLibcall(Instruction, Function, Target,
-                                  Subtarget.features().has(RISCVExtension::D)
-                                      ? CallingConvention::LP64D
-                                      : CallingConvention::LP64,
-                                  *Subtarget.getCallLowering());
-  }
-  if (Action == LegalizationAction::Custom &&
-      resultType(Function, Instruction)
-          .transform([](LowLevelType Type) { return Type.scalarBitWidth() > 64; })
-          .value_or(false)) {
-    const Opcode Value = Instruction.opcode();
-    if (Value == opcode(GenericOpcode::G_ADD) || Value == opcode(GenericOpcode::G_SUB) ||
-        Value == opcode(GenericOpcode::G_AND) || Value == opcode(GenericOpcode::G_OR) ||
-        Value == opcode(GenericOpcode::G_XOR)) {
-      return lowerWideBinary(Instruction, Function, Target,
-                             static_cast<GenericOpcode>(Value.value()));
-    }
-    if (Value == opcode(GenericOpcode::G_SHL))
-      return lowerWideShiftLeft(Instruction, Function, Target);
-    if (Value == opcode(GenericOpcode::G_PHI))
-      return lowerWidePhi(Instruction, Function, Target);
-  }
-  if (Action == LegalizationAction::Custom &&
-      Instruction.opcode() == opcode(GenericOpcode::G_MERGE_VALUES))
-    return lowerMergeValues(Instruction, Function, Target);
-  if (Action == LegalizationAction::Custom &&
-      Instruction.opcode() == opcode(GenericOpcode::G_UNMERGE_VALUES))
-    return lowerUnmergeValues(Instruction, Function, Target);
-  if (Action == LegalizationAction::Custom &&
-      Instruction.opcode() == opcode(GenericOpcode::G_BRCOND))
-    return lowerNarrowBranchCondition(Instruction, Function, Target);
-  if (Action == LegalizationAction::Custom && Instruction.opcode() == opcode(GenericOpcode::G_PHI))
-    return lowerNarrowPhi(Instruction, Function, Target);
-  if (Action == LegalizationAction::Custom && Instruction.opcode() == opcode(GenericOpcode::G_ICMP))
+RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &Subtarget) {
+  const LowLevelType S8 = *LowLevelType::makeCategoryNeutralScalar(8);
+  const LowLevelType S16 = *LowLevelType::makeCategoryNeutralScalar(16);
+  const LowLevelType S32 = *LowLevelType::makeCategoryNeutralScalar(32);
+  const LowLevelType S64 = *LowLevelType::makeCategoryNeutralScalar(64);
+  const LowLevelType S128 = *LowLevelType::makeCategoryNeutralScalar(128);
+  const LowLevelType P0 = *LowLevelType::makePointer(0, 64);
+  const bool HasM = Subtarget.features().has(RISCVExtension::M);
+  const bool HasA = Subtarget.features().has(RISCVExtension::A);
+  const bool HasF = Subtarget.features().has(RISCVExtension::F);
+  const bool HasD = Subtarget.features().has(RISCVExtension::D);
+
+  getActionDefinitionsBuilder({opcode(GenericOpcode::G_ASSERT_SEXT),
+                               opcode(GenericOpcode::G_ASSERT_ZEXT),
+                               opcode(GenericOpcode::G_ASSERT_ALIGN)})
+      .alwaysLegal();
+
+  getActionDefinitionsBuilder({opcode(GenericOpcode::G_ADD), opcode(GenericOpcode::G_SUB)})
+      .legalFor({S64})
+      .customFor({S32})
+      .widenScalarToNextPow2(0)
+      .clampScalar(0, S64, S64);
+
+  getActionDefinitionsBuilder(
+      {opcode(GenericOpcode::G_AND), opcode(GenericOpcode::G_OR), opcode(GenericOpcode::G_XOR)})
+      .legalFor({S64})
+      .widenScalarToNextPow2(0)
+      .clampScalar(0, S64, S64);
+
+  getActionDefinitionsBuilder({opcode(GenericOpcode::G_UADDE), opcode(GenericOpcode::G_UADDO),
+                               opcode(GenericOpcode::G_USUBE), opcode(GenericOpcode::G_USUBO),
+                               opcode(GenericOpcode::G_READ_REGISTER),
+                               opcode(GenericOpcode::G_WRITE_REGISTER)})
+      .lower();
+  getActionDefinitionsBuilder({opcode(GenericOpcode::G_SADDE), opcode(GenericOpcode::G_SADDO),
+                               opcode(GenericOpcode::G_SSUBE), opcode(GenericOpcode::G_SSUBO)})
+      .minScalar(0, S64)
+      .lower();
+  getActionDefinitionsBuilder({opcode(GenericOpcode::G_UADDSAT), opcode(GenericOpcode::G_SADDSAT),
+                               opcode(GenericOpcode::G_USUBSAT), opcode(GenericOpcode::G_SSUBSAT),
+                               opcode(GenericOpcode::G_SSHLSAT), opcode(GenericOpcode::G_USHLSAT)})
+      .lower();
+
+  getActionDefinitionsBuilder(
+      {opcode(GenericOpcode::G_SHL), opcode(GenericOpcode::G_ASHR), opcode(GenericOpcode::G_LSHR)})
+      .legalFor({{S64, S64}})
+      .customFor({{S32, S32}})
+      .widenScalarToNextPow2(0)
+      .clampScalar(1, S64, S64)
+      .clampScalar(0, S64, S64);
+
+  getActionDefinitionsBuilder({opcode(GenericOpcode::G_ZEXT), opcode(GenericOpcode::G_SEXT),
+                               opcode(GenericOpcode::G_ANYEXT)})
+      .legalFor({{S32, S16}, {S64, S16}, {S64, S32}})
+      .maxScalar(0, S64);
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_TRUNC)).alwaysLegal();
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_SEXT_INREG))
+      .customFor({S64})
+      .clampScalar(0, S64, S64)
+      .lower();
+
+  for (const GenericOpcode Generic :
+       {GenericOpcode::G_MERGE_VALUES, GenericOpcode::G_UNMERGE_VALUES}) {
+    const std::size_t BigTypeIndex = Generic == GenericOpcode::G_MERGE_VALUES ? 0 : 1;
+    const std::size_t LittleTypeIndex = Generic == GenericOpcode::G_MERGE_VALUES ? 1 : 0;
+    getActionDefinitionsBuilder(opcode(Generic))
+        .widenScalarToNextPow2(LittleTypeIndex, 64)
+        .widenScalarToNextPow2(BigTypeIndex, 64)
+        .clampScalar(LittleTypeIndex, S64, S64)
+        .clampScalar(BigTypeIndex, S64, S64);
+  }
+  getActionDefinitionsBuilder({opcode(GenericOpcode::G_FSHL), opcode(GenericOpcode::G_FSHR)})
+      .lower();
+  getActionDefinitionsBuilder({opcode(GenericOpcode::G_ROTR), opcode(GenericOpcode::G_ROTL)})
+      .lower();
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_BITREVERSE)).maxScalar(0, S64).lower();
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_BSWAP)).maxScalar(0, S64).lower();
+
+  getActionDefinitionsBuilder(
+      {opcode(GenericOpcode::G_CTLZ), opcode(GenericOpcode::G_CTTZ), opcode(GenericOpcode::G_CTLS)})
+      .maxScalar(0, S64)
+      .scalarSameSizeAs(1, 0)
+      .lower();
+  getActionDefinitionsBuilder(
+      {opcode(GenericOpcode::G_CTLZ_ZERO_POISON), opcode(GenericOpcode::G_CTTZ_ZERO_POISON)})
+      .maxScalar(0, S64)
+      .scalarSameSizeAs(1, 0)
+      .lower();
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_CTPOP))
+      .widenScalarToNextPow2(0, 8)
+      .clampScalar(0, S8, S64)
+      .scalarSameSizeAs(1, 0)
+      .lower();
+
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_CONSTANT))
+      .legalFor({P0})
+      .customFor({S64})
+      .widenScalarToNextPow2(0)
+      .clampScalar(0, S64, S64);
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_FREEZE))
+      .legalFor({S16, S32, S64, P0})
+      .widenScalarToNextPow2(0)
+      .clampScalar(0, S16, S64);
+  getActionDefinitionsBuilder(
+      {opcode(GenericOpcode::G_IMPLICIT_DEF), opcode(GenericOpcode::G_CONSTANT_FOLD_BARRIER)})
+      .legalFor({S32, S64, P0})
+      .widenScalarToNextPow2(0)
+      .clampScalar(0, S32, S64);
+
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_ICMP))
+      .legalFor({{S64, S64}, {S64, P0}})
+      .customIf([](const LegalizationQuery &Query) {
+        return Query.Types.size() == 2 && Query.Types[0].isCategoryNeutralScalar() &&
+               (Query.Types[1].isCategoryNeutralScalar() || Query.Types[1].isPointer());
+      });
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_SELECT))
+      .legalFor({{S32, S64}, {S64, S64}, {P0, S64}})
+      .widenScalarToNextPow2(0)
+      .clampScalar(0, S32, S64)
+      .clampScalar(1, S64, S64);
+
+  auto &LoadRules = getActionDefinitionsBuilder(opcode(GenericOpcode::G_LOAD));
+  auto &StoreRules = getActionDefinitionsBuilder(opcode(GenericOpcode::G_STORE));
+  auto &ExtLoadRules = getActionDefinitionsBuilder(
+      {opcode(GenericOpcode::G_SEXTLOAD), opcode(GenericOpcode::G_ZEXTLOAD)});
+  const std::uint64_t Align8 =
+      Subtarget.features().has(RISCVSubtargetOption::UnalignedScalarMemory) ? 8 : 8;
+  const std::uint64_t Align16 =
+      Subtarget.features().has(RISCVSubtargetOption::UnalignedScalarMemory) ? 8 : 16;
+  const std::uint64_t Align32 =
+      Subtarget.features().has(RISCVSubtargetOption::UnalignedScalarMemory) ? 8 : 32;
+  const std::uint64_t Align64 =
+      Subtarget.features().has(RISCVSubtargetOption::UnalignedScalarMemory) ? 8 : 64;
+  LoadRules
+      .legalForTypesWithMemDesc({{S16, P0, S8, Align8},
+                                 {S32, P0, S8, Align8},
+                                 {S16, P0, S16, Align16},
+                                 {S32, P0, S16, Align16},
+                                 {S32, P0, S32, Align32},
+                                 {S64, P0, S8, Align8},
+                                 {S64, P0, S16, Align16},
+                                 {S64, P0, S32, Align32},
+                                 {S64, P0, S64, Align64},
+                                 {P0, P0, S64, Align64}})
+      .widenScalarToNextPow2(0, 8)
+      .lowerIfMemSizeNotByteSizePow2()
+      .clampScalar(0, S16, S64)
+      .lower();
+  StoreRules
+      .legalForTypesWithMemDesc({{S16, P0, S8, Align8},
+                                 {S32, P0, S8, Align8},
+                                 {S16, P0, S16, Align16},
+                                 {S32, P0, S16, Align16},
+                                 {S32, P0, S32, Align32},
+                                 {S64, P0, S8, Align8},
+                                 {S64, P0, S16, Align16},
+                                 {S64, P0, S32, Align32},
+                                 {S64, P0, S64, Align64},
+                                 {P0, P0, S64, Align64}})
+      .clampScalar(0, S16, S64)
+      .lowerIfMemSizeNotByteSizePow2()
+      .lower();
+  ExtLoadRules
+      .legalForTypesWithMemDesc(
+          {{S64, P0, S8, Align8}, {S64, P0, S16, Align16}, {S64, P0, S32, Align32}})
+      .widenScalarToNextPow2(0)
+      .clampScalar(0, S64, S64)
+      .lower();
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_FPEXTLOAD))
+      .legalForTypesWithMemDesc({{S64, P0, S32, Align32}})
+      .lower();
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_FPTRUNCSTORE))
+      .legalForTypesWithMemDesc({{S64, P0, S32, Align32}})
+      .lower();
+
+  getActionDefinitionsBuilder({opcode(GenericOpcode::G_PTR_ADD), opcode(GenericOpcode::G_PTRMASK)})
+      .legalFor({{P0, S64}});
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_PTRTOINT))
+      .legalFor({{S64, P0}})
+      .clampScalar(0, S64, S64);
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_INTTOPTR))
+      .legalFor({{P0, S64}})
+      .clampScalar(1, S64, S64);
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_BR)).alwaysLegal();
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_BRCOND)).legalFor({S64}).minScalar(0, S64);
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_BRJT)).customFor({{P0, S64}});
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_BRINDIRECT)).legalFor({P0});
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_PHI))
+      .legalFor({P0, S32, S64})
+      .widenScalarToNextPow2(0)
+      .clampScalar(0, S32, S64);
+  getActionDefinitionsBuilder(
+      {opcode(GenericOpcode::G_GLOBAL_VALUE), opcode(GenericOpcode::G_JUMP_TABLE),
+       opcode(GenericOpcode::G_CONSTANT_POOL), opcode(GenericOpcode::G_FRAME_INDEX)})
+      .legalFor({P0});
+
+  if (HasM) {
+    getActionDefinitionsBuilder(opcode(GenericOpcode::G_MUL))
+        .legalFor({S64})
+        .widenScalarToNextPow2(0)
+        .clampScalar(0, S64, S64);
+    getActionDefinitionsBuilder({opcode(GenericOpcode::G_SMULH), opcode(GenericOpcode::G_UMULH)})
+        .legalFor({S64})
+        .lower();
+    getActionDefinitionsBuilder({opcode(GenericOpcode::G_SMULO), opcode(GenericOpcode::G_UMULO)})
+        .minScalar(0, S64)
+        .lower();
+  } else {
+    getActionDefinitionsBuilder(opcode(GenericOpcode::G_MUL))
+        .libcallFor({S64, S128})
+        .widenScalarToNextPow2(0)
+        .clampScalar(0, S64, S128);
+    getActionDefinitionsBuilder({opcode(GenericOpcode::G_SMULH), opcode(GenericOpcode::G_UMULH)})
+        .lowerFor({S64});
+    getActionDefinitionsBuilder({opcode(GenericOpcode::G_SMULO), opcode(GenericOpcode::G_UMULO)})
+        .minScalar(0, S64)
+        .lower();
+  }
+  if (HasM) {
+    getActionDefinitionsBuilder({opcode(GenericOpcode::G_SDIV), opcode(GenericOpcode::G_UDIV),
+                                 opcode(GenericOpcode::G_UREM)})
+        .legalFor({S64})
+        .customFor({S32})
+        .libcallFor({S128})
+        .clampScalar(0, S32, S128)
+        .widenScalarToNextPow2(0);
+    getActionDefinitionsBuilder(opcode(GenericOpcode::G_SREM))
+        .legalFor({S64})
+        .libcallFor({S128})
+        .clampScalar(0, S64, S128)
+        .widenScalarToNextPow2(0);
+  } else {
+    getActionDefinitionsBuilder({opcode(GenericOpcode::G_SDIV), opcode(GenericOpcode::G_UDIV),
+                                 opcode(GenericOpcode::G_SREM), opcode(GenericOpcode::G_UREM)})
+        .libcallFor({S64, S128})
+        .clampScalar(0, S64, S128)
+        .widenScalarToNextPow2(0);
+  }
+  getActionDefinitionsBuilder({opcode(GenericOpcode::G_SDIVREM), opcode(GenericOpcode::G_UDIVREM)})
+      .lower();
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_ABS)).lower();
+  getActionDefinitionsBuilder({opcode(GenericOpcode::G_ABDS), opcode(GenericOpcode::G_ABDU)})
+      .lower();
+  getActionDefinitionsBuilder({opcode(GenericOpcode::G_UMAX), opcode(GenericOpcode::G_UMIN),
+                               opcode(GenericOpcode::G_SMAX), opcode(GenericOpcode::G_SMIN),
+                               opcode(GenericOpcode::G_SCMP), opcode(GenericOpcode::G_UCMP)})
+      .lower();
+  getActionDefinitionsBuilder({opcode(GenericOpcode::G_MEMCPY), opcode(GenericOpcode::G_MEMMOVE),
+                               opcode(GenericOpcode::G_MEMSET)})
+      .libcall();
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_MEMCPY_INLINE)).lower();
+  getActionDefinitionsBuilder({opcode(GenericOpcode::G_DYN_STACKALLOC),
+                               opcode(GenericOpcode::G_STACKSAVE),
+                               opcode(GenericOpcode::G_STACKRESTORE)})
+      .lower();
+
+  getActionDefinitionsBuilder(
+      {opcode(GenericOpcode::G_FADD), opcode(GenericOpcode::G_FSUB), opcode(GenericOpcode::G_FMUL),
+       opcode(GenericOpcode::G_FDIV), opcode(GenericOpcode::G_FMA), opcode(GenericOpcode::G_FSQRT),
+       opcode(GenericOpcode::G_FMAXNUM), opcode(GenericOpcode::G_FMINNUM),
+       opcode(GenericOpcode::G_FMAXIMUMNUM), opcode(GenericOpcode::G_FMINIMUMNUM)})
+      .legalFor(HasF, {S32})
+      .legalFor(HasD, {S64})
+      .libcallFor({S32, S64})
+      .libcallFor({S128});
+  getActionDefinitionsBuilder({opcode(GenericOpcode::G_FNEG), opcode(GenericOpcode::G_FABS)})
+      .legalFor(HasF, {S32})
+      .legalFor(HasD, {S64})
+      .lowerFor({S32, S64, S128});
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_FREM))
+      .libcallFor({S32, S64, S128})
+      .minScalar(0, S32);
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_FCOPYSIGN))
+      .legalFor(HasF, {{S32, S32}})
+      .legalFor(HasD, {{S64, S64}, {S32, S64}, {S64, S32}})
+      .lower();
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_FPTRUNC))
+      .legalFor(HasD, {{S32, S64}})
+      .libcallFor({{S32, S64}, {S32, S128}, {S64, S128}});
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_FPEXT))
+      .legalFor(HasD, {{S64, S32}})
+      .libcallFor({{S64, S32}, {S128, S32}, {S128, S64}});
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_FCMP))
+      .legalFor(HasF, {{S64, S32}})
+      .legalFor(HasD, {{S64, S64}})
+      .clampScalar(0, S64, S64)
+      .libcallFor({{S64, S32}, {S64, S64}, {S64, S128}});
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_IS_FPCLASS)).lower();
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_FCONSTANT))
+      .legalFor(HasF, {S32})
+      .legalFor(HasD, {S64})
+      .customFor({S32, S64})
+      .lowerFor({S64, S128});
+  getActionDefinitionsBuilder({opcode(GenericOpcode::G_FPTOSI), opcode(GenericOpcode::G_FPTOUI)})
+      .legalFor(HasF, {{S64, S32}})
+      .legalFor(HasD, {{S64, S64}})
+      .customFor(HasF, {{S32, S32}})
+      .customFor(HasD, {{S32, S64}})
+      .widenScalarToNextPow2(0)
+      .minScalar(0, S32)
+      .libcallFor({{S32, S32},
+                   {S64, S32},
+                   {S32, S64},
+                   {S64, S64},
+                   {S32, S128},
+                   {S64, S128},
+                   {S128, S32},
+                   {S128, S64},
+                   {S128, S128}});
+  getActionDefinitionsBuilder({opcode(GenericOpcode::G_SITOFP), opcode(GenericOpcode::G_UITOFP)})
+      .legalFor(HasF, {{S32, S64}})
+      .legalFor(HasD, {{S64, S64}})
+      .widenScalarToNextPow2(1)
+      .minScalar(1, S32)
+      .libcallFor({{S32, S32},
+                   {S64, S32},
+                   {S32, S64},
+                   {S64, S64},
+                   {S128, S32},
+                   {S128, S64},
+                   {S32, S128},
+                   {S64, S128},
+                   {S128, S128}});
+  getActionDefinitionsBuilder({opcode(GenericOpcode::G_FCEIL), opcode(GenericOpcode::G_FFLOOR),
+                               opcode(GenericOpcode::G_FRINT), opcode(GenericOpcode::G_FNEARBYINT),
+                               opcode(GenericOpcode::G_INTRINSIC_TRUNC),
+                               opcode(GenericOpcode::G_INTRINSIC_ROUND),
+                               opcode(GenericOpcode::G_INTRINSIC_ROUNDEVEN)})
+      .libcallFor({S32, S64, S128});
+  getActionDefinitionsBuilder({opcode(GenericOpcode::G_FCOS), opcode(GenericOpcode::G_FSIN),
+                               opcode(GenericOpcode::G_FTAN), opcode(GenericOpcode::G_FPOW),
+                               opcode(GenericOpcode::G_FLOG), opcode(GenericOpcode::G_FLOG2),
+                               opcode(GenericOpcode::G_FLOG10), opcode(GenericOpcode::G_FEXP),
+                               opcode(GenericOpcode::G_FEXP2), opcode(GenericOpcode::G_FEXP10),
+                               opcode(GenericOpcode::G_FACOS), opcode(GenericOpcode::G_FASIN),
+                               opcode(GenericOpcode::G_FATAN), opcode(GenericOpcode::G_FATAN2),
+                               opcode(GenericOpcode::G_FCOSH), opcode(GenericOpcode::G_FSINH),
+                               opcode(GenericOpcode::G_FTANH), opcode(GenericOpcode::G_FMODF)})
+      .libcallFor({S32, S64, S128});
+  getActionDefinitionsBuilder({opcode(GenericOpcode::G_FPOWI), opcode(GenericOpcode::G_FLDEXP)})
+      .libcallFor({{S32, S32}, {S64, S32}, {S128, S32}});
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_FCANONICALIZE))
+      .legalFor(HasF, {S32})
+      .legalFor(HasD, {S64});
+
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_VASTART)).customFor({P0});
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_VAARG)).clampScalar(0, S64, S64).lower();
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS))
+      .customIf([](const LegalizationQuery &Query) {
+        return Query.Types.size() >= 3 && Query.Types[0].isCategoryNeutralScalar() &&
+               Query.Types[2].isPointer();
+      });
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_ATOMIC_CMPXCHG))
+      .legalFor(HasA, {{S64, P0}})
+      .customIf([](const LegalizationQuery &Query) {
+        return Query.Types.size() >= 2 && Query.Types[0].isCategoryNeutralScalar() &&
+               Query.Types[1].isPointer();
+      })
+      .libcallFor(!HasA, {{S8, P0}, {S16, P0}, {S32, P0}, {S64, P0}})
+      .clampScalar(0, S64, S64);
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_ATOMICRMW_ADD))
+      .legalFor(HasA, {{S64, P0}})
+      .libcallFor(!HasA, {{S8, P0}, {S16, P0}, {S32, P0}, {S64, P0}})
+      .clampScalar(0, S64, S64);
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_ATOMICRMW_SUB))
+      .libcallFor(!HasA, {{S8, P0}, {S16, P0}, {S32, P0}, {S64, P0}})
+      .clampScalar(0, S64, S64)
+      .lower();
+  getActionDefinitionsBuilder(
+      {opcode(GenericOpcode::G_ATOMICRMW_XCHG), opcode(GenericOpcode::G_ATOMICRMW_AND),
+       opcode(GenericOpcode::G_ATOMICRMW_OR), opcode(GenericOpcode::G_ATOMICRMW_XOR),
+       opcode(GenericOpcode::G_ATOMICRMW_MAX), opcode(GenericOpcode::G_ATOMICRMW_MIN),
+       opcode(GenericOpcode::G_ATOMICRMW_UMAX), opcode(GenericOpcode::G_ATOMICRMW_UMIN)})
+      .legalFor(HasA, {{S64, P0}})
+      .libcallFor(!HasA, {{S8, P0}, {S16, P0}, {S32, P0}, {S64, P0}})
+      .clampScalar(0, S64, S64);
+  getActionDefinitionsBuilder(
+      {opcode(GenericOpcode::G_INTRINSIC), opcode(GenericOpcode::G_INTRINSIC_W_SIDE_EFFECTS)})
+      .alwaysLegal();
+  getActionDefinitionsBuilder(opcode(GenericOpcode::G_FENCE)).alwaysLegal();
+  getActionDefinitionsBuilder({opcode(GenericOpcode::G_TRAP), opcode(GenericOpcode::G_DEBUGTRAP),
+                               opcode(GenericOpcode::G_UBSANTRAP)})
+      .alwaysLegal();
+}
+
+Expected<void> RISCVLegalizerInfo::legalizeCustom(LegalizerHelper &Helper,
+                                                  MachineInstr &Instruction,
+                                                  const LegalizationDecision &Decision) const {
+  if (Decision.Action != LegalizationAction::Custom)
+    return makeUnexpected(ErrorCode::InvalidArgument,
+                          "RISC-V custom legalizer received a non-custom action");
+
+  MachineFunction &Function = Helper.MIRBuilder.function();
+  const TargetDescription &Target = Helper.MIRBuilder.target();
+  if (Instruction.opcode() == opcode(GenericOpcode::G_ICMP))
     return lowerIntegerCompare(Instruction, Function, Target);
-  if (Action == LegalizationAction::Custom &&
-      Instruction.opcode() == opcode(GenericOpcode::G_ATOMIC_CMPXCHG))
+  if (Instruction.opcode() == opcode(GenericOpcode::G_ATOMIC_CMPXCHG))
     return lowerAtomicCompareExchange(Instruction, Function, Target);
-  if (Action == LegalizationAction::Custom &&
-      Instruction.opcode() == opcode(GenericOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS))
+  if (Instruction.opcode() == opcode(GenericOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS))
     return lowerAtomicCompareExchangeWithSuccess(Instruction, Function, Target);
-  if (Action == LegalizationAction::Custom &&
-      (Instruction.opcode() == opcode(GenericOpcode::G_ANYEXT) ||
-       Instruction.opcode() == opcode(GenericOpcode::G_SEXT) ||
-       Instruction.opcode() == opcode(GenericOpcode::G_ZEXT))) {
-    if (Instruction.opcode() == opcode(GenericOpcode::G_ANYEXT) &&
-        narrowAnyExtBinaryPlan(Instruction, Function).has_value())
-      return foldNarrowAnyExtBinary(Instruction, Function, Target);
+  if (Instruction.opcode() == opcode(GenericOpcode::G_ANYEXT) ||
+      Instruction.opcode() == opcode(GenericOpcode::G_SEXT) ||
+      Instruction.opcode() == opcode(GenericOpcode::G_ZEXT)) {
     return foldConstantExtension(Instruction, Function,
                                  static_cast<GenericOpcode>(Instruction.opcode().value()));
   }
-  if (Action == LegalizationAction::Custom &&
-      Instruction.opcode() == opcode(GenericOpcode::G_CONSTANT)) {
+  if (Instruction.opcode() == opcode(GenericOpcode::G_CONSTANT)) {
     const MachineOperand *Immediate = Instruction.operandAt(1);
     const std::optional<LowLevelType> Type = resultType(Function, Instruction);
     if (Immediate != nullptr && Type && Type->isCategoryNeutralScalar() &&
@@ -3789,8 +1113,10 @@ Expected<void> RISCVLegalizerInfo::lower(MachineInstr &Instruction, MachineFunct
       auto Value = signedImmediate(*Immediate);
       if (Value && shouldUseRV64IntegerConstantPool(*Value))
         return lowerConstantToPool(Instruction, Function, Target);
+      return {};
     }
-    return foldWideConstant(Instruction, Function, Target);
+    return makeUnexpected(ErrorCode::Unsupported,
+                          "RISC-V constant custom legalization is unavailable");
   }
   return makeUnexpected(ErrorCode::Unsupported,
                         "RISC-V legalizer has no lowering rule for this instruction");
diff --git a/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.h b/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.h
index f6ba000..d8e5e3e 100644
--- a/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.h
+++ b/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.h
@@ -11,17 +11,11 @@ class RISCVSubtarget;
 
 class RISCVLegalizerInfo final : public LegalizerInfo {
 public:
-  explicit RISCVLegalizerInfo(const RISCVSubtarget &Subtarget) : Subtarget(Subtarget) {}
-
-  [[nodiscard]] Expected<LegalizationAction>
-  classify(const MachineInstr &Instruction, const MachineFunction &Function,
-           const TargetDescription &Target) const override;
-  [[nodiscard]] Expected<void> lower(MachineInstr &Instruction, MachineFunction &Function,
-                                     const TargetDescription &Target,
-                                     LegalizationAction Action) const override;
+  explicit RISCVLegalizerInfo(const RISCVSubtarget &Subtarget);
+  [[nodiscard]] Expected<void> legalizeCustom(LegalizerHelper &Helper, MachineInstr &Instruction,
+                                              const LegalizationDecision &Decision) const override;
 
 private:
-  const RISCVSubtarget &Subtarget;
 };
 
 } // namespace rcc::riscv