[RISCV] Split the GlobalISel target components

7 files changed, 8207 insertions, 0 deletions

diff --git a/backend/lib/Target/RISCV/GISel/RISCVCallLowering.cpp b/backend/lib/Target/RISCV/GISel/RISCVCallLowering.cpp
new file mode 100644
index 0000000..cfe8bb6
--- /dev/null
+++ b/backend/lib/Target/RISCV/GISel/RISCVCallLowering.cpp
@@ -0,0 +1,1495 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "RISCVCallLowering.h"
+
+#include "rcc/CodeGen/GenericOpcodes.h"
+#include "rcc/CodeGen/MachineBasicBlock.h"
+#include "rcc/CodeGen/MachineFrameInfo.h"
+#include "rcc/CodeGen/MachineFunction.h"
+#include "rcc/CodeGen/MachineIRBuilder.h"
+#include "rcc/CodeGen/MachineInstr.h"
+#include "rcc/CodeGen/MachineMemOperand.h"
+#include "rcc/CodeGen/MachineRegisterInfo.h"
+#include "rcc/CodeGen/TargetDescription.h"
+#include "rcc/Target/RISCV/RISCVTarget.h"
+
+#include <algorithm>
+#include <array>
+#include <cstddef>
+#include <cstdint>
+#include <limits>
+#include <optional>
+#include <span>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+#include <utility>
+#include <vector>
+
+namespace rcc::riscv {
+namespace {
+
+constexpr std::string_view PlanTarget = "riscv64-unknown-linux-gnu";
+constexpr std::uint64_t XLenBytes = 8;
+constexpr std::uint64_t ABIStackAlignmentBytes = 16;
+constexpr std::uint32_t AllowedDirectCallFlags =
+    static_cast<std::uint32_t>(CallTargetFlag::DirectCall);
+
+[[nodiscard]] Opcode targetOpcode(InstructionOpcode Value) noexcept {
+  return Opcode::fromValue(static_cast<std::uint32_t>(Value));
+}
+
+[[nodiscard]] std::uint64_t stackObjectFlag(CallStackObjectFlag Flag) noexcept {
+  return static_cast<std::uint64_t>(Flag);
+}
+
+[[nodiscard]] Expected<Alignment> alignment(std::uint64_t Bytes) {
+  return Alignment::create(Bytes);
+}
+
+[[nodiscard]] Expected<std::uint64_t> alignTo(std::uint64_t Value, std::uint64_t AlignmentBytes) {
+  const std::uint64_t Remainder = Value % AlignmentBytes;
+  if (Remainder == 0)
+    return Value;
+  const std::uint64_t Increment = AlignmentBytes - Remainder;
+  if (Value > std::numeric_limits<std::uint64_t>::max() - Increment) {
+    return makeUnexpected(ErrorCode::Unsupported, "RISC-V ABI stack-area size overflows uint64");
+  }
+  return Value + Increment;
+}
+
+[[nodiscard]] std::uint64_t valueSizeBytes(LLT Type) noexcept {
+  return (static_cast<std::uint64_t>(Type.scalarBitWidth()) + 7U) / 8U;
+}
+
+[[nodiscard]] bool isSupportedFloat(LLT Type) noexcept {
+  if (!Type.isFloat())
+    return false;
+  const auto Semantics = Type.floatSemantics();
+  return Semantics &&
+         (*Semantics == FloatSemantics::IEEESingle || *Semantics == FloatSemantics::IEEEDouble);
+}
+
+[[nodiscard]] Expected<PhysicalRegister> floatingRegisterView(PhysicalRegister DoubleView,
+                                                              LLT Type) {
+  if (!isSupportedFloat(Type)) {
+    return makeUnexpected(ErrorCode::InvalidArgument,
+                          "RISC-V floating-register view requires f32 or f64");
+  }
+  if (Type.scalarBitWidth() == 64)
+    return DoubleView;
+
+  constexpr std::uint32_t FirstDouble = static_cast<std::uint32_t>(PhysicalRegisterID::F0_D);
+  constexpr std::uint32_t LastDouble = static_cast<std::uint32_t>(PhysicalRegisterID::F31_D);
+  constexpr std::uint32_t FirstSingle = static_cast<std::uint32_t>(PhysicalRegisterID::F0_F);
+  const std::uint32_t Number = DoubleView.number();
+  if (Number < FirstDouble || Number > LastDouble) {
+    return makeUnexpected(ErrorCode::InternalError,
+                          "RISC-V ABI floating register is not an FPR64 view");
+  }
+  return PhysicalRegister::create(FirstSingle + Number - FirstDouble);
+}
+
+[[nodiscard]] bool requiresIntegerTransport(const ABIValue &Value) noexcept {
+  return !isSupportedFloat(Value.Type) || Value.Flags.has(ABIValueFlag::ByVal) ||
+         Value.Flags.has(ABIValueFlag::StructReturn) || Value.Flags.has(ABIValueFlag::Indirect);
+}
+
+[[nodiscard]] Expected<void> verifyABIValue(const ABIValue &Value, bool RequireRegister) {
+  if (RequireRegister && Value.Value.isNoRegister()) {
+    return makeUnexpected(ErrorCode::InvalidArgument,
+                          "RISC-V ABI value requires a source or destination register");
+  }
+  if (!Value.Type.isValid() || !Value.Type.isScalarValue() || Value.Type.scalarBitWidth() > 64) {
+    return makeUnexpected(
+        ErrorCode::Unsupported,
+        "RISC-V ABI accepts explicitly classified integer, pointer, f32, and f64 scalar fragments");
+  }
+  if (Value.Type.isFloat() && !isSupportedFloat(Value.Type)) {
+    return makeUnexpected(ErrorCode::Unsupported,
+                          "RISC-V LP64 ABIs only support f32 and f64 floating fragments");
+  }
+  if (Value.Type.isPointer() && Value.Type.scalarBitWidth() != 64) {
+    return makeUnexpected(ErrorCode::Unsupported, "RISC-V LP64 ABIs require 64-bit pointers");
+  }
+  if ((Value.Type.isInteger() || Value.Type.isAnyScalar()) && Value.Type.scalarBitWidth() < 64) {
+    if (Value.Extension == ABIExtensionKind::None) {
+      return makeUnexpected(ErrorCode::InvalidArgument,
+                            "narrow RISC-V ABI integers require an explicit extension kind");
+    }
+  } else if (Value.Extension != ABIExtensionKind::None) {
+    return makeUnexpected(ErrorCode::InvalidArgument,
+                          "only narrow integer ABI fragments may carry an extension kind");
+  }
+  if (Value.Extension >= ABIExtensionKind::Count || Value.FragmentCount == 0 ||
+      Value.FragmentIndex >= Value.FragmentCount || Value.FragmentCount > 2) {
+    return makeUnexpected(ErrorCode::InvalidArgument,
+                          "RISC-V ABI fragment coordinates or extension are invalid");
+  }
+
+  const bool IsByVal = Value.Flags.has(ABIValueFlag::ByVal);
+  const bool IsSRet = Value.Flags.has(ABIValueFlag::StructReturn);
+  const bool IsIndirect = Value.Flags.has(ABIValueFlag::Indirect);
+  const unsigned SpecialFlagCount = static_cast<unsigned>(IsByVal) + static_cast<unsigned>(IsSRet) +
+                                    static_cast<unsigned>(IsIndirect);
+  if (SpecialFlagCount > 1) {
+    return makeUnexpected(ErrorCode::InvalidArgument,
+                          "byval, sret, and indirect ABI flags are mutually exclusive");
+  }
+  if ((IsByVal || IsSRet || IsIndirect) &&
+      (!Value.Type.isPointer() || Value.FragmentCount != 1 || Value.FragmentIndex != 0 ||
+       Value.FragmentOffsetBytes != 0)) {
+    return makeUnexpected(ErrorCode::InvalidArgument,
+                          "byval, sret, and indirect values require one explicit pointer fragment");
+  }
+  if (IsByVal) {
+    if (Value.ByValSizeBytes == 0) {
+      return makeUnexpected(ErrorCode::InvalidArgument,
+                            "byval arguments require a nonzero copy size");
+    }
+  } else if (Value.ByValSizeBytes != 0) {
+    return makeUnexpected(ErrorCode::InvalidArgument,
+                          "non-byval values cannot carry a byval copy size");
+  }
+  return {};
+}
+
+[[nodiscard]] Expected<std::size_t> verifyValueSequence(std::span<const ABIValue> Values,
+                                                        bool RequireRegisters) {
+  std::size_t ExpectedOriginal = 0;
+  std::size_t Index = 0;
+  while (Index < Values.size()) {
+    const ABIValue &First = Values[Index];
+    auto ValueVerification = verifyABIValue(First, RequireRegisters);
+    if (!ValueVerification)
+      return std::unexpected(ValueVerification.error());
+    if (First.OriginalValue != ExpectedOriginal || First.FragmentIndex != 0 ||
+        Index + First.FragmentCount > Values.size()) {
+      return makeUnexpected(
+          ErrorCode::InvalidArgument,
+          "RISC-V ABI values must form contiguous, zero-based original-value groups");
+    }
+
+    std::uint64_t PreviousEnd = 0;
+    for (std::size_t Fragment = 0; Fragment < First.FragmentCount; ++Fragment) {
+      const ABIValue &Value = Values[Index + Fragment];
+      auto FragmentVerification = verifyABIValue(Value, RequireRegisters);
+      if (!FragmentVerification)
+        return std::unexpected(FragmentVerification.error());
+      if (Value.OriginalValue != First.OriginalValue || Value.FragmentIndex != Fragment ||
+          Value.FragmentCount != First.FragmentCount ||
+          Value.OriginalAlignment != First.OriginalAlignment || Value.Flags != First.Flags ||
+          Value.ByValSizeBytes != First.ByValSizeBytes ||
+          Value.ByValAlignment != First.ByValAlignment || Value.FragmentOffsetBytes < PreviousEnd) {
+        return makeUnexpected(ErrorCode::InvalidArgument,
+                              "RISC-V ABI fragments of one value have inconsistent metadata");
+      }
+      PreviousEnd =
+          static_cast<std::uint64_t>(Value.FragmentOffsetBytes) + valueSizeBytes(Value.Type);
+    }
+    Index += First.FragmentCount;
+    ++ExpectedOriginal;
+  }
+  return ExpectedOriginal;
+}
+
+[[nodiscard]] CCAssignmentFlags assignmentFlags(const ABIValue &Value, bool IsVariadic,
+                                                bool IsBitcast) {
+  CCAssignmentFlags Result;
+  if (IsVariadic)
+    Result.set(CCAssignmentFlag::Variadic);
+  if (Value.Flags.has(ABIValueFlag::ByVal))
+    Result.set(CCAssignmentFlag::ByVal);
+  if (Value.Flags.has(ABIValueFlag::StructReturn))
+    Result.set(CCAssignmentFlag::StructReturn);
+  if (Value.Flags.has(ABIValueFlag::Indirect))
+    Result.set(CCAssignmentFlag::Indirect);
+  if (Value.Flags.has(ABIValueFlag::InRegister))
+    Result.set(CCAssignmentFlag::InRegister);
+  if (IsBitcast)
+    Result.set(CCAssignmentFlag::Bitcast);
+  return Result;
+}
+
+[[nodiscard]] CCLocationKind locationKind(const ABIValue &Value, bool IsRegister) noexcept {
+  if (Value.Flags.has(ABIValueFlag::Indirect))
+    return CCLocationKind::Indirect;
+  return IsRegister ? CCLocationKind::Register : CCLocationKind::Stack;
+}
+
+struct ArgumentAssignmentResult final {
+  std::vector<CCAssignment> Assignments;
+  StackArgumentArea StackArea;
+  std::size_t FirstUnallocatedGPR = 0;
+};
+
+struct AssignmentState final {
+  std::size_t GPR = 0;
+  std::size_t FPR = 0;
+  std::uint64_t StackOffset = 0;
+  bool VariadicStackOnly = false;
+};
+
+[[nodiscard]] Expected<CCAssignment> registerAssignment(const ABIValue &Value,
+                                                        std::size_t ValueIndex,
+                                                        PhysicalRegister Location, LLT LocationType,
+                                                        bool IsVariadic, bool IsBitcast) {
+  auto Align =
+      alignment(std::min<std::uint64_t>(Value.OriginalAlignment.bytes(), ABIStackAlignmentBytes));
+  if (!Align)
+    return std::unexpected(Align.error());
+  return CCAssignment{
+      .ValueIndex = ValueIndex,
+      .OriginalValue = Value.Value,
+      .OriginalValueIndex = Value.OriginalValue,
+      .FragmentIndex = Value.FragmentIndex,
+      .FragmentCount = Value.FragmentCount,
+      .ValueType = Value.Type,
+      .LocationType = LocationType,
+      .LocationKind = locationKind(Value, true),
+      .LocationRegister = Location,
+      .StackOffsetBytes = std::nullopt,
+      .LocationSizeBytes = valueSizeBytes(LocationType),
+      .StackSlotSizeBytes = XLenBytes,
+      .LocationAlignment = *Align,
+      .Extension = Value.Extension,
+      .CustomFlags = assignmentFlags(Value, IsVariadic, IsBitcast),
+  };
+}
+
+[[nodiscard]] Expected<CCAssignment> stackAssignment(const ABIValue &Value, std::size_t ValueIndex,
+                                                     std::uint64_t StackOffset, LLT LocationType,
+                                                     bool IsVariadic,
+                                                     std::uint64_t StackAlignment) {
+  auto Align = alignment(StackAlignment);
+  if (!Align)
+    return std::unexpected(Align.error());
+  return CCAssignment{
+      .ValueIndex = ValueIndex,
+      .OriginalValue = Value.Value,
+      .OriginalValueIndex = Value.OriginalValue,
+      .FragmentIndex = Value.FragmentIndex,
+      .FragmentCount = Value.FragmentCount,
+      .ValueType = Value.Type,
+      .LocationType = LocationType,
+      .LocationKind = locationKind(Value, false),
+      .LocationRegister = std::nullopt,
+      .StackOffsetBytes = StackOffset,
+      .LocationSizeBytes = valueSizeBytes(LocationType),
+      .StackSlotSizeBytes = XLenBytes,
+      .LocationAlignment = *Align,
+      .Extension = Value.Extension,
+      .CustomFlags = assignmentFlags(Value, IsVariadic, false),
+  };
+}
+
+[[nodiscard]] Expected<ArgumentAssignmentResult>
+assignArguments(std::span<const ABIValue> Values, std::size_t FixedArgumentCount, bool IsVariadic,
+                const CallingConventionInfo &Convention) {
+  auto I64 = LLT::makeInteger(64);
+  auto StackAlign = alignment(Convention.StackArgumentAlignmentBytes);
+  if (!I64 || !StackAlign)
+    return makeUnexpected(ErrorCode::InternalError, "RISC-V ABI primitive type facts are invalid");
+
+  ArgumentAssignmentResult Result;
+  AssignmentState State;
+  State.StackOffset = Convention.ShadowSpaceBytes;
+  Result.Assignments.reserve(Values.size());
+
+  std::size_t Index = 0;
+  while (Index < Values.size()) {
+    const ABIValue &First = Values[Index];
+    const std::size_t GroupSize = First.FragmentCount;
+    const bool VariadicArgument = IsVariadic && First.OriginalValue >= FixedArgumentCount;
+    bool HasPreferredFloat = false;
+    std::size_t RequiredGPRs = 0;
+    std::size_t RequiredFPRs = 0;
+    for (std::size_t Fragment = 0; Fragment < GroupSize; ++Fragment) {
+      const ABIValue &Value = Values[Index + Fragment];
+      if (!VariadicArgument && !requiresIntegerTransport(Value)) {
+        HasPreferredFloat = true;
+        ++RequiredFPRs;
+      } else {
+        ++RequiredGPRs;
+      }
+    }
+
+    const bool CanUsePreferredAggregate =
+        HasPreferredFloat &&
+        State.GPR + RequiredGPRs <= Convention.IntegerArgumentRegisters.size() &&
+        State.FPR + RequiredFPRs <= Convention.FloatingArgumentRegisters.size();
+    if (CanUsePreferredAggregate) {
+      for (std::size_t Fragment = 0; Fragment < GroupSize; ++Fragment) {
+        const ABIValue &Value = Values[Index + Fragment];
+        Expected<CCAssignment> Assignment = makeUnexpected(
+            ErrorCode::InternalError, "RISC-V ABI argument assignment was not initialized");
+        if (requiresIntegerTransport(Value)) {
+          Assignment = registerAssignment(Value, Index + Fragment,
+                                          Convention.IntegerArgumentRegisters[State.GPR++], *I64,
+                                          false, false);
+        } else {
+          auto View =
+              floatingRegisterView(Convention.FloatingArgumentRegisters[State.FPR++], Value.Type);
+          if (!View)
+            return std::unexpected(View.error());
+          Assignment = registerAssignment(Value, Index + Fragment, *View, Value.Type, false, false);
+        }
+        if (!Assignment)
+          return std::unexpected(Assignment.error());
+        Result.Assignments.push_back(*Assignment);
+      }
+      Index += GroupSize;
+      continue;
+    }
+
+    const bool RequiresAlignedPair =
+        VariadicArgument && GroupSize == 2 && First.OriginalAlignment.bytes() >= 16;
+    if (RequiresAlignedPair && State.GPR % 2 != 0 &&
+        State.GPR < Convention.IntegerArgumentRegisters.size()) {
+      ++State.GPR;
+    }
+    bool ForceEntireGroupToStack = State.VariadicStackOnly;
+    if (RequiresAlignedPair && State.GPR + GroupSize > Convention.IntegerArgumentRegisters.size()) {
+      ForceEntireGroupToStack = true;
+      State.GPR = Convention.IntegerArgumentRegisters.size();
+    }
+
+    bool FirstStackFragment = true;
+    for (std::size_t Fragment = 0; Fragment < GroupSize; ++Fragment) {
+      const ABIValue &Value = Values[Index + Fragment];
+      const bool HasGPR =
+          !ForceEntireGroupToStack && State.GPR < Convention.IntegerArgumentRegisters.size();
+      if (HasGPR) {
+        const bool Bitcast = isSupportedFloat(Value.Type);
+        auto Assignment = registerAssignment(Value, Index + Fragment,
+                                             Convention.IntegerArgumentRegisters[State.GPR++], *I64,
+                                             VariadicArgument, Bitcast);
+        if (!Assignment)
+          return std::unexpected(Assignment.error());
+        Result.Assignments.push_back(*Assignment);
+        continue;
+      }
+
+      if (Value.Flags.has(ABIValueFlag::InRegister)) {
+        return makeUnexpected(ErrorCode::Unsupported,
+                              "RISC-V inreg argument cannot be assigned to the stack");
+      }
+      std::uint64_t RequiredAlignment = XLenBytes;
+      if (FirstStackFragment && ForceEntireGroupToStack) {
+        RequiredAlignment = std::min<std::uint64_t>(
+            std::max<std::uint64_t>(XLenBytes, First.OriginalAlignment.bytes()),
+            ABIStackAlignmentBytes);
+      }
+      auto AlignedOffset = alignTo(State.StackOffset, RequiredAlignment);
+      if (!AlignedOffset)
+        return std::unexpected(AlignedOffset.error());
+      State.StackOffset = *AlignedOffset;
+      const LLT LocationType = isSupportedFloat(Value.Type) ? Value.Type : *I64;
+      auto Assignment = stackAssignment(Value, Index + Fragment, State.StackOffset, LocationType,
+                                        VariadicArgument, RequiredAlignment);
+      if (!Assignment)
+        return std::unexpected(Assignment.error());
+      Result.Assignments.push_back(*Assignment);
+      if (State.StackOffset > std::numeric_limits<std::uint64_t>::max() - XLenBytes) {
+        return makeUnexpected(ErrorCode::Unsupported, "RISC-V argument area overflows uint64");
+      }
+      State.StackOffset += XLenBytes;
+      FirstStackFragment = false;
+      if (VariadicArgument)
+        State.VariadicStackOnly = true;
+    }
+    Index += GroupSize;
+  }
+
+  auto FinalSize = alignTo(State.StackOffset, Convention.StackArgumentAlignmentBytes);
+  if (!FinalSize)
+    return std::unexpected(FinalSize.error());
+  Result.StackArea = StackArgumentArea{*FinalSize, Convention.ShadowSpaceBytes, *StackAlign};
+  Result.FirstUnallocatedGPR = State.GPR;
+  return Result;
+}
+
+[[nodiscard]] Expected<std::vector<CCAssignment>>
+assignReturns(std::span<const ABIValue> Values, bool RequireRegisters,
+              const CallingConventionInfo &Convention) {
+  auto Sequence = verifyValueSequence(Values, RequireRegisters);
+  if (!Sequence)
+    return std::unexpected(Sequence.error());
+  if (Values.size() > 2) {
+    return makeUnexpected(ErrorCode::Unsupported,
+                          "RISC-V direct returns are limited to two explicit fragments");
+  }
+  auto I64 = LLT::makeInteger(64);
+  if (!I64)
+    return std::unexpected(I64.error());
+
+  std::vector<CCAssignment> Result;
+  Result.reserve(Values.size());
+  std::size_t GPR = 0;
+  std::size_t FPR = 0;
+  for (std::size_t Index = 0; Index < Values.size(); ++Index) {
+    const ABIValue &Value = Values[Index];
+    if (Value.Flags.rawValue() != 0) {
+      return makeUnexpected(ErrorCode::InvalidArgument,
+                            "return fragments cannot carry argument-only ABI flags");
+    }
+    const bool UseFloatingRegister =
+        isSupportedFloat(Value.Type) && FPR < Convention.FloatingReturnRegisters.size();
+    Expected<CCAssignment> Assignment = makeUnexpected(
+        ErrorCode::Unsupported, "RISC-V calling convention has no return register available");
+    if (UseFloatingRegister) {
+      auto View = floatingRegisterView(Convention.FloatingReturnRegisters[FPR++], Value.Type);
+      if (!View)
+        return std::unexpected(View.error());
+      Assignment = registerAssignment(Value, Index, *View, Value.Type, false, false);
+    } else if (GPR < Convention.IntegerReturnRegisters.size()) {
+      Assignment = registerAssignment(Value, Index, Convention.IntegerReturnRegisters[GPR++], *I64,
+                                      false, isSupportedFloat(Value.Type));
+    }
+    if (!Assignment)
+      return std::unexpected(Assignment.error());
+    Result.push_back(*Assignment);
+  }
+  return Result;
+}
+
+[[nodiscard]] bool hasStructReturn(std::span<const ABIValue> Arguments) noexcept {
+  return std::ranges::any_of(
+      Arguments, [](const ABIValue &Value) { return Value.Flags.has(ABIValueFlag::StructReturn); });
+}
+
+[[nodiscard]] bool equivalentReturnLocations(std::span<const CCAssignment> Left,
+                                             std::span<const CCAssignment> Right) noexcept {
+  if (Left.size() != Right.size())
+    return false;
+  for (std::size_t Index = 0; Index < Left.size(); ++Index) {
+    if (Left[Index].ValueType != Right[Index].ValueType ||
+        Left[Index].LocationType != Right[Index].LocationType ||
+        Left[Index].LocationKind != Right[Index].LocationKind ||
+        Left[Index].LocationRegister != Right[Index].LocationRegister ||
+        Left[Index].Extension != Right[Index].Extension ||
+        Left[Index].CustomFlags != Right[Index].CustomFlags) {
+      return false;
+    }
+  }
+  return true;
+}
+
+[[nodiscard]] Expected<std::string>
+tailCallRejection(const CallLoweringRequest &Request, const ArgumentAssignmentResult &Arguments,
+                  std::span<const CCAssignment> Results,
+                  std::span<const ByValCopyAssignment> ByValCopies,
+                  const CallingConventionInfo &Convention) {
+  if (!Convention.SupportsTailCalls)
+    return std::string("the calling convention does not support tail calls");
+  if (!Request.TailContext)
+    return std::string("tail-call context is missing");
+  const TailCallContext &Context = *Request.TailContext;
+  if (Context.CallerConvention != Request.Convention)
+    return std::string("caller and callee calling conventions differ");
+  if (Context.CallerIsVariadic != Request.IsVariadic ||
+      (Request.IsVariadic && Context.CallerFixedArgumentCount != Request.FixedArgumentCount)) {
+    return std::string("caller and callee variadic signatures differ");
+  }
+  if (Context.CallerHasDynamicStackAllocation)
+    return std::string("the caller has a dynamic stack allocation");
+  if (!ByValCopies.empty())
+    return std::string("tail calls cannot create byval copies");
+  if (Context.CallerUsesStructReturn != hasStructReturn(Request.Arguments))
+    return std::string("caller and callee sret conventions differ");
+  if (Arguments.StackArea.SizeBytes > Context.ReusableIncomingArgumentBytes)
+    return std::string("outgoing stack arguments exceed the reusable incoming area");
+  if (std::ranges::any_of(Request.Results,
+                          [](const ABIValue &Value) { return !Value.Value.isNoRegister(); })) {
+    return std::string("tail-call result signatures must not request local result copies");
+  }
+  auto CallerReturns = assignReturns(Context.CallerReturns, false, Convention);
+  if (!CallerReturns)
+    return std::unexpected(CallerReturns.error());
+  if (!equivalentReturnLocations(Results, *CallerReturns))
+    return std::string("caller and callee return locations differ");
+  return std::string{};
+}
+
+[[nodiscard]] Expected<const CallingConventionInfo *>
+riscvConvention(CallingConventionID Convention) {
+  CallingConvention TargetConvention = CallingConvention::None;
+  if (Convention == callingConventionID(CallingConvention::LP64D))
+    TargetConvention = CallingConvention::LP64D;
+  else if (Convention == callingConventionID(CallingConvention::LP64))
+    TargetConvention = CallingConvention::LP64;
+  else
+    return makeUnexpected(ErrorCode::Unsupported, "RISC-V backend only supports LP64 and LP64D");
+  const CallingConventionInfo *Info = callingConvention(TargetConvention);
+  if (Info == nullptr)
+    return makeUnexpected(ErrorCode::InternalError,
+                          "RISC-V calling-convention target facts are unavailable");
+  return Info;
+}
+
+} // namespace
+
+Expected<CallLoweringPlan>
+RISCVCallLowering::planFormalArguments(const FormalArgumentRequest &Request) const {
+  auto Convention = riscvConvention(Request.Convention);
+  if (!Convention)
+    return std::unexpected(Convention.error());
+  auto OriginalCount = verifyValueSequence(Request.Arguments, true);
+  if (!OriginalCount)
+    return std::unexpected(OriginalCount.error());
+  if (Request.FixedArgumentCount != *OriginalCount) {
+    return makeUnexpected(ErrorCode::InvalidArgument,
+                          "formal-argument request must identify every declared argument as fixed");
+  }
+  if (!Request.IsVariadic && Request.FixedArgumentCount != *OriginalCount) {
+    return makeUnexpected(ErrorCode::InvalidArgument,
+                          "non-variadic formal request has an invalid fixed-argument boundary");
+  }
+  if (hasStructReturn(Request.Arguments) &&
+      (Request.Arguments.empty() || Request.Arguments.front().OriginalValue != 0 ||
+       !Request.Arguments.front().Flags.has(ABIValueFlag::StructReturn))) {
+    return makeUnexpected(ErrorCode::InvalidArgument,
+                          "RISC-V sret must be the first formal argument");
+  }
+
+  auto Assignments = assignArguments(Request.Arguments, Request.FixedArgumentCount,
+                                     Request.IsVariadic, **Convention);
+  if (!Assignments)
+    return std::unexpected(Assignments.error());
+
+  VarArgLoweringState VarArgs;
+  if (Request.IsVariadic) {
+    VarArgs.IsVariadic = true;
+    VarArgs.FixedArgumentCount = Request.FixedArgumentCount;
+    VarArgs.FirstUnallocatedIntegerRegister = Assignments->FirstUnallocatedGPR;
+    VarArgs.OverflowAreaOffsetBytes = Assignments->StackArea.SizeBytes;
+    const std::size_t First = Assignments->FirstUnallocatedGPR;
+    const std::size_t RegisterCount = (*Convention)->IntegerArgumentRegisters.size();
+    VarArgs.RegisterSaveAreaSizeBytes = (RegisterCount - First) * XLenBytes;
+    VarArgs.RegisterSaveAreaOffsetBytes =
+        VarArgs.RegisterSaveAreaSizeBytes == 0
+            ? static_cast<std::int64_t>(VarArgs.OverflowAreaOffsetBytes)
+            : -static_cast<std::int64_t>(VarArgs.RegisterSaveAreaSizeBytes);
+    if (VarArgs.RegisterSaveAreaSizeBytes != 0 && First % 2 != 0)
+      VarArgs.RegisterSavePaddingBytes = XLenBytes;
+    for (std::size_t Index = First; Index < RegisterCount; ++Index) {
+      VarArgs.RegistersToSave.push_back(
+          {(*Convention)->IntegerArgumentRegisters[Index], (Index - First) * XLenBytes});
+    }
+  }
+
+  CallLoweringPlanDesc Plan;
+  Plan.Kind = CallLoweringPlanKind::FormalArguments;
+  Plan.Target = PlanTarget;
+  Plan.Convention = Request.Convention;
+  Plan.ArgumentAssignments = std::move(Assignments->Assignments);
+  Plan.StackArguments = Assignments->StackArea;
+  Plan.VarArgs = std::move(VarArgs);
+  return CallLoweringPlan::create(std::move(Plan));
+}
+
+Expected<CallLoweringPlan> RISCVCallLowering::planCall(const CallLoweringRequest &Request) const {
+  auto Convention = riscvConvention(Request.Convention);
+  if (!Convention)
+    return std::unexpected(Convention.error());
+  auto OriginalCount = verifyValueSequence(Request.Arguments, true);
+  if (!OriginalCount)
+    return std::unexpected(OriginalCount.error());
+  if ((!Request.IsVariadic && Request.FixedArgumentCount != *OriginalCount) ||
+      (Request.IsVariadic && Request.FixedArgumentCount > *OriginalCount)) {
+    return makeUnexpected(ErrorCode::InvalidArgument,
+                          "call request has an invalid fixed/variadic argument boundary");
+  }
+  if (!Request.Callee.isValid())
+    return makeUnexpected(ErrorCode::InvalidArgument, "call request requires a valid callee");
+  if (hasStructReturn(Request.Arguments) &&
+      (Request.Arguments.empty() ||
+       !Request.Arguments.front().Flags.has(ABIValueFlag::StructReturn))) {
+    return makeUnexpected(ErrorCode::InvalidArgument,
+                          "RISC-V sret must be the first call argument");
+  }
+  if (hasStructReturn(Request.Arguments) && !Request.Results.empty()) {
+    return makeUnexpected(ErrorCode::InvalidArgument,
+                          "RISC-V sret calls cannot also request direct return values");
+  }
+
+  MachineCallTarget EffectiveCallee = Request.Callee;
+  switch (Request.Callee.kind()) {
+  case MachineCallTargetKind::GlobalSymbol: {
+    if ((Request.Callee.targetFlags() & ~AllowedDirectCallFlags) != 0) {
+      return makeUnexpected(ErrorCode::InvalidArgument,
+                            "direct RISC-V call uses unknown target flags");
+    }
+    auto Callee = MachineCallTarget::globalSymbol(
+        std::string(Request.Callee.symbolName()),
+        Request.Callee.targetFlags() | static_cast<std::uint32_t>(CallTargetFlag::DirectCall));
+    if (!Callee)
+      return std::unexpected(Callee.error());
+    EffectiveCallee = std::move(*Callee);
+    break;
+  }
+  case MachineCallTargetKind::ExternalSymbol: {
+    if ((Request.Callee.targetFlags() & ~AllowedDirectCallFlags) != 0) {
+      return makeUnexpected(ErrorCode::InvalidArgument,
+                            "direct RISC-V call uses unknown target flags");
+    }
+    auto Callee = MachineCallTarget::externalSymbol(
+        std::string(Request.Callee.symbolName()),
+        Request.Callee.targetFlags() | static_cast<std::uint32_t>(CallTargetFlag::DirectCall));
+    if (!Callee)
+      return std::unexpected(Callee.error());
+    EffectiveCallee = std::move(*Callee);
+    break;
+  }
+  case MachineCallTargetKind::IndirectRegister:
+    if (Request.Callee.targetFlags() != 0) {
+      return makeUnexpected(ErrorCode::InvalidArgument,
+                            "indirect RISC-V calls cannot carry direct-call target flags");
+    }
+    break;
+  case MachineCallTargetKind::Invalid:
+    return makeUnexpected(ErrorCode::InvalidArgument, "call target is invalid");
+  }
+
+  const bool RequireResultRegisters = Request.TailKind == TailCallKind::NotTail;
+  auto ResultSequence = verifyValueSequence(Request.Results, RequireResultRegisters);
+  if (!ResultSequence)
+    return std::unexpected(ResultSequence.error());
+  auto Arguments = assignArguments(Request.Arguments, Request.FixedArgumentCount,
+                                   Request.IsVariadic, **Convention);
+  if (!Arguments)
+    return std::unexpected(Arguments.error());
+  auto Results = assignReturns(Request.Results, RequireResultRegisters, **Convention);
+  if (!Results)
+    return std::unexpected(Results.error());
+
+  auto Align16 = alignment(ABIStackAlignmentBytes);
+  if (!Align16)
+    return std::unexpected(Align16.error());
+  std::uint64_t CallFrameCursor = Arguments->StackArea.SizeBytes;
+  std::vector<ByValCopyAssignment> ByValCopies;
+  for (std::size_t Index = 0; Index < Arguments->Assignments.size(); ++Index) {
+    const CCAssignment &Assignment = Arguments->Assignments[Index];
+    if (!Assignment.CustomFlags.has(CCAssignmentFlag::ByVal))
+      continue;
+    const ABIValue &Value = Request.Arguments[Assignment.ValueIndex];
+    const std::uint64_t CopyAlignment =
+        std::min<std::uint64_t>(Value.ByValAlignment.bytes(), ABIStackAlignmentBytes);
+    auto CopyOffset = alignTo(CallFrameCursor, CopyAlignment);
+    if (!CopyOffset)
+      return std::unexpected(CopyOffset.error());
+    if (*CopyOffset > std::numeric_limits<std::uint64_t>::max() - Value.ByValSizeBytes) {
+      return makeUnexpected(ErrorCode::Unsupported, "RISC-V byval area overflows uint64");
+    }
+    auto CopyAlign = alignment(CopyAlignment);
+    if (!CopyAlign)
+      return std::unexpected(CopyAlign.error());
+    ByValCopies.push_back(
+        {Index, Value.Value, Value.Type, *CopyOffset, Value.ByValSizeBytes, *CopyAlign});
+    CallFrameCursor = *CopyOffset + Value.ByValSizeBytes;
+  }
+  auto CallFrameSize = alignTo(CallFrameCursor, ABIStackAlignmentBytes);
+  if (!CallFrameSize)
+    return std::unexpected(CallFrameSize.error());
+
+  TailCallKind Decision = TailCallKind::NotTail;
+  std::string RejectionReason;
+  if (Request.TailKind != TailCallKind::NotTail) {
+    auto Rejection = tailCallRejection(Request, *Arguments, *Results, ByValCopies, **Convention);
+    if (!Rejection)
+      return std::unexpected(Rejection.error());
+    RejectionReason = std::move(*Rejection);
+    if (RejectionReason.empty()) {
+      Decision = Request.TailKind;
+    } else if (Request.TailKind == TailCallKind::MustTail) {
+      return makeUnexpected(ErrorCode::Unsupported,
+                            "musttail call is not ABI-compatible: " + RejectionReason);
+    } else {
+      auto NormalResults = verifyValueSequence(Request.Results, true);
+      if (!NormalResults)
+        return std::unexpected(NormalResults.error());
+    }
+  }
+
+  CallLoweringPlanDesc Plan;
+  Plan.Kind = CallLoweringPlanKind::Call;
+  Plan.Target = PlanTarget;
+  Plan.Convention = Request.Convention;
+  Plan.ArgumentAssignments = std::move(Arguments->Assignments);
+  Plan.ReturnAssignments = std::move(*Results);
+  Plan.StackArguments = Arguments->StackArea;
+  Plan.CallFrame = CallFrameRequirements{
+      Decision == TailCallKind::NotTail ? *CallFrameSize : Arguments->StackArea.SizeBytes, *Align16,
+      Decision == TailCallKind::NotTail, Decision == TailCallKind::NotTail,
+      Decision != TailCallKind::NotTail};
+  if (Request.IsVariadic) {
+    Plan.VarArgs.IsVariadic = true;
+    Plan.VarArgs.FixedArgumentCount = Request.FixedArgumentCount;
+  }
+  Plan.ByValCopies = std::move(ByValCopies);
+  Plan.PreservedMask = createCallPreservedMask((*Convention)->ID);
+  if (!Plan.PreservedMask)
+    return makeUnexpected(ErrorCode::InternalError, "RISC-V call-preserved mask is unavailable");
+  Plan.Callee = std::move(EffectiveCallee);
+  Plan.TailDecision = Decision;
+  Plan.TailRejectionReason = std::move(RejectionReason);
+  return CallLoweringPlan::create(std::move(Plan));
+}
+
+Expected<CallLoweringPlan>
+RISCVCallLowering::planReturn(const ReturnLoweringRequest &Request) const {
+  auto Convention = riscvConvention(Request.Convention);
+  if (!Convention)
+    return std::unexpected(Convention.error());
+  auto Values = assignReturns(Request.Values, true, **Convention);
+  if (!Values)
+    return std::unexpected(Values.error());
+
+  CallLoweringPlanDesc Plan;
+  Plan.Kind = CallLoweringPlanKind::Return;
+  Plan.Target = PlanTarget;
+  Plan.Convention = Request.Convention;
+  Plan.ReturnAssignments = std::move(*Values);
+  return CallLoweringPlan::create(std::move(Plan));
+}
+
+[[nodiscard]] CallValueTransform outgoingTransform(const CCAssignment &Assignment) noexcept {
+  if (Assignment.CustomFlags.has(CCAssignmentFlag::Bitcast))
+    return CallValueTransform::Bitcast;
+  switch (Assignment.Extension) {
+  case ABIExtensionKind::None:
+    return CallValueTransform::Full;
+  case ABIExtensionKind::AnyExtend:
+    return CallValueTransform::AnyExtend;
+  case ABIExtensionKind::SignExtend:
+    return CallValueTransform::SignExtend;
+  case ABIExtensionKind::ZeroExtend:
+    return CallValueTransform::ZeroExtend;
+  case ABIExtensionKind::Count:
+    break;
+  }
+  return CallValueTransform::Full;
+}
+
+[[nodiscard]] CallValueTransform incomingTransform(const CCAssignment &Assignment) noexcept {
+  if (Assignment.CustomFlags.has(CCAssignmentFlag::Bitcast))
+    return CallValueTransform::Bitcast;
+  if (Assignment.ValueType.scalarBitWidth() < Assignment.LocationType.scalarBitWidth())
+    return CallValueTransform::Truncate;
+  return CallValueTransform::Full;
+}
+
+[[nodiscard]] Expected<void> addUnsignedImmediate(MachineInstrBuilder &Builder,
+                                                  std::uint64_t Value) {
+  auto Immediate = IntegerImmediate::fromUnsigned(64, Value);
+  if (!Immediate)
+    return std::unexpected(Immediate.error());
+  return Builder.addIntegerImmediate(std::move(*Immediate));
+}
+
+[[nodiscard]] Expected<const MachineMemOperand *>
+createMemoryOperand(MachineFunction &Function, MemoryAccessFlags Accesses, std::uint64_t SizeBytes,
+                    Alignment Align, MemoryBase Base, std::int64_t Offset = 0) {
+  auto Size = MemorySize::fixed(SizeBytes);
+  if (!Size)
+    return std::unexpected(Size.error());
+  MachineMemOperandDesc Description;
+  Description.Accesses = Accesses;
+  Description.Size = *Size;
+  Description.Align = Align;
+  Description.Base = std::move(Base);
+  Description.Offset = Offset;
+  return Function.createMemOperand(std::move(Description));
+}
+
+[[nodiscard]] Expected<void> emitCCCopy(MachineIRBuilder &Builder, Register Destination,
+                                        Register Source, CallValueTransform Transform) {
+  const Opcode CopyOpcode = Transform == CallValueTransform::Full
+                                ? opcode(GenericOpcode::COPY)
+                                : targetOpcode(InstructionOpcode::PseudoCC_COPY);
+  auto Instruction = Builder.buildInstr(CopyOpcode);
+  if (!Instruction)
+    return std::unexpected(Instruction.error());
+  auto Def = Instruction->addDef(Destination);
+  if (!Def)
+    return std::unexpected(Def.error());
+  auto Use = Instruction->addUse(Source);
+  if (!Use)
+    return std::unexpected(Use.error());
+  if (Transform != CallValueTransform::Full) {
+    auto Action = Instruction->addImmediate(static_cast<std::int64_t>(Transform));
+    if (!Action)
+      return std::unexpected(Action.error());
+  }
+  auto Built = Instruction->build();
+  if (!Built)
+    return std::unexpected(Built.error());
+  return {};
+}
+
+[[nodiscard]] Expected<void> emitStackLoad(MachineIRBuilder &Builder,
+                                           const CCAssignment &Assignment, FrameIndex Object,
+                                           std::uint64_t ObjectOffset,
+                                           const MachineMemOperand &Memory) {
+  auto Instruction = Builder.buildInstr(targetOpcode(InstructionOpcode::PseudoLOAD_STACK_ARG));
+  if (!Instruction)
+    return std::unexpected(Instruction.error());
+  auto Def = Instruction->addDef(Assignment.OriginalValue);
+  if (!Def)
+    return std::unexpected(Def.error());
+  auto Frame = Instruction->addFrameIndex(Object);
+  if (!Frame)
+    return std::unexpected(Frame.error());
+  auto Offset = addUnsignedImmediate(*Instruction, ObjectOffset);
+  if (!Offset)
+    return std::unexpected(Offset.error());
+  auto Transform =
+      Instruction->addImmediate(static_cast<std::int64_t>(incomingTransform(Assignment)));
+  if (!Transform)
+    return std::unexpected(Transform.error());
+  auto Mem = Instruction->addMemOperand(Memory);
+  if (!Mem)
+    return std::unexpected(Mem.error());
+  auto Built = Instruction->build();
+  if (!Built)
+    return std::unexpected(Built.error());
+  return {};
+}
+
+[[nodiscard]] Expected<void> emitStackStore(MachineIRBuilder &Builder, Register Source,
+                                            FrameIndex Object, std::uint64_t ObjectOffset,
+                                            const MachineMemOperand &Memory,
+                                            CallValueTransform Transform) {
+  auto Instruction = Builder.buildInstr(targetOpcode(InstructionOpcode::PseudoSTORE_STACK_ARG));
+  if (!Instruction)
+    return std::unexpected(Instruction.error());
+  auto Use = Instruction->addUse(Source);
+  if (!Use)
+    return std::unexpected(Use.error());
+  auto Frame = Instruction->addFrameIndex(Object);
+  if (!Frame)
+    return std::unexpected(Frame.error());
+  auto Offset = addUnsignedImmediate(*Instruction, ObjectOffset);
+  if (!Offset)
+    return std::unexpected(Offset.error());
+  auto Action = Instruction->addImmediate(static_cast<std::int64_t>(Transform));
+  if (!Action)
+    return std::unexpected(Action.error());
+  auto Mem = Instruction->addMemOperand(Memory);
+  if (!Mem)
+    return std::unexpected(Mem.error());
+  auto Built = Instruction->build();
+  if (!Built)
+    return std::unexpected(Built.error());
+  return {};
+}
+
+[[nodiscard]] Expected<void> ensureFunctionLiveIn(MachineRegisterInfo &Registers,
+                                                  PhysicalRegister Reg,
+                                                  std::optional<Register> InitialValue) {
+  const auto Existing =
+      std::ranges::find(Registers.liveIns(), Reg, &PhysicalRegisterLiveIn::PhysReg);
+  if (Existing == Registers.liveIns().end())
+    return Registers.addLiveIn(Reg, InitialValue);
+  if (InitialValue && Existing->InitialVirtualRegister &&
+      Existing->InitialVirtualRegister != InitialValue) {
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "physical formal register already maps to another virtual live-in");
+  }
+  return {};
+}
+
+[[nodiscard]] Expected<void> ensureBlockLiveIn(MachineBasicBlock &Block, PhysicalRegister Reg) {
+  return Block.addLiveIn(Reg, LaneMask::all());
+}
+
+[[nodiscard]] Expected<void> verifyPlanRegisters(const CallLoweringPlan &Plan,
+                                                 const MachineFunction &Function,
+                                                 const TargetDescription &Target) {
+  const MachineRegisterInfo &Registers = Function.registerInfo();
+  const auto Verify = [&Registers, &Target](const CCAssignment &Assignment) -> Expected<void> {
+    if (Assignment.LocationRegister &&
+        !Target.registers().isKnownPhysicalRegister(*Assignment.LocationRegister)) {
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "call-lowering plan names an unknown ABI location register");
+    }
+    const Register Reg = Assignment.OriginalValue;
+    if (Reg.isNoRegister())
+      return {};
+    if (Reg.isPhysical()) {
+      if (!Target.registers().isKnownPhysicalRegister(*Reg.physicalRegister())) {
+        return makeUnexpected(ErrorCode::VerificationError,
+                              "call-lowering plan names an unknown physical value register");
+      }
+      return {};
+    }
+    auto Info = Registers.virtualRegisterInfo(Reg);
+    if (!Info)
+      return std::unexpected(Info.error());
+    if (!Info->get().Type || *Info->get().Type != Assignment.ValueType) {
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "call-lowering plan register LLT differs from MachineRegisterInfo");
+    }
+    return {};
+  };
+  for (const CCAssignment &Assignment : Plan.argumentAssignments()) {
+    auto Result = Verify(Assignment);
+    if (!Result)
+      return std::unexpected(Result.error());
+  }
+  for (const CCAssignment &Assignment : Plan.returnAssignments()) {
+    auto Result = Verify(Assignment);
+    if (!Result)
+      return std::unexpected(Result.error());
+  }
+  if (const MachineCallTarget *Callee = Plan.callee();
+      Callee != nullptr && Callee->kind() == MachineCallTargetKind::IndirectRegister) {
+    const Register CalleeReg = Callee->indirectRegister();
+    if (CalleeReg.isPhysical()) {
+      const std::optional<PhysicalRegister> Physical = CalleeReg.physicalRegister();
+      if (!Physical || !Target.registers().isKnownPhysicalRegister(*Physical)) {
+        return makeUnexpected(ErrorCode::VerificationError,
+                              "indirect RISC-V callee names an unknown physical register");
+      }
+    } else {
+      auto Info = Registers.virtualRegisterInfo(CalleeReg);
+      if (!Info)
+        return std::unexpected(Info.error());
+      const std::optional<LLT> Type = Info->get().Type;
+      if (!Type || ((!Type->isPointer() && !Type->isInteger()) || Type->scalarBitWidth() != 64)) {
+        return makeUnexpected(ErrorCode::VerificationError,
+                              "indirect RISC-V callee requires a 64-bit pointer or integer LLT");
+      }
+    }
+  }
+  return {};
+}
+
+[[nodiscard]] Expected<void> verifyRequiredOpcodes(const MachineIRBuilder &Builder) {
+  static constexpr std::array Required{
+      InstructionOpcode::PseudoCC_COPY,
+      InstructionOpcode::PseudoLOAD_STACK_ARG,
+      InstructionOpcode::PseudoSTORE_STACK_ARG,
+      InstructionOpcode::PseudoBYVAL_COPY,
+      InstructionOpcode::ADJCALLSTACKDOWN,
+      InstructionOpcode::ADJCALLSTACKUP,
+      InstructionOpcode::PseudoCALL,
+      InstructionOpcode::PseudoTAIL,
+      InstructionOpcode::PseudoRET,
+  };
+  for (const InstructionOpcode OpcodeValue : Required) {
+    if (Builder.target().instructionDescriptor(targetOpcode(OpcodeValue)) == nullptr) {
+      return makeUnexpected(ErrorCode::InternalError,
+                            "RISC-V target description lacks a call-lowering pseudo");
+    }
+  }
+  return {};
+}
+
+[[nodiscard]] Expected<void> verifyFormalLiveIns(const CallLoweringPlan &Plan,
+                                                 const MachineFunction &Function) {
+  if (Plan.kind() != CallLoweringPlanKind::FormalArguments)
+    return {};
+
+  const auto LiveIns = Function.registerInfo().liveIns();
+  for (const CCAssignment &Assignment : Plan.argumentAssignments()) {
+    if (!Assignment.LocationRegister || !Assignment.OriginalValue.isVirtual())
+      continue;
+    const auto Existing =
+        std::ranges::find(LiveIns, *Assignment.LocationRegister, &PhysicalRegisterLiveIn::PhysReg);
+    if (Existing != LiveIns.end() && Existing->InitialVirtualRegister &&
+        *Existing->InitialVirtualRegister != Assignment.OriginalValue) {
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "physical formal register already maps to another virtual live-in");
+    }
+  }
+  return {};
+}
+
+[[nodiscard]] Expected<void> preflightPlan(MachineIRBuilder &Builder,
+                                           const CallLoweringPlan &Plan) {
+  auto Convention = riscvConvention(Plan.convention());
+  if (!Convention)
+    return std::unexpected(Convention.error());
+  if (Plan.target() != PlanTarget) {
+    return makeUnexpected(ErrorCode::InvalidArgument,
+                          "call-lowering plan belongs to another target or ABI");
+  }
+  auto InsertPoint = Builder.verifyInsertPoint();
+  if (!InsertPoint)
+    return std::unexpected(InsertPoint.error());
+  auto Opcodes = verifyRequiredOpcodes(Builder);
+  if (!Opcodes)
+    return std::unexpected(Opcodes.error());
+  auto Registers = verifyPlanRegisters(Plan, Builder.function(), Builder.target());
+  if (!Registers)
+    return std::unexpected(Registers.error());
+  auto LiveIns = verifyFormalLiveIns(Plan, Builder.function());
+  if (!LiveIns)
+    return std::unexpected(LiveIns.error());
+  for (const CCAssignment &Assignment : Plan.argumentAssignments()) {
+    if (Assignment.StackOffsetBytes &&
+        *Assignment.StackOffsetBytes >
+            static_cast<std::uint64_t>(std::numeric_limits<std::int64_t>::max())) {
+      return makeUnexpected(ErrorCode::Unsupported,
+                            "RISC-V stack assignment offset exceeds the frame-index range");
+    }
+  }
+  for (const ByValCopyAssignment &Copy : Plan.byValCopies()) {
+    if (Copy.StackOffsetBytes >
+        static_cast<std::uint64_t>(std::numeric_limits<std::int64_t>::max())) {
+      return makeUnexpected(ErrorCode::Unsupported,
+                            "RISC-V byval offset exceeds the frame-index range");
+    }
+  }
+  if (Plan.kind() != CallLoweringPlanKind::Return && Builder.function().frameInfo().isFrozen()) {
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "call lowering cannot modify a frozen machine frame");
+  }
+  if (Plan.kind() == CallLoweringPlanKind::FormalArguments &&
+      (Builder.function().firstBlock() == nullptr ||
+       Builder.insertionBlock() != Builder.function().firstBlock())) {
+    return makeUnexpected(ErrorCode::InvalidArgument,
+                          "formal arguments must be lowered in the entry machine block");
+  }
+  return {};
+}
+
+[[nodiscard]] Expected<void> updateMaximumFrameAlignment(MachineFrameState &State,
+                                                         Alignment Required) {
+  if (!State.MaxAlignment || State.MaxAlignment->bytes() < Required.bytes())
+    State.MaxAlignment = Required;
+  return {};
+}
+
+[[nodiscard]] Expected<void> emitFormalArguments(MachineIRBuilder &Builder,
+                                                 const CallLoweringPlan &Plan) {
+  MachineFunction &Function = Builder.function();
+  MachineBasicBlock &Entry = *Function.firstBlock();
+  MachineRegisterInfo &Registers = Function.registerInfo();
+  MachineFrameInfo &Frame = Function.frameInfo();
+  Alignment RequiredFrameAlignment;
+
+  for (const CCAssignment &Assignment : Plan.argumentAssignments()) {
+    if (Assignment.LocationRegister) {
+      const PhysicalRegister Location = *Assignment.LocationRegister;
+      const std::optional<Register> Initial =
+          Assignment.OriginalValue.isVirtual() ? std::optional<Register>(Assignment.OriginalValue)
+                                               : std::nullopt;
+      auto FunctionLiveIn = ensureFunctionLiveIn(Registers, Location, Initial);
+      if (!FunctionLiveIn)
+        return std::unexpected(FunctionLiveIn.error());
+      auto BlockLiveIn = ensureBlockLiveIn(Entry, Location);
+      if (!BlockLiveIn)
+        return std::unexpected(BlockLiveIn.error());
+      auto Copy = emitCCCopy(Builder, Assignment.OriginalValue, Register::fromPhysical(Location),
+                             incomingTransform(Assignment));
+      if (!Copy)
+        return std::unexpected(Copy.error());
+      continue;
+    }
+
+    if (!Assignment.StackOffsetBytes) {
+      return makeUnexpected(ErrorCode::InternalError,
+                            "verified formal stack assignment has no stack offset");
+    }
+    StackObjectDesc Object;
+    Object.Kind = StackObjectKind::FixedABI;
+    Object.Size = Assignment.StackSlotSizeBytes;
+    Object.Align = Assignment.LocationAlignment;
+    Object.Offset = static_cast<std::int64_t>(*Assignment.StackOffsetBytes);
+    Object.IsImmutable = true;
+    Object.TargetFlags = stackObjectFlag(CallStackObjectFlag::IncomingArgument);
+    auto FrameObject = Frame.createObject(std::move(Object));
+    if (!FrameObject)
+      return std::unexpected(FrameObject.error());
+    auto Memory =
+        createMemoryOperand(Function, {MemoryAccessFlag::Load}, Assignment.LocationSizeBytes,
+                            Assignment.LocationAlignment, MemoryBase::frameIndex(*FrameObject));
+    if (!Memory)
+      return std::unexpected(Memory.error());
+    auto Load = emitStackLoad(Builder, Assignment, *FrameObject, 0, **Memory);
+    if (!Load)
+      return std::unexpected(Load.error());
+    if (RequiredFrameAlignment.bytes() < Assignment.LocationAlignment.bytes())
+      RequiredFrameAlignment = Assignment.LocationAlignment;
+  }
+
+  const VarArgLoweringState &VarArgs = Plan.varArgs();
+  if (!VarArgs.IsVariadic) {
+    if (RequiredFrameAlignment.bytes() == 1)
+      return {};
+    MachineFrameState State = Frame.state();
+    auto AlignmentUpdate = updateMaximumFrameAlignment(State, RequiredFrameAlignment);
+    if (!AlignmentUpdate)
+      return std::unexpected(AlignmentUpdate.error());
+    return Frame.setState(std::move(State));
+  }
+
+  auto Align8 = alignment(XLenBytes);
+  auto Align16 = alignment(ABIStackAlignmentBytes);
+  if (!Align8 || !Align16)
+    return makeUnexpected(ErrorCode::InternalError, "RISC-V ABI primitive type facts are invalid");
+
+  FrameIndex SaveObject;
+  if (VarArgs.RegisterSaveAreaSizeBytes == 0) {
+    StackObjectDesc Object;
+    Object.Kind = StackObjectKind::FixedABI;
+    Object.Size = XLenBytes;
+    Object.Align = *Align8;
+    Object.Offset = VarArgs.RegisterSaveAreaOffsetBytes;
+    Object.IsImmutable = true;
+    Object.TargetFlags = stackObjectFlag(CallStackObjectFlag::VarArgSaveArea);
+    auto Created = Frame.createObject(std::move(Object));
+    if (!Created)
+      return std::unexpected(Created.error());
+    SaveObject = *Created;
+  } else {
+    StackObjectDesc Object;
+    Object.Kind = StackObjectKind::FixedABI;
+    Object.Size = VarArgs.RegisterSaveAreaSizeBytes;
+    Object.Align = VarArgs.FirstUnallocatedIntegerRegister % 2 == 0 ? *Align16 : *Align8;
+    Object.Offset = VarArgs.RegisterSaveAreaOffsetBytes;
+    Object.TargetFlags = stackObjectFlag(CallStackObjectFlag::VarArgSaveArea);
+    auto Created = Frame.createObject(std::move(Object));
+    if (!Created)
+      return std::unexpected(Created.error());
+    SaveObject = *Created;
+
+    if (VarArgs.RegisterSavePaddingBytes != 0) {
+      StackObjectDesc Padding;
+      Padding.Kind = StackObjectKind::FixedABI;
+      Padding.Size = VarArgs.RegisterSavePaddingBytes;
+      Padding.Align = *Align8;
+      Padding.Offset = VarArgs.RegisterSaveAreaOffsetBytes -
+                       static_cast<std::int64_t>(VarArgs.RegisterSavePaddingBytes);
+      Padding.IsImmutable = true;
+      Padding.TargetFlags = stackObjectFlag(CallStackObjectFlag::VarArgAlignmentPadding);
+      auto CreatedPadding = Frame.createObject(std::move(Padding));
+      if (!CreatedPadding)
+        return std::unexpected(CreatedPadding.error());
+    }
+  }
+
+  for (const VarArgRegisterSave Save : VarArgs.RegistersToSave) {
+    auto FunctionLiveIn = ensureFunctionLiveIn(Registers, Save.Reg, std::nullopt);
+    if (!FunctionLiveIn)
+      return std::unexpected(FunctionLiveIn.error());
+    auto BlockLiveIn = ensureBlockLiveIn(Entry, Save.Reg);
+    if (!BlockLiveIn)
+      return std::unexpected(BlockLiveIn.error());
+    auto Memory = createMemoryOperand(Function, {MemoryAccessFlag::Store}, XLenBytes, *Align8,
+                                      MemoryBase::frameIndex(SaveObject),
+                                      static_cast<std::int64_t>(Save.OffsetBytes));
+    if (!Memory)
+      return std::unexpected(Memory.error());
+    auto Store = emitStackStore(Builder, Register::fromPhysical(Save.Reg), SaveObject,
+                                Save.OffsetBytes, **Memory, CallValueTransform::Full);
+    if (!Store)
+      return std::unexpected(Store.error());
+  }
+
+  MachineFrameState State = Frame.state();
+  const Alignment VarArgAlignment =
+      RequiredFrameAlignment.bytes() > Align16->bytes() ? RequiredFrameAlignment : *Align16;
+  auto AlignmentUpdate = updateMaximumFrameAlignment(State, VarArgAlignment);
+  if (!AlignmentUpdate)
+    return std::unexpected(AlignmentUpdate.error());
+  return Frame.setState(std::move(State));
+}
+
+[[nodiscard]] Expected<void> emitCallFrameAdjustment(MachineIRBuilder &Builder,
+                                                     InstructionOpcode OpcodeValue,
+                                                     const CallFrameRequirements &Frame) {
+  if (Frame.SizeBytes > static_cast<std::uint64_t>(std::numeric_limits<std::int64_t>::max())) {
+    return makeUnexpected(ErrorCode::Unsupported,
+                          "RISC-V call-frame size does not fit a MIR immediate");
+  }
+  auto Instruction = Builder.buildInstr(targetOpcode(OpcodeValue));
+  if (!Instruction)
+    return std::unexpected(Instruction.error());
+  auto Size = Instruction->addImmediate(static_cast<std::int64_t>(Frame.SizeBytes));
+  if (!Size)
+    return std::unexpected(Size.error());
+  auto PreparedBytes = Instruction->addImmediate(0);
+  if (!PreparedBytes)
+    return std::unexpected(PreparedBytes.error());
+  auto Built = Instruction->build();
+  if (!Built)
+    return std::unexpected(Built.error());
+  return {};
+}
+
+[[nodiscard]] Expected<std::unordered_map<std::size_t, Register>>
+emitByValCopies(MachineIRBuilder &Builder, const CallLoweringPlan &Plan) {
+  MachineFunction &Function = Builder.function();
+  MachineFrameInfo &Frame = Function.frameInfo();
+  std::unordered_map<std::size_t, Register> Replacements;
+  for (const ByValCopyAssignment Copy : Plan.byValCopies()) {
+    StackObjectDesc Object;
+    Object.Kind = StackObjectKind::Target;
+    Object.Size = Copy.SizeBytes;
+    Object.Align = Copy.Align;
+    Object.LocalOffset = static_cast<std::int64_t>(Copy.StackOffsetBytes);
+    Object.TargetFlags = stackObjectFlag(CallStackObjectFlag::ByValCopy);
+    auto FrameObject = Frame.createObject(std::move(Object));
+    if (!FrameObject)
+      return std::unexpected(FrameObject.error());
+    auto Temporary = Function.registerInfo().createVirtualRegister(Copy.PointerType);
+    if (!Temporary)
+      return std::unexpected(Temporary.error());
+
+    auto SourceMemory = createMemoryOperand(Function, {MemoryAccessFlag::Load}, Copy.SizeBytes,
+                                            Copy.Align, MemoryBase{});
+    if (!SourceMemory)
+      return std::unexpected(SourceMemory.error());
+    auto DestinationMemory =
+        createMemoryOperand(Function, {MemoryAccessFlag::Store}, Copy.SizeBytes, Copy.Align,
+                            MemoryBase::frameIndex(*FrameObject));
+    if (!DestinationMemory)
+      return std::unexpected(DestinationMemory.error());
+
+    auto Instruction = Builder.buildInstr(targetOpcode(InstructionOpcode::PseudoBYVAL_COPY));
+    if (!Instruction)
+      return std::unexpected(Instruction.error());
+    auto Def = Instruction->addDef(*Temporary);
+    if (!Def)
+      return std::unexpected(Def.error());
+    auto Use = Instruction->addUse(Copy.Source);
+    if (!Use)
+      return std::unexpected(Use.error());
+    auto FrameOperand = Instruction->addFrameIndex(*FrameObject);
+    if (!FrameOperand)
+      return std::unexpected(FrameOperand.error());
+    auto Size = addUnsignedImmediate(*Instruction, Copy.SizeBytes);
+    if (!Size)
+      return std::unexpected(Size.error());
+    auto Align = addUnsignedImmediate(*Instruction, Copy.Align.bytes());
+    if (!Align)
+      return std::unexpected(Align.error());
+    auto SourceMem = Instruction->addMemOperand(**SourceMemory);
+    if (!SourceMem)
+      return std::unexpected(SourceMem.error());
+    auto DestinationMem = Instruction->addMemOperand(**DestinationMemory);
+    if (!DestinationMem)
+      return std::unexpected(DestinationMem.error());
+    auto Built = Instruction->build();
+    if (!Built)
+      return std::unexpected(Built.error());
+    Replacements.emplace(Copy.ArgumentAssignmentIndex, *Temporary);
+  }
+  return Replacements;
+}
+
+[[nodiscard]] Expected<void>
+emitOutgoingArguments(MachineIRBuilder &Builder, const CallLoweringPlan &Plan,
+                      const std::unordered_map<std::size_t, Register> &Replacements) {
+  MachineFunction &Function = Builder.function();
+  MachineFrameInfo &Frame = Function.frameInfo();
+  const auto Assignments = Plan.argumentAssignments();
+  for (std::size_t Index = 0; Index < Assignments.size(); ++Index) {
+    const CCAssignment &Assignment = Assignments[Index];
+    const auto Replacement = Replacements.find(Index);
+    const Register Source =
+        Replacement == Replacements.end() ? Assignment.OriginalValue : Replacement->second;
+    if (Assignment.LocationRegister) {
+      auto Copy = emitCCCopy(Builder, Register::fromPhysical(*Assignment.LocationRegister), Source,
+                             outgoingTransform(Assignment));
+      if (!Copy)
+        return std::unexpected(Copy.error());
+      continue;
+    }
+
+    if (!Assignment.StackOffsetBytes) {
+      return makeUnexpected(ErrorCode::InternalError,
+                            "verified outgoing stack assignment has no stack offset");
+    }
+    StackObjectDesc Object;
+    Object.Size = Assignment.StackSlotSizeBytes;
+    Object.Align = Assignment.LocationAlignment;
+    Object.TargetFlags = stackObjectFlag(CallStackObjectFlag::OutgoingArgument);
+    if (Plan.tailDecision() == TailCallKind::NotTail) {
+      Object.Kind = StackObjectKind::Target;
+      Object.LocalOffset = static_cast<std::int64_t>(*Assignment.StackOffsetBytes);
+    } else {
+      Object.Kind = StackObjectKind::FixedABI;
+      Object.Offset = static_cast<std::int64_t>(*Assignment.StackOffsetBytes);
+    }
+    auto FrameObject = Frame.createObject(std::move(Object));
+    if (!FrameObject)
+      return std::unexpected(FrameObject.error());
+    auto Memory =
+        createMemoryOperand(Function, {MemoryAccessFlag::Store}, Assignment.LocationSizeBytes,
+                            Assignment.LocationAlignment, MemoryBase::frameIndex(*FrameObject));
+    if (!Memory)
+      return std::unexpected(Memory.error());
+    auto Store =
+        emitStackStore(Builder, Source, *FrameObject, 0, **Memory, outgoingTransform(Assignment));
+    if (!Store)
+      return std::unexpected(Store.error());
+  }
+  return {};
+}
+
+[[nodiscard]] Expected<void> addCallTarget(MachineInstrBuilder &Instruction,
+                                           const MachineCallTarget &Callee) {
+  switch (Callee.kind()) {
+  case MachineCallTargetKind::GlobalSymbol:
+    return Instruction.addGlobalSymbol(std::string(Callee.symbolName()), 0, Callee.targetFlags());
+  case MachineCallTargetKind::ExternalSymbol:
+    return Instruction.addExternalSymbol(std::string(Callee.symbolName()), 0, Callee.targetFlags());
+  case MachineCallTargetKind::IndirectRegister:
+    return Instruction.addUse(Callee.indirectRegister());
+  case MachineCallTargetKind::Invalid:
+    break;
+  }
+  return makeUnexpected(ErrorCode::InternalError,
+                        "verified call-lowering plan has an invalid callee");
+}
+
+[[nodiscard]] Expected<PhysicalRegister> requiredLocationRegister(const CCAssignment &Assignment) {
+  if (!Assignment.LocationRegister) {
+    return makeUnexpected(ErrorCode::InternalError,
+                          "verified return assignment has no location register");
+  }
+  return *Assignment.LocationRegister;
+}
+
+[[nodiscard]] Expected<void> emitCallInstruction(MachineIRBuilder &Builder,
+                                                 const CallLoweringPlan &Plan) {
+  const bool IsTail = Plan.tailDecision() != TailCallKind::NotTail;
+  auto Instruction = Builder.buildInstr(
+      targetOpcode(IsTail ? InstructionOpcode::PseudoTAIL : InstructionOpcode::PseudoCALL));
+  if (!Instruction)
+    return std::unexpected(Instruction.error());
+  auto Callee = addCallTarget(*Instruction, *Plan.callee());
+  if (!Callee)
+    return std::unexpected(Callee.error());
+  auto Mask = Instruction->addRegisterMask(Plan.preservedMask());
+  if (!Mask)
+    return std::unexpected(Mask.error());
+
+  std::unordered_set<PhysicalRegister> DynamicUses;
+  for (const CCAssignment &Assignment : Plan.argumentAssignments()) {
+    if (Assignment.LocationRegister && DynamicUses.insert(*Assignment.LocationRegister).second) {
+      auto Use = Instruction->addImplicitUse(Register::fromPhysical(*Assignment.LocationRegister));
+      if (!Use)
+        return std::unexpected(Use.error());
+    }
+  }
+  std::unordered_set<PhysicalRegister> DynamicDefs;
+  if (!IsTail) {
+    for (const CCAssignment &Assignment : Plan.returnAssignments()) {
+      if (Assignment.LocationRegister && DynamicDefs.insert(*Assignment.LocationRegister).second) {
+        auto Def =
+            Instruction->addImplicitDef(Register::fromPhysical(*Assignment.LocationRegister));
+        if (!Def)
+          return std::unexpected(Def.error());
+      }
+    }
+  }
+  auto Built = Instruction->build();
+  if (!Built)
+    return std::unexpected(Built.error());
+  return {};
+}
+
+[[nodiscard]] Expected<void> emitCall(MachineIRBuilder &Builder, const CallLoweringPlan &Plan) {
+  if (Plan.callFrame().EmitSetup) {
+    auto Down =
+        emitCallFrameAdjustment(Builder, InstructionOpcode::ADJCALLSTACKDOWN, Plan.callFrame());
+    if (!Down)
+      return std::unexpected(Down.error());
+  }
+
+  auto Replacements = emitByValCopies(Builder, Plan);
+  if (!Replacements)
+    return std::unexpected(Replacements.error());
+  auto Arguments = emitOutgoingArguments(Builder, Plan, *Replacements);
+  if (!Arguments)
+    return std::unexpected(Arguments.error());
+  auto Call = emitCallInstruction(Builder, Plan);
+  if (!Call)
+    return std::unexpected(Call.error());
+
+  if (Plan.callFrame().EmitDestroy) {
+    auto Up = emitCallFrameAdjustment(Builder, InstructionOpcode::ADJCALLSTACKUP, Plan.callFrame());
+    if (!Up)
+      return std::unexpected(Up.error());
+  }
+  if (Plan.tailDecision() == TailCallKind::NotTail) {
+    for (const CCAssignment &Assignment : Plan.returnAssignments()) {
+      auto Location = requiredLocationRegister(Assignment);
+      if (!Location)
+        return std::unexpected(Location.error());
+      auto Copy = emitCCCopy(Builder, Assignment.OriginalValue, Register::fromPhysical(*Location),
+                             incomingTransform(Assignment));
+      if (!Copy)
+        return std::unexpected(Copy.error());
+    }
+  }
+
+  MachineFrameInfo &Frame = Builder.function().frameInfo();
+  MachineFrameState State = Frame.state();
+  State.HasCalls = true;
+  State.HasTailCall = State.HasTailCall || Plan.tailDecision() != TailCallKind::NotTail;
+  State.AdjustsStack =
+      State.AdjustsStack || (Plan.callFrame().EmitSetup && Plan.callFrame().SizeBytes != 0);
+  State.HasMustTailInVarArgFunc =
+      State.HasMustTailInVarArgFunc ||
+      (Plan.tailDecision() == TailCallKind::MustTail && Plan.varArgs().IsVariadic);
+  if (Plan.tailDecision() == TailCallKind::NotTail) {
+    State.MaxCallFrameSize =
+        std::max(State.MaxCallFrameSize.value_or(0), Plan.callFrame().SizeBytes);
+  }
+  auto AlignmentUpdate = updateMaximumFrameAlignment(State, Plan.callFrame().Align);
+  if (!AlignmentUpdate)
+    return std::unexpected(AlignmentUpdate.error());
+  return Frame.setState(std::move(State));
+}
+
+[[nodiscard]] Expected<void> emitReturn(MachineIRBuilder &Builder, const CallLoweringPlan &Plan) {
+  for (const CCAssignment &Assignment : Plan.returnAssignments()) {
+    auto Location = requiredLocationRegister(Assignment);
+    if (!Location)
+      return std::unexpected(Location.error());
+    auto Copy = emitCCCopy(Builder, Register::fromPhysical(*Location), Assignment.OriginalValue,
+                           outgoingTransform(Assignment));
+    if (!Copy)
+      return std::unexpected(Copy.error());
+  }
+
+  auto Return = Builder.buildInstr(targetOpcode(InstructionOpcode::PseudoRET));
+  if (!Return)
+    return std::unexpected(Return.error());
+  for (const CCAssignment &Assignment : Plan.returnAssignments()) {
+    auto Location = requiredLocationRegister(Assignment);
+    if (!Location)
+      return std::unexpected(Location.error());
+    auto Use = Return->addImplicitUse(Register::fromPhysical(*Location));
+    if (!Use)
+      return std::unexpected(Use.error());
+  }
+  auto Built = Return->build();
+  if (!Built)
+    return std::unexpected(Built.error());
+  return {};
+}
+
+Expected<void> RISCVCallLowering::emitPlan(MachineIRBuilder &Builder,
+                                           const CallLoweringPlan &Plan) const {
+  auto Preflight = preflightPlan(Builder, Plan);
+  if (!Preflight)
+    return std::unexpected(Preflight.error());
+  switch (Plan.kind()) {
+  case CallLoweringPlanKind::FormalArguments:
+    return emitFormalArguments(Builder, Plan);
+  case CallLoweringPlanKind::Call:
+    return emitCall(Builder, Plan);
+  case CallLoweringPlanKind::Return:
+    return emitReturn(Builder, Plan);
+  }
+  return makeUnexpected(ErrorCode::InternalError, "call-lowering plan kind is invalid");
+}
+
+} // namespace rcc::riscv
diff --git a/backend/lib/Target/RISCV/GISel/RISCVCallLowering.h b/backend/lib/Target/RISCV/GISel/RISCVCallLowering.h
new file mode 100644
index 0000000..dad0875
--- /dev/null
+++ b/backend/lib/Target/RISCV/GISel/RISCVCallLowering.h
@@ -0,0 +1,53 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef RCC_LIB_TARGET_RISCV_GISEL_RISCVCALLLOWERING_H
+#define RCC_LIB_TARGET_RISCV_GISEL_RISCVCALLLOWERING_H
+
+#include "rcc/CodeGen/TargetCallLowering.h"
+#include "rcc/Target/RISCV/RISCVInstructions.h"
+
+#include <cstdint>
+
+namespace rcc::riscv {
+
+enum class CallTargetFlag : std::uint32_t {
+  DirectCall = 1U << 0U,
+};
+
+enum class CallStackObjectFlag : std::uint64_t {
+  IncomingArgument = 1ULL << 0U,
+  OutgoingArgument = 1ULL << 1U,
+  ByValCopy = 1ULL << 2U,
+  VarArgSaveArea = 1ULL << 3U,
+  VarArgAlignmentPadding = 1ULL << 4U,
+};
+
+enum class CallValueTransform : std::uint8_t {
+  Full,
+  AnyExtend,
+  SignExtend,
+  ZeroExtend,
+  Truncate,
+  Bitcast,
+};
+
+[[nodiscard]] constexpr CallingConventionID
+callingConventionID(CallingConvention Convention) noexcept {
+  return CallingConventionID::fromValue(static_cast<std::uint32_t>(Convention));
+}
+
+class RISCVCallLowering final : public TargetCallLowering {
+public:
+  [[nodiscard]] Expected<CallLoweringPlan>
+  planFormalArguments(const FormalArgumentRequest &Request) const override;
+  [[nodiscard]] Expected<CallLoweringPlan>
+  planCall(const CallLoweringRequest &Request) const override;
+  [[nodiscard]] Expected<CallLoweringPlan>
+  planReturn(const ReturnLoweringRequest &Request) const override;
+  [[nodiscard]] Expected<void> emitPlan(MachineIRBuilder &Builder,
+                                        const CallLoweringPlan &Plan) const override;
+};
+
+} // namespace rcc::riscv
+
+#endif // RCC_LIB_TARGET_RISCV_GISEL_RISCVCALLLOWERING_H
diff --git a/backend/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp b/backend/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp
new file mode 100644
index 0000000..a2c7194
--- /dev/null
+++ b/backend/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp
@@ -0,0 +1,2627 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "RISCV.h"
+#include "RISCVRegisterBankInfo.h"
+#include "rcc/CodeGen/GenericOpcodes.h"
+#include "rcc/CodeGen/GlobalISel/InstructionSelector.h"
+#include "rcc/CodeGen/GlobalISel/LegalizerInfo.h"
+#include "rcc/CodeGen/LowLevelType.h"
+#include "rcc/CodeGen/MachineBasicBlock.h"
+#include "rcc/CodeGen/MachineFunction.h"
+#include "rcc/CodeGen/MachineIRBuilder.h"
+#include "rcc/CodeGen/MachineInstr.h"
+#include "rcc/CodeGen/MachineMemOperand.h"
+#include "rcc/CodeGen/MachineOperand.h"
+#include "rcc/CodeGen/MachineRegisterInfo.h"
+#include "rcc/CodeGen/TargetDescription.h"
+#include "rcc/Target/RISCV/RISCVSubtarget.h"
+#include "rcc/Target/RISCV/RISCVTarget.h"
+
+#include <algorithm>
+#include <array>
+#include <bit>
+#include <cstdint>
+#include <limits>
+#include <optional>
+#include <ranges>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace rcc::riscv {
+namespace {
+
+[[nodiscard]] Opcode targetOpcode(InstructionOpcode Value) noexcept {
+  return Opcode::fromValue(static_cast<std::uint32_t>(Value));
+}
+
+[[nodiscard]] std::optional<LLT> registerType(const MachineFunction &Function,
+                                              const MachineOperand &Operand) {
+  const Register *Reg = Operand.getIf<Register>();
+  if (Reg == nullptr || !Reg->isVirtual())
+    return std::nullopt;
+  auto Info = Function.registerInfo().virtualRegisterInfo(*Reg);
+  if (!Info)
+    return std::nullopt;
+  return Info->get().Type;
+}
+
+[[nodiscard]] std::optional<LLT> resultType(const MachineFunction &Function,
+                                            const MachineInstr &Instruction) {
+  const MachineOperand *Result = Instruction.operandAt(0);
+  return Result == nullptr ? std::nullopt : registerType(Function, *Result);
+}
+
+[[nodiscard]] std::optional<LLT> operandType(const MachineFunction &Function,
+                                             const MachineInstr &Instruction, std::size_t Index) {
+  const MachineOperand *Operand = Instruction.operandAt(Index);
+  return Operand == nullptr ? std::nullopt : registerType(Function, *Operand);
+}
+
+[[nodiscard]] bool isRVFloatType(LLT Type) noexcept {
+  if (!Type.isFloat())
+    return false;
+  const std::optional<FloatSemantics> Semantics = Type.floatSemantics();
+  return Semantics &&
+         (*Semantics == FloatSemantics::IEEESingle || *Semantics == FloatSemantics::IEEEDouble);
+}
+
+[[nodiscard]] bool isSupportedIntegerType(LLT Type) noexcept {
+  return ((Type.isInteger() || Type.isAnyScalar()) && Type.scalarBitWidth() != 0 &&
+          Type.scalarBitWidth() <= 64) ||
+         (Type.isPointer() && Type.scalarBitWidth() == 64);
+}
+
+[[nodiscard]] bool isSupportedIntegerScalar(LLT Type) noexcept {
+  return (Type.isInteger() || Type.isAnyScalar()) && Type.scalarBitWidth() != 0 &&
+         Type.scalarBitWidth() <= 64;
+}
+
+[[nodiscard]] bool isScalarGPRType(LLT Type) noexcept {
+  return ((Type.isInteger() || Type.isAnyScalar()) && Type.scalarBitWidth() != 0 &&
+          Type.scalarBitWidth() <= 64) ||
+         (Type.isPointer() && Type.scalarBitWidth() == 64);
+}
+
+[[nodiscard]] bool fitsSigned(std::int64_t Value, std::uint32_t Bits) noexcept {
+  const std::int64_t Minimum = -(std::int64_t{1} << (Bits - 1U));
+  const std::int64_t Maximum = (std::int64_t{1} << (Bits - 1U)) - 1;
+  return Value >= Minimum && Value <= Maximum;
+}
+
+[[nodiscard]] std::optional<std::uint64_t>
+memoryAccessBytes(const MachineInstr &Instruction) noexcept {
+  if (Instruction.memOperands().size() != 1)
+    return std::nullopt;
+  const MachineMemOperandDesc &Description = Instruction.memOperands().front()->description();
+  if (!Description.Size.isKnown() || Description.Size.isScalable())
+    return std::nullopt;
+  return Description.Size.minimumBytes();
+}
+
+[[nodiscard]] Expected<void> appendImmediate(MachineInstr &Instruction, std::int64_t Value) {
+  return Instruction.addOperand(MachineOperand::immediate(Value));
+}
+
+[[nodiscard]] Expected<void> insertPhysicalRegister(MachineInstr &Instruction, std::size_t Index,
+                                                    PhysicalRegister Reg,
+                                                    RegisterOperandFlag Flag) {
+  auto Operand = MachineOperand::registerOperand(Register::fromPhysical(Reg), {Flag});
+  if (!Operand)
+    return std::unexpected(Operand.error());
+  return Instruction.insertOperand(Index, std::move(*Operand));
+}
+
+[[nodiscard]] Expected<void> insertVirtualUse(MachineInstr &Instruction, std::size_t Index,
+                                              Register Reg) {
+  auto Operand = MachineOperand::registerOperand(Reg, {RegisterOperandFlag::Use});
+  if (!Operand)
+    return std::unexpected(Operand.error());
+  return Instruction.insertOperand(Index, std::move(*Operand));
+}
+
+[[nodiscard]] Expected<std::int64_t> signedImmediate(const MachineOperand &Operand) {
+  if (const std::int64_t *Value = Operand.getIf<std::int64_t>())
+    return *Value;
+  const auto *Integer = Operand.getIf<IntegerImmediateOperand>();
+  if (Integer == nullptr || Integer->Value.bitWidth() == 0 || Integer->Value.bitWidth() > 64)
+    return makeUnexpected(ErrorCode::Unsupported,
+                          "RISC-V constant selection requires an immediate no wider than 64 bits");
+  std::uint64_t Raw = Integer->Value.words().empty() ? 0 : Integer->Value.words().front();
+  const std::uint32_t Width = Integer->Value.bitWidth();
+  if (Width < 64 && (Raw & (std::uint64_t{1} << (Width - 1U))) != 0)
+    Raw |= ~std::uint64_t{0} << Width;
+  return std::bit_cast<std::int64_t>(Raw);
+}
+
+class RISCVInstructionSelector final : public InstructionSelector {
+public:
+  RISCVInstructionSelector(const RISCVSubtarget &Subtarget,
+                           const RISCVRegisterBankInfo &RegisterBanks)
+      : Subtarget(Subtarget), RegisterBanks(RegisterBanks) {}
+
+  [[nodiscard]] Expected<void>
+  constrainSelectedInstruction(MachineInstr &Instruction, MachineFunction &Function,
+                               const TargetDescription &Target) const override {
+    return RegisterBanks.constrainInstruction(Instruction, Function, Target);
+  }
+
+  [[nodiscard]] Expected<void> canSelect(const MachineInstr &Instruction,
+                                         const MachineFunction &Function,
+                                         const TargetDescription &Target) const override {
+    if (Instruction.opcode() == targetOpcode(InstructionOpcode::G_SLLW)) {
+      const std::optional<LLT> Type = resultType(Function, Instruction);
+      const std::optional<LLT> Left = operandType(Function, Instruction, 1);
+      const std::optional<LLT> Right = operandType(Function, Instruction, 2);
+      return Type && Left && Right && isScalarGPRType(*Type) && Type->scalarBitWidth() == 64 &&
+                     *Left == *Type && *Right == *Type
+                 ? Expected<void>{}
+                 : makeUnexpected(ErrorCode::Unsupported,
+                                  "RISC-V G_SLLW requires three 64-bit GPR operands");
+    }
+    if (!isGenericOpcode(Instruction.opcode()))
+      return {};
+    const GenericOpcode Generic = static_cast<GenericOpcode>(Instruction.opcode().value());
+    const std::optional<LLT> Type = resultType(Function, Instruction);
+    const auto Integer = [Type] { return Type && isSupportedIntegerType(*Type); };
+    const auto Float = [Type] { return Type && isRVFloatType(*Type); };
+    auto Classification = Subtarget.getLegalizerInfo()->classify(Instruction, Function, Target);
+    if (!Classification)
+      return std::unexpected(Classification.error());
+    if (*Classification != LegalizationAction::Legal) {
+      std::string Message = "RISC-V selector cannot select opcode '" +
+                            std::string(getGenericOpcodeName(Generic)) + "'";
+      if (Type)
+        Message += " with result type '" + Type->toString() + "'";
+      Message += ": the legalization rule explicitly marks this combination unsupported";
+      return makeUnexpected(ErrorCode::Unsupported, std::move(Message));
+    }
+    switch (Generic) {
+    case GenericOpcode::G_ADD:
+    case GenericOpcode::G_SUB:
+    case GenericOpcode::G_MUL:
+    case GenericOpcode::G_SDIV:
+    case GenericOpcode::G_UDIV:
+    case GenericOpcode::G_SREM:
+    case GenericOpcode::G_UREM:
+    case GenericOpcode::G_AND:
+    case GenericOpcode::G_OR:
+    case GenericOpcode::G_XOR:
+    case GenericOpcode::G_SHL:
+    case GenericOpcode::G_LSHR:
+    case GenericOpcode::G_ASHR:
+    case GenericOpcode::G_PTR_ADD:
+    case GenericOpcode::G_PTRMASK:
+    case GenericOpcode::G_CONSTANT:
+      if (!Integer())
+        return makeUnexpected(ErrorCode::Unsupported,
+                              "RISC-V selector requires an i1-i64 integer or 64-bit pointer value");
+      if (Generic == GenericOpcode::G_CONSTANT) {
+        const MachineOperand *Immediate = Instruction.operandAt(1);
+        if (Immediate == nullptr || (Immediate->kind() != MachineOperand::Kind::Immediate &&
+                                     Immediate->kind() != MachineOperand::Kind::IntegerImmediate))
+          return makeUnexpected(ErrorCode::VerificationError,
+                                "RISC-V integer constant requires an immediate operand");
+      }
+      return {};
+    case GenericOpcode::G_LOAD:
+    case GenericOpcode::G_SEXTLOAD:
+    case GenericOpcode::G_ZEXTLOAD:
+    case GenericOpcode::G_FPEXTLOAD:
+    case GenericOpcode::G_FPTRUNCSTORE:
+    case GenericOpcode::G_STORE:
+    case GenericOpcode::G_BR:
+    case GenericOpcode::G_BRCOND:
+      return {};
+    case GenericOpcode::G_FADD:
+    case GenericOpcode::G_FSUB:
+    case GenericOpcode::G_FMUL:
+    case GenericOpcode::G_FDIV:
+    case GenericOpcode::G_FMA:
+    case GenericOpcode::G_FSQRT:
+    case GenericOpcode::G_FNEG:
+    case GenericOpcode::G_FABS:
+    case GenericOpcode::G_FCOPYSIGN:
+      if (!Float())
+        return makeUnexpected(ErrorCode::Unsupported,
+                              "RISC-V selector requires f32 or f64 floating values");
+      return {};
+    case GenericOpcode::G_FCONSTANT: {
+      const MachineOperand *Immediate = Instruction.operandAt(1);
+      const auto *Floating =
+          Immediate == nullptr ? nullptr : Immediate->getIf<FloatingImmediateOperand>();
+      const std::optional<FloatSemantics> Semantics = Type ? Type->floatSemantics() : std::nullopt;
+      if (!Float() || Floating == nullptr || !Semantics || Floating->Bits.semantics() != *Semantics)
+        return makeUnexpected(ErrorCode::Unsupported,
+                              "RISC-V selector requires a matching floating constant");
+      return {};
+    }
+    case GenericOpcode::G_ANYEXT:
+    case GenericOpcode::G_ASSERT_SEXT:
+    case GenericOpcode::G_ASSERT_ZEXT:
+    case GenericOpcode::G_ASSERT_ALIGN:
+    case GenericOpcode::G_FREEZE:
+    case GenericOpcode::G_CONSTANT_FOLD_BARRIER:
+    case GenericOpcode::G_PTRTOINT:
+    case GenericOpcode::G_INTTOPTR:
+    case GenericOpcode::G_BITCAST:
+    case GenericOpcode::G_TRUNC:
+    case GenericOpcode::G_SEXT:
+    case GenericOpcode::G_SEXT_INREG:
+    case GenericOpcode::G_ZEXT:
+    case GenericOpcode::G_ICMP:
+    case GenericOpcode::G_FCMP:
+    case GenericOpcode::G_FPEXT:
+    case GenericOpcode::G_FPTRUNC:
+    case GenericOpcode::G_FPTOSI:
+    case GenericOpcode::G_FPTOUI:
+    case GenericOpcode::G_SITOFP:
+    case GenericOpcode::G_UITOFP:
+    case GenericOpcode::G_FMINNUM:
+    case GenericOpcode::G_FMAXNUM:
+    case GenericOpcode::G_SDIVREM:
+    case GenericOpcode::G_UDIVREM:
+    case GenericOpcode::G_SMULH:
+    case GenericOpcode::G_UMULH:
+    case GenericOpcode::G_SMIN:
+    case GenericOpcode::G_SMAX:
+    case GenericOpcode::G_UMIN:
+    case GenericOpcode::G_UMAX:
+    case GenericOpcode::G_SELECT:
+    case GenericOpcode::G_ABS:
+    case GenericOpcode::G_UADDO:
+    case GenericOpcode::G_USUBO:
+    case GenericOpcode::G_SADDO:
+    case GenericOpcode::G_SSUBO:
+    case GenericOpcode::G_UMULO:
+    case GenericOpcode::G_SMULO:
+    case GenericOpcode::G_UADDE:
+    case GenericOpcode::G_USUBE:
+    case GenericOpcode::G_SADDE:
+    case GenericOpcode::G_SSUBE:
+    case GenericOpcode::G_ATOMICRMW_XCHG:
+    case GenericOpcode::G_ATOMICRMW_ADD:
+    case GenericOpcode::G_ATOMICRMW_SUB:
+    case GenericOpcode::G_ATOMICRMW_AND:
+    case GenericOpcode::G_ATOMICRMW_OR:
+    case GenericOpcode::G_ATOMICRMW_XOR:
+    case GenericOpcode::G_ATOMICRMW_MAX:
+    case GenericOpcode::G_ATOMICRMW_MIN:
+    case GenericOpcode::G_ATOMICRMW_UMAX:
+    case GenericOpcode::G_ATOMICRMW_UMIN:
+    case GenericOpcode::G_FENCE:
+    case GenericOpcode::G_BRINDIRECT:
+    case GenericOpcode::G_TRAP:
+    case GenericOpcode::G_DEBUGTRAP:
+    case GenericOpcode::G_UBSANTRAP:
+    case GenericOpcode::G_PHI:
+    case GenericOpcode::G_IMPLICIT_DEF:
+      return {};
+    default:
+      return makeUnexpected(ErrorCode::Unsupported,
+                            "RISC-V instruction selector has no rule for generic opcode");
+    }
+  }
+
+  [[nodiscard]] Expected<void> select(MachineInstr &Instruction, MachineFunction &Function,
+                                      const TargetDescription &Target) const override {
+    auto Match = canSelect(Instruction, Function, Target);
+    if (!Match)
+      return std::unexpected(Match.error());
+    if (Instruction.opcode() == targetOpcode(InstructionOpcode::G_SLLW))
+      return Instruction.setOpcode(targetOpcode(InstructionOpcode::SLLW));
+    const GenericOpcode Generic = static_cast<GenericOpcode>(Instruction.opcode().value());
+    const std::optional<LLT> Type = resultType(Function, Instruction);
+
+    const auto SelectDirect = [&Instruction](InstructionOpcode Selected) {
+      return Instruction.setOpcode(targetOpcode(Selected));
+    };
+    switch (Generic) {
+    case GenericOpcode::G_ADD:
+      return selectIntegerBinary(Instruction, Type, InstructionOpcode::ADD,
+                                 InstructionOpcode::ADDW);
+    case GenericOpcode::G_SUB:
+      return selectIntegerBinary(Instruction, Type, InstructionOpcode::SUB,
+                                 InstructionOpcode::SUBW);
+    case GenericOpcode::G_MUL:
+      return selectIntegerBinary(Instruction, Type, InstructionOpcode::MUL,
+                                 InstructionOpcode::MULW);
+    case GenericOpcode::G_SDIV:
+      return selectNormalizedIntegerBinary(Instruction, Function, Target, Type,
+                                           InstructionOpcode::DIV, InstructionOpcode::DIVW, true,
+                                           true);
+    case GenericOpcode::G_UDIV:
+      return selectNormalizedIntegerBinary(Instruction, Function, Target, Type,
+                                           InstructionOpcode::DIVU, InstructionOpcode::DIVUW, false,
+                                           true);
+    case GenericOpcode::G_SREM:
+      return selectNormalizedIntegerBinary(Instruction, Function, Target, Type,
+                                           InstructionOpcode::REM, InstructionOpcode::REMW, true,
+                                           true);
+    case GenericOpcode::G_UREM:
+      return selectNormalizedIntegerBinary(Instruction, Function, Target, Type,
+                                           InstructionOpcode::REMU, InstructionOpcode::REMUW, false,
+                                           true);
+    case GenericOpcode::G_SDIVREM:
+      return selectDivRem(Instruction, Function, Target, Type, true);
+    case GenericOpcode::G_UDIVREM:
+      return selectDivRem(Instruction, Function, Target, Type, false);
+    case GenericOpcode::G_AND:
+      return SelectDirect(InstructionOpcode::AND);
+    case GenericOpcode::G_OR:
+      return SelectDirect(InstructionOpcode::OR);
+    case GenericOpcode::G_XOR:
+      return SelectDirect(InstructionOpcode::XOR);
+    case GenericOpcode::G_SHL:
+      return selectIntegerBinary(Instruction, Type, InstructionOpcode::SLL,
+                                 InstructionOpcode::SLLW);
+    case GenericOpcode::G_LSHR:
+      return selectNormalizedIntegerBinary(Instruction, Function, Target, Type,
+                                           InstructionOpcode::SRL, InstructionOpcode::SRLW, false,
+                                           false);
+    case GenericOpcode::G_ASHR:
+      return selectNormalizedIntegerBinary(Instruction, Function, Target, Type,
+                                           InstructionOpcode::SRA, InstructionOpcode::SRAW, true,
+                                           false);
+    case GenericOpcode::G_PTR_ADD:
+      return SelectDirect(InstructionOpcode::ADD);
+    case GenericOpcode::G_PTRMASK:
+      return SelectDirect(InstructionOpcode::AND);
+    case GenericOpcode::G_LOAD:
+      return selectLoad(Instruction, Type);
+    case GenericOpcode::G_SEXTLOAD:
+      return selectExtLoad(Instruction, Type, true);
+    case GenericOpcode::G_ZEXTLOAD:
+      return selectExtLoad(Instruction, Type, false);
+    case GenericOpcode::G_FPEXTLOAD:
+      return selectFloatExtLoad(Instruction, Function, Target);
+    case GenericOpcode::G_STORE:
+      return selectStore(Instruction, Function);
+    case GenericOpcode::G_FPTRUNCSTORE:
+      return selectFloatTruncStore(Instruction, Function, Target);
+    case GenericOpcode::G_BR:
+      return selectBranch(Instruction);
+    case GenericOpcode::G_BRCOND:
+      return selectConditionalBranch(Instruction);
+    case GenericOpcode::G_CONSTANT:
+      return selectIntegerConstant(Instruction, Function, Target, Type);
+    case GenericOpcode::G_FCONSTANT:
+      return selectFloatingConstant(Instruction, Function, Target, Type);
+    case GenericOpcode::G_ASSERT_SEXT:
+    case GenericOpcode::G_ASSERT_ZEXT:
+    case GenericOpcode::G_ASSERT_ALIGN:
+      if (Instruction.operandCount() == 3) {
+        auto EraseImmediate = Instruction.eraseOperand(2);
+        if (!EraseImmediate)
+          return std::unexpected(EraseImmediate.error());
+      }
+      return Instruction.setOpcode(opcode(GenericOpcode::COPY));
+    case GenericOpcode::G_FREEZE:
+    case GenericOpcode::G_CONSTANT_FOLD_BARRIER:
+    case GenericOpcode::G_PTRTOINT:
+    case GenericOpcode::G_INTTOPTR:
+      return Instruction.setOpcode(opcode(GenericOpcode::COPY));
+    case GenericOpcode::G_BITCAST:
+      return selectBitcast(Instruction, Function);
+    case GenericOpcode::G_FADD:
+      return selectRoundingBinary(Instruction, Type, InstructionOpcode::FADD_S,
+                                  InstructionOpcode::FADD_D);
+    case GenericOpcode::G_FSUB:
+      return selectRoundingBinary(Instruction, Type, InstructionOpcode::FSUB_S,
+                                  InstructionOpcode::FSUB_D);
+    case GenericOpcode::G_FMUL:
+      return selectRoundingBinary(Instruction, Type, InstructionOpcode::FMUL_S,
+                                  InstructionOpcode::FMUL_D);
+    case GenericOpcode::G_FDIV:
+      return selectRoundingBinary(Instruction, Type, InstructionOpcode::FDIV_S,
+                                  InstructionOpcode::FDIV_D);
+    case GenericOpcode::G_FMA:
+      return selectRoundingTernary(Instruction, Type, InstructionOpcode::FMADD_S,
+                                   InstructionOpcode::FMADD_D);
+    case GenericOpcode::G_FSQRT:
+      return selectRoundingUnary(Instruction, Type, InstructionOpcode::FSQRT_S,
+                                 InstructionOpcode::FSQRT_D);
+    case GenericOpcode::G_FNEG:
+      return selectSignInjection(Instruction, Type, InstructionOpcode::FSGNJN_S,
+                                 InstructionOpcode::FSGNJN_D);
+    case GenericOpcode::G_FABS:
+      return selectSignInjection(Instruction, Type, InstructionOpcode::FSGNJX_S,
+                                 InstructionOpcode::FSGNJX_D);
+    case GenericOpcode::G_FCOPYSIGN:
+      return selectFloatingMinMax(Instruction, Type, InstructionOpcode::FSGNJ_S,
+                                  InstructionOpcode::FSGNJ_D);
+    case GenericOpcode::G_ANYEXT:
+    case GenericOpcode::G_SEXT:
+    case GenericOpcode::G_ZEXT:
+      return selectIntegerExtension(Instruction, Function, Target, Generic, Type);
+    case GenericOpcode::G_TRUNC:
+      return Instruction.setOpcode(opcode(GenericOpcode::COPY));
+    case GenericOpcode::G_SEXT_INREG:
+      return selectSignExtendInRegister(Instruction, Function, Target, Type);
+    case GenericOpcode::G_ICMP:
+      return selectIntegerCompare(Instruction, Function, Target);
+    case GenericOpcode::G_FCMP:
+      return selectFloatingCompare(Instruction, Function, Target);
+    case GenericOpcode::G_FPEXT:
+      return selectRoundingUnary(Instruction, Type, InstructionOpcode::FCVT_D_S,
+                                 InstructionOpcode::FCVT_D_S);
+    case GenericOpcode::G_FPTRUNC:
+      return selectRoundingUnary(Instruction, Type, InstructionOpcode::FCVT_S_D,
+                                 InstructionOpcode::FCVT_S_D);
+    case GenericOpcode::G_FPTOSI:
+    case GenericOpcode::G_FPTOUI:
+      return selectFloatToInteger(Instruction, Function, Type, Generic == GenericOpcode::G_FPTOUI);
+    case GenericOpcode::G_SITOFP:
+    case GenericOpcode::G_UITOFP:
+      return selectIntegerToFloat(Instruction, Function, Target, Type,
+                                  Generic == GenericOpcode::G_UITOFP);
+    case GenericOpcode::G_FMINNUM:
+      return selectFloatingMinMax(Instruction, Type, InstructionOpcode::FMIN_S,
+                                  InstructionOpcode::FMIN_D);
+    case GenericOpcode::G_FMAXNUM:
+      return selectFloatingMinMax(Instruction, Type, InstructionOpcode::FMAX_S,
+                                  InstructionOpcode::FMAX_D);
+    case GenericOpcode::G_SMULH:
+      return selectMultiplyHigh(Instruction, Function, Target, Type, true);
+    case GenericOpcode::G_UMULH:
+      return selectMultiplyHigh(Instruction, Function, Target, Type, false);
+    case GenericOpcode::G_SMIN:
+    case GenericOpcode::G_SMAX:
+    case GenericOpcode::G_UMIN:
+    case GenericOpcode::G_UMAX:
+      return selectIntegerMinMax(Instruction, Function, Target, Type, Generic);
+    case GenericOpcode::G_SELECT:
+      return selectValue(Instruction, Function, Target, Type);
+    case GenericOpcode::G_ABS:
+      return selectAbsoluteValue(Instruction, Function, Target, Type);
+    case GenericOpcode::G_UADDO:
+    case GenericOpcode::G_USUBO:
+    case GenericOpcode::G_SADDO:
+    case GenericOpcode::G_SSUBO:
+    case GenericOpcode::G_UMULO:
+    case GenericOpcode::G_SMULO:
+      return selectOverflow(Instruction, Function, Target, Type, Generic);
+    case GenericOpcode::G_UADDE:
+    case GenericOpcode::G_USUBE:
+    case GenericOpcode::G_SADDE:
+    case GenericOpcode::G_SSUBE:
+      return selectExtendedOverflow(Instruction, Function, Target, Type, Generic);
+    case GenericOpcode::G_ATOMICRMW_XCHG:
+    case GenericOpcode::G_ATOMICRMW_ADD:
+    case GenericOpcode::G_ATOMICRMW_SUB:
+    case GenericOpcode::G_ATOMICRMW_AND:
+    case GenericOpcode::G_ATOMICRMW_OR:
+    case GenericOpcode::G_ATOMICRMW_XOR:
+    case GenericOpcode::G_ATOMICRMW_MAX:
+    case GenericOpcode::G_ATOMICRMW_MIN:
+    case GenericOpcode::G_ATOMICRMW_UMAX:
+    case GenericOpcode::G_ATOMICRMW_UMIN:
+      return selectAtomicRMW(Instruction, Function, Target, Type, Generic);
+    case GenericOpcode::G_FENCE:
+      return selectFence(Instruction);
+    case GenericOpcode::G_BRINDIRECT:
+      return selectIndirectBranch(Instruction);
+    case GenericOpcode::G_TRAP:
+    case GenericOpcode::G_DEBUGTRAP:
+    case GenericOpcode::G_UBSANTRAP:
+      return selectTrap(Instruction);
+    case GenericOpcode::G_PHI:
+      return Instruction.setOpcode(opcode(GenericOpcode::PHI));
+    case GenericOpcode::G_IMPLICIT_DEF:
+      return Instruction.setOpcode(opcode(GenericOpcode::IMPLICIT_DEF));
+    case GenericOpcode::COPY:
+    case GenericOpcode::IMPLICIT_DEF:
+    case GenericOpcode::PHI:
+    case GenericOpcode::REG_SEQUENCE:
+    case GenericOpcode::INSERT_SUBREG:
+    case GenericOpcode::EXTRACT_SUBREG:
+    case GenericOpcode::SUBREG_TO_REG:
+    case GenericOpcode::KILL:
+    case GenericOpcode::BUNDLE:
+    case GenericOpcode::INLINEASM:
+    case GenericOpcode::CORE_PSEUDO_END:
+      break;
+    default:
+      return makeUnexpected(ErrorCode::Unsupported,
+                            "RISC-V instruction selector has no rule for generic opcode");
+    }
+    return makeUnexpected(ErrorCode::Unsupported,
+                          "RISC-V instruction selector has no rule for generic opcode");
+  }
+
+private:
+  [[nodiscard]] Expected<Register> createTemporary(MachineFunction &Function, LLT Type,
+                                                   const TargetDescription &Target) const {
+    auto Temporary = Function.registerInfo().createVirtualRegister(Type);
+    if (!Temporary)
+      return std::unexpected(Temporary.error());
+    auto Constraint = RegisterBanks.constrainRegister(Function, *Temporary, Target);
+    if (!Constraint)
+      return std::unexpected(Constraint.error());
+    return *Temporary;
+  }
+
+  [[nodiscard]] Expected<void> replaceImmediate(MachineInstr &Instruction, std::size_t Index,
+                                                std::int64_t Value) const {
+    if (Instruction.operandAt(Index) == nullptr)
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "RISC-V instruction is missing an immediate operand");
+    auto Erased = Instruction.eraseOperand(Index);
+    if (!Erased)
+      return std::unexpected(Erased.error());
+    return Instruction.insertOperand(Index, MachineOperand::immediate(Value));
+  }
+
+  [[nodiscard]] Expected<MachineInstr *> buildBinaryAfter(MachineIRBuilder &Builder,
+                                                          MachineInstr &Anchor,
+                                                          InstructionOpcode Opcode, Register Def,
+                                                          Register Left, Register Right) const {
+    auto Point = Builder.setInsertPointAfter(Anchor);
+    if (!Point)
+      return std::unexpected(Point.error());
+    auto Pending = Builder.buildInstr(targetOpcode(Opcode));
+    if (!Pending)
+      return std::unexpected(Pending.error());
+    auto DefResult = Pending->addDef(Def);
+    if (!DefResult)
+      return std::unexpected(DefResult.error());
+    auto LeftResult = Pending->addUse(Left);
+    if (!LeftResult)
+      return std::unexpected(LeftResult.error());
+    auto RightResult = Pending->addUse(Right);
+    if (!RightResult)
+      return std::unexpected(RightResult.error());
+    auto Built = Pending->build();
+    if (!Built)
+      return std::unexpected(Built.error());
+    return *Built;
+  }
+
+  [[nodiscard]] Expected<MachineInstr *>
+  buildImmediateAfter(MachineIRBuilder &Builder, MachineInstr &Anchor, InstructionOpcode Opcode,
+                      Register Def, Register Source, std::int64_t Value) const {
+    auto Point = Builder.setInsertPointAfter(Anchor);
+    if (!Point)
+      return std::unexpected(Point.error());
+    auto Pending = Builder.buildInstr(targetOpcode(Opcode));
+    if (!Pending)
+      return std::unexpected(Pending.error());
+    auto DefResult = Pending->addDef(Def);
+    if (!DefResult)
+      return std::unexpected(DefResult.error());
+    auto SourceResult = Pending->addUse(Source);
+    if (!SourceResult)
+      return std::unexpected(SourceResult.error());
+    auto ImmediateResult = Pending->addImmediate(Value);
+    if (!ImmediateResult)
+      return std::unexpected(ImmediateResult.error());
+    auto Built = Pending->build();
+    if (!Built)
+      return std::unexpected(Built.error());
+    return *Built;
+  }
+
+  [[nodiscard]] Expected<MachineInstr *>
+  buildImmediateBefore(MachineIRBuilder &Builder, MachineInstr &Anchor, InstructionOpcode Opcode,
+                       Register Def, Register Source, std::int64_t Value) const {
+    auto Point = Builder.setInsertPointBefore(Anchor);
+    if (!Point)
+      return std::unexpected(Point.error());
+    auto Pending = Builder.buildInstr(targetOpcode(Opcode));
+    if (!Pending)
+      return std::unexpected(Pending.error());
+    auto DefResult = Pending->addDef(Def);
+    if (!DefResult)
+      return std::unexpected(DefResult.error());
+    auto SourceResult = Pending->addUse(Source);
+    if (!SourceResult)
+      return std::unexpected(SourceResult.error());
+    auto ImmediateResult = Pending->addImmediate(Value);
+    if (!ImmediateResult)
+      return std::unexpected(ImmediateResult.error());
+    auto Built = Pending->build();
+    if (!Built)
+      return std::unexpected(Built.error());
+    return *Built;
+  }
+
+  [[nodiscard]] Expected<Register> normalizeIntegerBefore(MachineInstr &Anchor,
+                                                          MachineFunction &Function,
+                                                          const TargetDescription &Target,
+                                                          Register Source, LLT Type,
+                                                          bool Signed) const {
+    if (!isSupportedIntegerScalar(Type))
+      return makeUnexpected(ErrorCode::Unsupported,
+                            "RISC-V integer normalization requires a scalar i1-i64 type");
+    const std::uint32_t Bits = Type.scalarBitWidth();
+    if (Bits == 64)
+      return Source;
+
+    MachineIRBuilder Builder(Function, Target);
+    if (Signed && Bits == 32) {
+      auto Result = createTemporary(Function, Type, Target);
+      if (!Result)
+        return std::unexpected(Result.error());
+      auto Extended =
+          buildImmediateBefore(Builder, Anchor, InstructionOpcode::ADDIW, *Result, Source, 0);
+      if (!Extended)
+        return std::unexpected(Extended.error());
+      return *Result;
+    }
+
+    const std::uint32_t ContainerBits = Bits < 32 ? 32 : 64;
+    const std::int64_t Shift = static_cast<std::int64_t>(ContainerBits - Bits);
+    auto Shifted = createTemporary(Function, Type, Target);
+    auto Result = createTemporary(Function, Type, Target);
+    if (!Shifted || !Result)
+      return std::unexpected(!Shifted ? Shifted.error() : Result.error());
+    const InstructionOpcode ShiftLeft =
+        ContainerBits == 32 ? InstructionOpcode::SLLIW : InstructionOpcode::SLLI;
+    const InstructionOpcode ShiftRight =
+        Signed ? (ContainerBits == 32 ? InstructionOpcode::SRAIW : InstructionOpcode::SRAI)
+               : (ContainerBits == 32 ? InstructionOpcode::SRLIW : InstructionOpcode::SRLI);
+    auto Left = buildImmediateBefore(Builder, Anchor, ShiftLeft, *Shifted, Source, Shift);
+    if (!Left)
+      return std::unexpected(Left.error());
+    auto Right = buildImmediateBefore(Builder, Anchor, ShiftRight, *Result, *Shifted, Shift);
+    if (!Right)
+      return std::unexpected(Right.error());
+    return *Result;
+  }
+
+  [[nodiscard]] Expected<MachineInstr *> buildUnaryAfter(MachineIRBuilder &Builder,
+                                                         MachineInstr &Anchor,
+                                                         InstructionOpcode Opcode, Register Def,
+                                                         Register Source) const {
+    auto Point = Builder.setInsertPointAfter(Anchor);
+    if (!Point)
+      return std::unexpected(Point.error());
+    auto Pending = Builder.buildInstr(targetOpcode(Opcode));
+    if (!Pending)
+      return std::unexpected(Pending.error());
+    auto DefResult = Pending->addDef(Def);
+    if (!DefResult)
+      return std::unexpected(DefResult.error());
+    auto SourceResult = Pending->addUse(Source);
+    if (!SourceResult)
+      return std::unexpected(SourceResult.error());
+    auto Built = Pending->build();
+    if (!Built)
+      return std::unexpected(Built.error());
+    return *Built;
+  }
+
+  [[nodiscard]] Expected<MachineInstr *>
+  buildGPRSelectAfter(MachineIRBuilder &Builder, MachineInstr &Anchor, MachineFunction &Function,
+                      const TargetDescription &Target, LLT ValueType, Register Destination,
+                      Register Condition, Register TrueValue, Register FalseValue) const {
+    auto IntegerType = LLT::makeInteger(ValueType.scalarBitWidth());
+    if (!IntegerType)
+      return std::unexpected(IntegerType.error());
+    auto Mask = createTemporary(Function, *IntegerType, Target);
+    auto Difference = createTemporary(Function, *IntegerType, Target);
+    auto MaskedDifference = createTemporary(Function, *IntegerType, Target);
+    if (!Mask || !Difference || !MaskedDifference) {
+      return std::unexpected(
+          (!Mask ? Mask.error() : (!Difference ? Difference.error() : MaskedDifference.error())));
+    }
+    auto Zero = physicalRegister(PhysicalRegisterID::X0);
+    if (!Zero)
+      return std::unexpected(Zero.error());
+    const InstructionOpcode Subtract =
+        ValueType.scalarBitWidth() == 32 ? InstructionOpcode::SUBW : InstructionOpcode::SUB;
+    auto MaskInstruction = buildBinaryAfter(Builder, Anchor, Subtract, *Mask,
+                                            Register::fromPhysical(*Zero), Condition);
+    if (!MaskInstruction)
+      return std::unexpected(MaskInstruction.error());
+    auto DifferenceInstruction = buildBinaryAfter(
+        Builder, **MaskInstruction, InstructionOpcode::XOR, *Difference, TrueValue, FalseValue);
+    if (!DifferenceInstruction)
+      return std::unexpected(DifferenceInstruction.error());
+    auto MaskInstructionResult =
+        buildBinaryAfter(Builder, **DifferenceInstruction, InstructionOpcode::AND,
+                         *MaskedDifference, *Difference, *Mask);
+    if (!MaskInstructionResult)
+      return std::unexpected(MaskInstructionResult.error());
+    return buildBinaryAfter(Builder, **MaskInstructionResult, InstructionOpcode::XOR, Destination,
+                            FalseValue, *MaskedDifference);
+  }
+
+  [[nodiscard]] Expected<void> selectDivRem(MachineInstr &Instruction, MachineFunction &Function,
+                                            const TargetDescription &Target,
+                                            std::optional<LLT> Type, bool Signed) const {
+    const MachineOperand *RemainderOperand = Instruction.operandAt(1);
+    const MachineOperand *LeftOperand = Instruction.operandAt(2);
+    const MachineOperand *RightOperand = Instruction.operandAt(3);
+    const Register *Remainder =
+        RemainderOperand == nullptr ? nullptr : RemainderOperand->getIf<Register>();
+    const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf<Register>();
+    const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf<Register>();
+    if (!Type || Remainder == nullptr || Left == nullptr || Right == nullptr)
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "RISC-V division-with-remainder has invalid operands");
+    const Register RemainderRegister = *Remainder;
+    auto LeftRegister = normalizeIntegerBefore(Instruction, Function, Target, *Left, *Type, Signed);
+    auto RightRegister =
+        normalizeIntegerBefore(Instruction, Function, Target, *Right, *Type, Signed);
+    if (!LeftRegister || !RightRegister)
+      return std::unexpected(!LeftRegister ? LeftRegister.error() : RightRegister.error());
+    auto RemoveRemainder = Instruction.eraseOperand(1);
+    if (!RemoveRemainder)
+      return std::unexpected(RemoveRemainder.error());
+    auto SetLeft = Instruction.operandAt(1)->setRegister(*LeftRegister);
+    if (!SetLeft)
+      return std::unexpected(SetLeft.error());
+    auto SetRight = Instruction.operandAt(2)->setRegister(*RightRegister);
+    if (!SetRight)
+      return std::unexpected(SetRight.error());
+    const bool IsWord = Type->scalarBitWidth() <= 32;
+    const InstructionOpcode Division =
+        IsWord ? (Signed ? InstructionOpcode::DIVW : InstructionOpcode::DIVUW)
+               : (Signed ? InstructionOpcode::DIV : InstructionOpcode::DIVU);
+    const InstructionOpcode RemainderOpcode =
+        IsWord ? (Signed ? InstructionOpcode::REMW : InstructionOpcode::REMUW)
+               : (Signed ? InstructionOpcode::REM : InstructionOpcode::REMU);
+    auto SetDivision = Instruction.setOpcode(targetOpcode(Division));
+    if (!SetDivision)
+      return std::unexpected(SetDivision.error());
+    MachineIRBuilder Builder(Function, Target);
+    auto BuiltRemainder = buildBinaryAfter(Builder, Instruction, RemainderOpcode, RemainderRegister,
+                                           *LeftRegister, *RightRegister);
+    if (!BuiltRemainder)
+      return std::unexpected(BuiltRemainder.error());
+    return {};
+  }
+
+  [[nodiscard]] Expected<void> selectMultiplyHigh(MachineInstr &Instruction,
+                                                  MachineFunction &Function,
+                                                  const TargetDescription &Target,
+                                                  std::optional<LLT> Type, bool Signed) const {
+    if (!Type)
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "RISC-V high multiplication has no result type");
+    if (Type->scalarBitWidth() == 64) {
+      return Instruction.setOpcode(
+          targetOpcode(Signed ? InstructionOpcode::MULH : InstructionOpcode::MULHU));
+    }
+
+    const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+    const MachineOperand *LeftOperand = Instruction.operandAt(1);
+    const MachineOperand *RightOperand = Instruction.operandAt(2);
+    const Register *Destination =
+        DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+    const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf<Register>();
+    const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf<Register>();
+    if (Destination == nullptr || Left == nullptr || Right == nullptr)
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "RISC-V high multiplication has invalid register operands");
+    const Register DestinationRegister = *Destination;
+    const Register RightRegister = *Right;
+    auto I64 = LLT::makeInteger(64);
+    if (!I64)
+      return std::unexpected(I64.error());
+    auto ExtendedLeft = createTemporary(Function, *I64, Target);
+    auto ExtendedRight = createTemporary(Function, *I64, Target);
+    auto Product = createTemporary(Function, *I64, Target);
+    if (!ExtendedLeft || !ExtendedRight || !Product) {
+      return std::unexpected(!ExtendedLeft
+                                 ? ExtendedLeft.error()
+                                 : (!ExtendedRight ? ExtendedRight.error() : Product.error()));
+    }
+
+    auto RemoveRight = Instruction.eraseOperand(2);
+    if (!RemoveRight)
+      return std::unexpected(RemoveRight.error());
+    MachineIRBuilder Builder(Function, Target);
+    MachineInstr *Anchor = &Instruction;
+    if (Signed) {
+      auto SetDestination = Instruction.operandAt(0)->setRegister(*ExtendedLeft);
+      if (!SetDestination)
+        return std::unexpected(SetDestination.error());
+      auto Immediate = appendImmediate(Instruction, 0);
+      if (!Immediate)
+        return std::unexpected(Immediate.error());
+      auto SetOpcode = Instruction.setOpcode(targetOpcode(InstructionOpcode::ADDIW));
+      if (!SetOpcode)
+        return std::unexpected(SetOpcode.error());
+      auto ExtendRight = buildImmediateAfter(Builder, *Anchor, InstructionOpcode::ADDIW,
+                                             *ExtendedRight, RightRegister, 0);
+      if (!ExtendRight)
+        return std::unexpected(ExtendRight.error());
+      Anchor = *ExtendRight;
+    } else {
+      auto ShiftedLeft = createTemporary(Function, *I64, Target);
+      auto ShiftedRight = createTemporary(Function, *I64, Target);
+      if (!ShiftedLeft || !ShiftedRight)
+        return std::unexpected(!ShiftedLeft ? ShiftedLeft.error() : ShiftedRight.error());
+      auto SetDestination = Instruction.operandAt(0)->setRegister(*ShiftedLeft);
+      if (!SetDestination)
+        return std::unexpected(SetDestination.error());
+      auto Immediate = appendImmediate(Instruction, 32);
+      if (!Immediate)
+        return std::unexpected(Immediate.error());
+      auto SetOpcode = Instruction.setOpcode(targetOpcode(InstructionOpcode::SLLI));
+      if (!SetOpcode)
+        return std::unexpected(SetOpcode.error());
+      auto ExtendLeft = buildImmediateAfter(Builder, *Anchor, InstructionOpcode::SRLI,
+                                            *ExtendedLeft, *ShiftedLeft, 32);
+      if (!ExtendLeft)
+        return std::unexpected(ExtendLeft.error());
+      auto ShiftRight = buildImmediateAfter(Builder, **ExtendLeft, InstructionOpcode::SLLI,
+                                            *ShiftedRight, RightRegister, 32);
+      if (!ShiftRight)
+        return std::unexpected(ShiftRight.error());
+      auto ExtendRight = buildImmediateAfter(Builder, **ShiftRight, InstructionOpcode::SRLI,
+                                             *ExtendedRight, *ShiftedRight, 32);
+      if (!ExtendRight)
+        return std::unexpected(ExtendRight.error());
+      Anchor = *ExtendRight;
+    }
+    auto Multiply = buildBinaryAfter(Builder, *Anchor, InstructionOpcode::MUL, *Product,
+                                     *ExtendedLeft, *ExtendedRight);
+    if (!Multiply)
+      return std::unexpected(Multiply.error());
+    auto Extract = buildImmediateAfter(Builder, **Multiply,
+                                       Signed ? InstructionOpcode::SRAI : InstructionOpcode::SRLI,
+                                       DestinationRegister, *Product, 32);
+    if (!Extract)
+      return std::unexpected(Extract.error());
+    return {};
+  }
+
+  [[nodiscard]] Expected<void> selectIntegerMinMax(MachineInstr &Instruction,
+                                                   MachineFunction &Function,
+                                                   const TargetDescription &Target,
+                                                   std::optional<LLT> Type,
+                                                   GenericOpcode Generic) const {
+    const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+    const MachineOperand *LeftOperand = Instruction.operandAt(1);
+    const MachineOperand *RightOperand = Instruction.operandAt(2);
+    const Register *Destination =
+        DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+    const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf<Register>();
+    const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf<Register>();
+    if (!Type || Destination == nullptr || Left == nullptr || Right == nullptr)
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "RISC-V integer min/max has invalid operands");
+    auto I1 = LLT::makeInteger(1);
+    if (!I1)
+      return std::unexpected(I1.error());
+    auto Condition = createTemporary(Function, *I1, Target);
+    if (!Condition)
+      return std::unexpected(Condition.error());
+    const Register DestinationRegister = *Destination;
+    const Register LeftRegister = *Left;
+    const Register RightRegister = *Right;
+    const bool Unsigned = Generic == GenericOpcode::G_UMIN || Generic == GenericOpcode::G_UMAX;
+    auto NormalizedLeft =
+        normalizeIntegerBefore(Instruction, Function, Target, LeftRegister, *Type, !Unsigned);
+    auto NormalizedRight =
+        normalizeIntegerBefore(Instruction, Function, Target, RightRegister, *Type, !Unsigned);
+    if (!NormalizedLeft || !NormalizedRight)
+      return std::unexpected(!NormalizedLeft ? NormalizedLeft.error() : NormalizedRight.error());
+    auto SetCondition = Instruction.operandAt(0)->setRegister(*Condition);
+    if (!SetCondition)
+      return std::unexpected(SetCondition.error());
+    auto SetLeft = Instruction.operandAt(1)->setRegister(*NormalizedLeft);
+    if (!SetLeft)
+      return std::unexpected(SetLeft.error());
+    auto SetRight = Instruction.operandAt(2)->setRegister(*NormalizedRight);
+    if (!SetRight)
+      return std::unexpected(SetRight.error());
+    auto SetCompare = Instruction.setOpcode(
+        targetOpcode(Unsigned ? InstructionOpcode::SLTU : InstructionOpcode::SLT));
+    if (!SetCompare)
+      return std::unexpected(SetCompare.error());
+    const bool IsMaximum = Generic == GenericOpcode::G_SMAX || Generic == GenericOpcode::G_UMAX;
+    const Register TrueValue = IsMaximum ? RightRegister : LeftRegister;
+    const Register FalseValue = IsMaximum ? LeftRegister : RightRegister;
+    MachineIRBuilder Builder(Function, Target);
+    auto Selected = buildGPRSelectAfter(Builder, Instruction, Function, Target, *Type,
+                                        DestinationRegister, *Condition, TrueValue, FalseValue);
+    if (!Selected)
+      return std::unexpected(Selected.error());
+    return {};
+  }
+
+  [[nodiscard]] Expected<void> selectValue(MachineInstr &Instruction, MachineFunction &Function,
+                                           const TargetDescription &Target,
+                                           std::optional<LLT> Type) const {
+    const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+    const MachineOperand *ConditionOperand = Instruction.operandAt(1);
+    const MachineOperand *TrueOperand = Instruction.operandAt(2);
+    const MachineOperand *FalseOperand = Instruction.operandAt(3);
+    const Register *Destination =
+        DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+    const Register *Condition =
+        ConditionOperand == nullptr ? nullptr : ConditionOperand->getIf<Register>();
+    const Register *TrueValue = TrueOperand == nullptr ? nullptr : TrueOperand->getIf<Register>();
+    const Register *FalseValue =
+        FalseOperand == nullptr ? nullptr : FalseOperand->getIf<Register>();
+    if (!Type || Destination == nullptr || Condition == nullptr || TrueValue == nullptr ||
+        FalseValue == nullptr) {
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "RISC-V value selection has invalid operands");
+    }
+    const Register DestinationRegister = *Destination;
+    const Register TrueRegister = *TrueValue;
+    const Register FalseRegister = *FalseValue;
+    auto I1 = LLT::makeInteger(1);
+    if (!I1)
+      return std::unexpected(I1.error());
+    auto NormalizedCondition = createTemporary(Function, *I1, Target);
+    if (!NormalizedCondition)
+      return std::unexpected(NormalizedCondition.error());
+    auto SetCondition = Instruction.operandAt(0)->setRegister(*NormalizedCondition);
+    if (!SetCondition)
+      return std::unexpected(SetCondition.error());
+    auto EraseFalse = Instruction.eraseOperand(3);
+    if (!EraseFalse)
+      return std::unexpected(EraseFalse.error());
+    auto EraseTrue = Instruction.eraseOperand(2);
+    if (!EraseTrue)
+      return std::unexpected(EraseTrue.error());
+    auto Mask = appendImmediate(Instruction, 1);
+    if (!Mask)
+      return std::unexpected(Mask.error());
+    auto SetNormalize = Instruction.setOpcode(targetOpcode(InstructionOpcode::ANDI));
+    if (!SetNormalize)
+      return std::unexpected(SetNormalize.error());
+
+    MachineIRBuilder Builder(Function, Target);
+    if (!Type->isFloat()) {
+      auto Selected =
+          buildGPRSelectAfter(Builder, Instruction, Function, Target, *Type, DestinationRegister,
+                              *NormalizedCondition, TrueRegister, FalseRegister);
+      if (!Selected)
+        return std::unexpected(Selected.error());
+      return {};
+    }
+
+    auto IntegerType = LLT::makeInteger(Type->scalarBitWidth());
+    if (!IntegerType)
+      return std::unexpected(IntegerType.error());
+    auto IntegerTrue = createTemporary(Function, *IntegerType, Target);
+    auto IntegerFalse = createTemporary(Function, *IntegerType, Target);
+    auto IntegerResult = createTemporary(Function, *IntegerType, Target);
+    if (!IntegerTrue || !IntegerFalse || !IntegerResult) {
+      return std::unexpected(!IntegerTrue
+                                 ? IntegerTrue.error()
+                                 : (!IntegerFalse ? IntegerFalse.error() : IntegerResult.error()));
+    }
+    const InstructionOpcode ToInteger =
+        Type->scalarBitWidth() == 32 ? InstructionOpcode::FMV_X_W : InstructionOpcode::FMV_X_D;
+    const InstructionOpcode ToFloat =
+        Type->scalarBitWidth() == 32 ? InstructionOpcode::FMV_W_X : InstructionOpcode::FMV_D_X;
+    auto MoveTrue = buildUnaryAfter(Builder, Instruction, ToInteger, *IntegerTrue, TrueRegister);
+    if (!MoveTrue)
+      return std::unexpected(MoveTrue.error());
+    auto MoveFalse = buildUnaryAfter(Builder, **MoveTrue, ToInteger, *IntegerFalse, FalseRegister);
+    if (!MoveFalse)
+      return std::unexpected(MoveFalse.error());
+    auto Selected =
+        buildGPRSelectAfter(Builder, **MoveFalse, Function, Target, *IntegerType, *IntegerResult,
+                            *NormalizedCondition, *IntegerTrue, *IntegerFalse);
+    if (!Selected)
+      return std::unexpected(Selected.error());
+    auto MoveResult =
+        buildUnaryAfter(Builder, **Selected, ToFloat, DestinationRegister, *IntegerResult);
+    if (!MoveResult)
+      return std::unexpected(MoveResult.error());
+    return {};
+  }
+
+  [[nodiscard]] Expected<void> selectAbsoluteValue(MachineInstr &Instruction,
+                                                   MachineFunction &Function,
+                                                   const TargetDescription &Target,
+                                                   std::optional<LLT> Type) const {
+    const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+    const MachineOperand *SourceOperand = Instruction.operandAt(1);
+    const Register *Destination =
+        DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+    const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>();
+    if (!Type || Destination == nullptr || Source == nullptr)
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "RISC-V absolute value has invalid operands");
+    const Register DestinationRegister = *Destination;
+    const Register SourceRegister = *Source;
+    auto NormalizedSource =
+        normalizeIntegerBefore(Instruction, Function, Target, SourceRegister, *Type, true);
+    if (!NormalizedSource)
+      return std::unexpected(NormalizedSource.error());
+    auto SignMask = createTemporary(Function, *Type, Target);
+    auto Flipped = createTemporary(Function, *Type, Target);
+    if (!SignMask || !Flipped)
+      return std::unexpected(!SignMask ? SignMask.error() : Flipped.error());
+    auto SetMask = Instruction.operandAt(0)->setRegister(*SignMask);
+    if (!SetMask)
+      return std::unexpected(SetMask.error());
+    auto SetSource = Instruction.operandAt(1)->setRegister(*NormalizedSource);
+    if (!SetSource)
+      return std::unexpected(SetSource.error());
+    const std::uint32_t ContainerBits = Type->scalarBitWidth() <= 32 ? 32 : 64;
+    auto Shift = appendImmediate(Instruction, ContainerBits - 1U);
+    if (!Shift)
+      return std::unexpected(Shift.error());
+    const bool IsWord = Type->scalarBitWidth() <= 32;
+    auto SetShift = Instruction.setOpcode(
+        targetOpcode(IsWord ? InstructionOpcode::SRAIW : InstructionOpcode::SRAI));
+    if (!SetShift)
+      return std::unexpected(SetShift.error());
+    MachineIRBuilder Builder(Function, Target);
+    auto Flip = buildBinaryAfter(Builder, Instruction, InstructionOpcode::XOR, *Flipped,
+                                 *NormalizedSource, *SignMask);
+    if (!Flip)
+      return std::unexpected(Flip.error());
+    auto Result =
+        buildBinaryAfter(Builder, **Flip, IsWord ? InstructionOpcode::SUBW : InstructionOpcode::SUB,
+                         DestinationRegister, *Flipped, *SignMask);
+    if (!Result)
+      return std::unexpected(Result.error());
+    return {};
+  }
+
+  [[nodiscard]] Expected<std::pair<Register, MachineInstr *>>
+  buildSignedOverflowAfter(MachineIRBuilder &Builder, MachineInstr &Anchor,
+                           MachineFunction &Function, const TargetDescription &Target, LLT Type,
+                           Register Result, Register Left, Register Right, bool IsAdd) const {
+    auto FirstDifference = createTemporary(Function, Type, Target);
+    auto SecondDifference = createTemporary(Function, Type, Target);
+    auto SignBits = createTemporary(Function, Type, Target);
+    auto I1 = LLT::makeInteger(1);
+    if (!I1)
+      return std::unexpected(I1.error());
+    auto Overflow = createTemporary(Function, *I1, Target);
+    if (!FirstDifference || !SecondDifference || !SignBits || !Overflow) {
+      return std::unexpected(!FirstDifference    ? FirstDifference.error()
+                             : !SecondDifference ? SecondDifference.error()
+                             : !SignBits         ? SignBits.error()
+                                                 : Overflow.error());
+    }
+    const Register FirstLeft = IsAdd ? Result : Left;
+    const Register FirstRight = IsAdd ? Left : Right;
+    const Register SecondLeft = Result;
+    const Register SecondRight = IsAdd ? Right : Left;
+    auto First = buildBinaryAfter(Builder, Anchor, InstructionOpcode::XOR, *FirstDifference,
+                                  FirstLeft, FirstRight);
+    if (!First)
+      return std::unexpected(First.error());
+    auto Second = buildBinaryAfter(Builder, **First, InstructionOpcode::XOR, *SecondDifference,
+                                   SecondLeft, SecondRight);
+    if (!Second)
+      return std::unexpected(Second.error());
+    auto Both = buildBinaryAfter(Builder, **Second, InstructionOpcode::AND, *SignBits,
+                                 *FirstDifference, *SecondDifference);
+    if (!Both)
+      return std::unexpected(Both.error());
+    const InstructionOpcode Shift =
+        Type.scalarBitWidth() == 32 ? InstructionOpcode::SRLIW : InstructionOpcode::SRLI;
+    auto Extract = buildImmediateAfter(Builder, **Both, Shift, *Overflow, *SignBits,
+                                       Type.scalarBitWidth() - 1U);
+    if (!Extract)
+      return std::unexpected(Extract.error());
+    return std::pair{*Overflow, *Extract};
+  }
+
+  [[nodiscard]] Expected<void> selectOverflow(MachineInstr &Instruction, MachineFunction &Function,
+                                              const TargetDescription &Target,
+                                              std::optional<LLT> Type,
+                                              GenericOpcode Generic) const {
+    const MachineOperand *ResultOperand = Instruction.operandAt(0);
+    const MachineOperand *OverflowOperand = Instruction.operandAt(1);
+    const MachineOperand *LeftOperand = Instruction.operandAt(2);
+    const MachineOperand *RightOperand = Instruction.operandAt(3);
+    const Register *Result = ResultOperand == nullptr ? nullptr : ResultOperand->getIf<Register>();
+    const Register *Overflow =
+        OverflowOperand == nullptr ? nullptr : OverflowOperand->getIf<Register>();
+    const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf<Register>();
+    const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf<Register>();
+    if (!Type || Result == nullptr || Overflow == nullptr || Left == nullptr || Right == nullptr) {
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "RISC-V overflow operation has invalid operands");
+    }
+    const Register ResultRegister = *Result;
+    const Register OverflowRegister = *Overflow;
+    const Register LeftRegister = *Left;
+    const Register RightRegister = *Right;
+    auto RemoveOverflow = Instruction.eraseOperand(1);
+    if (!RemoveOverflow)
+      return std::unexpected(RemoveOverflow.error());
+    const bool IsWord = Type->scalarBitWidth() <= 32;
+    const bool IsAdd = Generic == GenericOpcode::G_UADDO || Generic == GenericOpcode::G_SADDO;
+    const bool IsSub = Generic == GenericOpcode::G_USUBO || Generic == GenericOpcode::G_SSUBO;
+    const bool IsUnsigned = Generic == GenericOpcode::G_UADDO ||
+                            Generic == GenericOpcode::G_USUBO || Generic == GenericOpcode::G_UMULO;
+    const InstructionOpcode Arithmetic =
+        IsAdd   ? (IsWord ? InstructionOpcode::ADDW : InstructionOpcode::ADD)
+        : IsSub ? (IsWord ? InstructionOpcode::SUBW : InstructionOpcode::SUB)
+                : (IsWord ? InstructionOpcode::MULW : InstructionOpcode::MUL);
+    auto SetArithmetic = Instruction.setOpcode(targetOpcode(Arithmetic));
+    if (!SetArithmetic)
+      return std::unexpected(SetArithmetic.error());
+    MachineIRBuilder Builder(Function, Target);
+    if (IsUnsigned && IsAdd) {
+      auto Carry = buildBinaryAfter(Builder, Instruction, InstructionOpcode::SLTU, OverflowRegister,
+                                    ResultRegister, LeftRegister);
+      if (!Carry)
+        return std::unexpected(Carry.error());
+      return {};
+    }
+    if (IsUnsigned && IsSub) {
+      auto Borrow = buildBinaryAfter(Builder, Instruction, InstructionOpcode::SLTU,
+                                     OverflowRegister, LeftRegister, RightRegister);
+      if (!Borrow)
+        return std::unexpected(Borrow.error());
+      return {};
+    }
+    if (IsAdd || IsSub) {
+      auto FirstDifference = createTemporary(Function, *Type, Target);
+      auto SecondDifference = createTemporary(Function, *Type, Target);
+      auto SignBits = createTemporary(Function, *Type, Target);
+      if (!FirstDifference || !SecondDifference || !SignBits) {
+        return std::unexpected(
+            !FirstDifference ? FirstDifference.error()
+                             : (!SecondDifference ? SecondDifference.error() : SignBits.error()));
+      }
+      const Register FirstLeft = IsAdd ? ResultRegister : LeftRegister;
+      const Register FirstRight = IsAdd ? LeftRegister : RightRegister;
+      const Register SecondLeft = ResultRegister;
+      const Register SecondRight = IsAdd ? RightRegister : LeftRegister;
+      auto First = buildBinaryAfter(Builder, Instruction, InstructionOpcode::XOR, *FirstDifference,
+                                    FirstLeft, FirstRight);
+      if (!First)
+        return std::unexpected(First.error());
+      auto Second = buildBinaryAfter(Builder, **First, InstructionOpcode::XOR, *SecondDifference,
+                                     SecondLeft, SecondRight);
+      if (!Second)
+        return std::unexpected(Second.error());
+      auto Both = buildBinaryAfter(Builder, **Second, InstructionOpcode::AND, *SignBits,
+                                   *FirstDifference, *SecondDifference);
+      if (!Both)
+        return std::unexpected(Both.error());
+      auto OverflowResult = buildImmediateAfter(
+          Builder, **Both, IsWord ? InstructionOpcode::SRLIW : InstructionOpcode::SRLI,
+          OverflowRegister, *SignBits, Type->scalarBitWidth() - 1U);
+      if (!OverflowResult)
+        return std::unexpected(OverflowResult.error());
+      return {};
+    }
+
+    auto I64 = LLT::makeInteger(64);
+    if (!I64)
+      return std::unexpected(I64.error());
+    auto Zero = physicalRegister(PhysicalRegisterID::X0);
+    if (!Zero)
+      return std::unexpected(Zero.error());
+    if (!IsWord) {
+      auto High = createTemporary(Function, *I64, Target);
+      if (!High)
+        return std::unexpected(High.error());
+      auto HighProduct = buildBinaryAfter(
+          Builder, Instruction, IsUnsigned ? InstructionOpcode::MULHU : InstructionOpcode::MULH,
+          *High, LeftRegister, RightRegister);
+      if (!HighProduct)
+        return std::unexpected(HighProduct.error());
+      if (IsUnsigned) {
+        auto OverflowResult =
+            buildBinaryAfter(Builder, **HighProduct, InstructionOpcode::SLTU, OverflowRegister,
+                             Register::fromPhysical(*Zero), *High);
+        if (!OverflowResult)
+          return std::unexpected(OverflowResult.error());
+        return {};
+      }
+      auto ExpectedHigh = createTemporary(Function, *I64, Target);
+      auto Difference = createTemporary(Function, *I64, Target);
+      if (!ExpectedHigh || !Difference)
+        return std::unexpected(!ExpectedHigh ? ExpectedHigh.error() : Difference.error());
+      auto Sign = buildImmediateAfter(Builder, **HighProduct, InstructionOpcode::SRAI,
+                                      *ExpectedHigh, ResultRegister, 63);
+      if (!Sign)
+        return std::unexpected(Sign.error());
+      auto Compare = buildBinaryAfter(Builder, **Sign, InstructionOpcode::XOR, *Difference, *High,
+                                      *ExpectedHigh);
+      if (!Compare)
+        return std::unexpected(Compare.error());
+      auto OverflowResult =
+          buildBinaryAfter(Builder, **Compare, InstructionOpcode::SLTU, OverflowRegister,
+                           Register::fromPhysical(*Zero), *Difference);
+      if (!OverflowResult)
+        return std::unexpected(OverflowResult.error());
+      return {};
+    }
+
+    auto ExtendedLeft = createTemporary(Function, *I64, Target);
+    auto ExtendedRight = createTemporary(Function, *I64, Target);
+    auto FullProduct = createTemporary(Function, *I64, Target);
+    if (!ExtendedLeft || !ExtendedRight || !FullProduct) {
+      return std::unexpected(!ExtendedLeft
+                                 ? ExtendedLeft.error()
+                                 : (!ExtendedRight ? ExtendedRight.error() : FullProduct.error()));
+    }
+    MachineInstr *Anchor = &Instruction;
+    if (IsUnsigned) {
+      auto ShiftedLeft = createTemporary(Function, *I64, Target);
+      auto ShiftedRight = createTemporary(Function, *I64, Target);
+      if (!ShiftedLeft || !ShiftedRight)
+        return std::unexpected(!ShiftedLeft ? ShiftedLeft.error() : ShiftedRight.error());
+      auto LeftShift = buildImmediateAfter(Builder, *Anchor, InstructionOpcode::SLLI, *ShiftedLeft,
+                                           LeftRegister, 32);
+      if (!LeftShift)
+        return std::unexpected(LeftShift.error());
+      auto LeftExtend = buildImmediateAfter(Builder, **LeftShift, InstructionOpcode::SRLI,
+                                            *ExtendedLeft, *ShiftedLeft, 32);
+      if (!LeftExtend)
+        return std::unexpected(LeftExtend.error());
+      auto RightShift = buildImmediateAfter(Builder, **LeftExtend, InstructionOpcode::SLLI,
+                                            *ShiftedRight, RightRegister, 32);
+      if (!RightShift)
+        return std::unexpected(RightShift.error());
+      auto RightExtend = buildImmediateAfter(Builder, **RightShift, InstructionOpcode::SRLI,
+                                             *ExtendedRight, *ShiftedRight, 32);
+      if (!RightExtend)
+        return std::unexpected(RightExtend.error());
+      Anchor = *RightExtend;
+    } else {
+      auto LeftExtend = buildImmediateAfter(Builder, *Anchor, InstructionOpcode::ADDIW,
+                                            *ExtendedLeft, LeftRegister, 0);
+      if (!LeftExtend)
+        return std::unexpected(LeftExtend.error());
+      auto RightExtend = buildImmediateAfter(Builder, **LeftExtend, InstructionOpcode::ADDIW,
+                                             *ExtendedRight, RightRegister, 0);
+      if (!RightExtend)
+        return std::unexpected(RightExtend.error());
+      Anchor = *RightExtend;
+    }
+    auto Product = buildBinaryAfter(Builder, *Anchor, InstructionOpcode::MUL, *FullProduct,
+                                    *ExtendedLeft, *ExtendedRight);
+    if (!Product)
+      return std::unexpected(Product.error());
+    if (IsUnsigned) {
+      auto High = createTemporary(Function, *I64, Target);
+      if (!High)
+        return std::unexpected(High.error());
+      auto ExtractHigh =
+          buildImmediateAfter(Builder, **Product, InstructionOpcode::SRLI, *High, *FullProduct, 32);
+      if (!ExtractHigh)
+        return std::unexpected(ExtractHigh.error());
+      auto OverflowResult =
+          buildBinaryAfter(Builder, **ExtractHigh, InstructionOpcode::SLTU, OverflowRegister,
+                           Register::fromPhysical(*Zero), *High);
+      if (!OverflowResult)
+        return std::unexpected(OverflowResult.error());
+      return {};
+    }
+    auto Difference = createTemporary(Function, *I64, Target);
+    if (!Difference)
+      return std::unexpected(Difference.error());
+    auto Compare = buildBinaryAfter(Builder, **Product, InstructionOpcode::XOR, *Difference,
+                                    *FullProduct, ResultRegister);
+    if (!Compare)
+      return std::unexpected(Compare.error());
+    auto OverflowResult =
+        buildBinaryAfter(Builder, **Compare, InstructionOpcode::SLTU, OverflowRegister,
+                         Register::fromPhysical(*Zero), *Difference);
+    if (!OverflowResult)
+      return std::unexpected(OverflowResult.error());
+    return {};
+  }
+
+  [[nodiscard]] Expected<void> selectExtendedOverflow(MachineInstr &Instruction,
+                                                      MachineFunction &Function,
+                                                      const TargetDescription &Target,
+                                                      std::optional<LLT> Type,
+                                                      GenericOpcode Generic) const {
+    const MachineOperand *ResultOperand = Instruction.operandAt(0);
+    const MachineOperand *CarryOutOperand = Instruction.operandAt(1);
+    const MachineOperand *LeftOperand = Instruction.operandAt(2);
+    const MachineOperand *RightOperand = Instruction.operandAt(3);
+    const MachineOperand *CarryInOperand = Instruction.operandAt(4);
+    const Register *Result = ResultOperand == nullptr ? nullptr : ResultOperand->getIf<Register>();
+    const Register *CarryOut =
+        CarryOutOperand == nullptr ? nullptr : CarryOutOperand->getIf<Register>();
+    const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf<Register>();
+    const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf<Register>();
+    const Register *CarryIn =
+        CarryInOperand == nullptr ? nullptr : CarryInOperand->getIf<Register>();
+    if (!Type || Result == nullptr || CarryOut == nullptr || Left == nullptr || Right == nullptr ||
+        CarryIn == nullptr) {
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "RISC-V extended overflow operation has invalid operands");
+    }
+    const Register ResultRegister = *Result;
+    const Register CarryOutRegister = *CarryOut;
+    const Register LeftRegister = *Left;
+    const Register RightRegister = *Right;
+    const Register CarryInRegister = *CarryIn;
+    const bool IsAdd = Generic == GenericOpcode::G_UADDE || Generic == GenericOpcode::G_SADDE;
+    const bool IsUnsigned = Generic == GenericOpcode::G_UADDE || Generic == GenericOpcode::G_USUBE;
+    const bool IsWord = Type->scalarBitWidth() <= 32;
+    const InstructionOpcode Arithmetic =
+        IsAdd ? (IsWord ? InstructionOpcode::ADDW : InstructionOpcode::ADD)
+              : (IsWord ? InstructionOpcode::SUBW : InstructionOpcode::SUB);
+    auto Partial = createTemporary(Function, *Type, Target);
+    auto I1 = LLT::makeInteger(1);
+    if (!Partial || !I1)
+      return std::unexpected(!Partial ? Partial.error() : I1.error());
+    auto NormalizedCarry = createTemporary(Function, *I1, Target);
+    if (!NormalizedCarry)
+      return std::unexpected(NormalizedCarry.error());
+    auto SetPartial = Instruction.operandAt(0)->setRegister(*Partial);
+    if (!SetPartial)
+      return std::unexpected(SetPartial.error());
+    auto RemoveCarryOut = Instruction.eraseOperand(1);
+    if (!RemoveCarryOut)
+      return std::unexpected(RemoveCarryOut.error());
+    auto RemoveCarryIn = Instruction.eraseOperand(3);
+    if (!RemoveCarryIn)
+      return std::unexpected(RemoveCarryIn.error());
+    auto SetArithmetic = Instruction.setOpcode(targetOpcode(Arithmetic));
+    if (!SetArithmetic)
+      return std::unexpected(SetArithmetic.error());
+    MachineIRBuilder Builder(Function, Target);
+    auto Normalize = buildImmediateAfter(Builder, Instruction, InstructionOpcode::ANDI,
+                                         *NormalizedCarry, CarryInRegister, 1);
+    if (!Normalize)
+      return std::unexpected(Normalize.error());
+    auto Final = buildBinaryAfter(Builder, **Normalize, Arithmetic, ResultRegister, *Partial,
+                                  *NormalizedCarry);
+    if (!Final)
+      return std::unexpected(Final.error());
+
+    if (IsUnsigned) {
+      auto FirstCarry = createTemporary(Function, *I1, Target);
+      auto SecondCarry = createTemporary(Function, *I1, Target);
+      if (!FirstCarry || !SecondCarry)
+        return std::unexpected(!FirstCarry ? FirstCarry.error() : SecondCarry.error());
+      auto First =
+          buildBinaryAfter(Builder, **Final, InstructionOpcode::SLTU, *FirstCarry,
+                           IsAdd ? *Partial : LeftRegister, IsAdd ? LeftRegister : RightRegister);
+      if (!First)
+        return std::unexpected(First.error());
+      auto Second =
+          buildBinaryAfter(Builder, **First, InstructionOpcode::SLTU, *SecondCarry,
+                           IsAdd ? ResultRegister : *Partial, IsAdd ? *Partial : *NormalizedCarry);
+      if (!Second)
+        return std::unexpected(Second.error());
+      auto Combined = buildBinaryAfter(Builder, **Second, InstructionOpcode::OR, CarryOutRegister,
+                                       *FirstCarry, *SecondCarry);
+      if (!Combined)
+        return std::unexpected(Combined.error());
+      return {};
+    }
+
+    auto FirstOverflow = buildSignedOverflowAfter(Builder, **Final, Function, Target, *Type,
+                                                  *Partial, LeftRegister, RightRegister, IsAdd);
+    if (!FirstOverflow)
+      return std::unexpected(FirstOverflow.error());
+    auto SecondOverflow =
+        buildSignedOverflowAfter(Builder, *FirstOverflow->second, Function, Target, *Type,
+                                 ResultRegister, *Partial, *NormalizedCarry, IsAdd);
+    if (!SecondOverflow)
+      return std::unexpected(SecondOverflow.error());
+    auto Combined = buildBinaryAfter(Builder, *SecondOverflow->second, InstructionOpcode::OR,
+                                     CarryOutRegister, FirstOverflow->first, SecondOverflow->first);
+    if (!Combined)
+      return std::unexpected(Combined.error());
+    return {};
+  }
+
+  [[nodiscard]] Expected<void> selectAtomicRMW(MachineInstr &Instruction, MachineFunction &Function,
+                                               const TargetDescription &Target,
+                                               std::optional<LLT> Type,
+                                               GenericOpcode Generic) const {
+    const MachineOperand *AddressOperand = Instruction.operandAt(1);
+    const MachineOperand *ValueOperand = Instruction.operandAt(2);
+    const Register *Address =
+        AddressOperand == nullptr ? nullptr : AddressOperand->getIf<Register>();
+    const Register *Value = ValueOperand == nullptr ? nullptr : ValueOperand->getIf<Register>();
+    if (!Type || Address == nullptr || Value == nullptr)
+      return makeUnexpected(ErrorCode::VerificationError, "RISC-V atomic RMW has invalid operands");
+    Register SelectedValue = *Value;
+    const bool IsWord = Type->scalarBitWidth() == 32;
+    if (Generic == GenericOpcode::G_ATOMICRMW_SUB) {
+      auto Negated = createTemporary(Function, *Type, Target);
+      auto Zero = physicalRegister(PhysicalRegisterID::X0);
+      if (!Negated || !Zero)
+        return std::unexpected(!Negated ? Negated.error() : Zero.error());
+      MachineIRBuilder Builder(Function, Target);
+      auto Point = Builder.setInsertPointBefore(Instruction);
+      if (!Point)
+        return std::unexpected(Point.error());
+      auto Pending = Builder.buildInstr(
+          targetOpcode(IsWord ? InstructionOpcode::SUBW : InstructionOpcode::SUB));
+      if (!Pending)
+        return std::unexpected(Pending.error());
+      auto Def = Pending->addDef(*Negated);
+      if (!Def)
+        return std::unexpected(Def.error());
+      auto Left = Pending->addUse(Register::fromPhysical(*Zero));
+      if (!Left)
+        return std::unexpected(Left.error());
+      auto Right = Pending->addUse(*Value);
+      if (!Right)
+        return std::unexpected(Right.error());
+      auto Built = Pending->build();
+      if (!Built)
+        return std::unexpected(Built.error());
+      SelectedValue = *Negated;
+    }
+    const auto SelectOpcode = [Generic, IsWord]() -> InstructionOpcode {
+      switch (Generic) {
+      case GenericOpcode::G_ATOMICRMW_XCHG:
+        return IsWord ? InstructionOpcode::AMOSWAP_W : InstructionOpcode::AMOSWAP_D;
+      case GenericOpcode::G_ATOMICRMW_ADD:
+      case GenericOpcode::G_ATOMICRMW_SUB:
+        return IsWord ? InstructionOpcode::AMOADD_W : InstructionOpcode::AMOADD_D;
+      case GenericOpcode::G_ATOMICRMW_AND:
+        return IsWord ? InstructionOpcode::AMOAND_W : InstructionOpcode::AMOAND_D;
+      case GenericOpcode::G_ATOMICRMW_OR:
+        return IsWord ? InstructionOpcode::AMOOR_W : InstructionOpcode::AMOOR_D;
+      case GenericOpcode::G_ATOMICRMW_XOR:
+        return IsWord ? InstructionOpcode::AMOXOR_W : InstructionOpcode::AMOXOR_D;
+      case GenericOpcode::G_ATOMICRMW_MAX:
+        return IsWord ? InstructionOpcode::AMOMAX_W : InstructionOpcode::AMOMAX_D;
+      case GenericOpcode::G_ATOMICRMW_MIN:
+        return IsWord ? InstructionOpcode::AMOMIN_W : InstructionOpcode::AMOMIN_D;
+      case GenericOpcode::G_ATOMICRMW_UMAX:
+        return IsWord ? InstructionOpcode::AMOMAXU_W : InstructionOpcode::AMOMAXU_D;
+      case GenericOpcode::G_ATOMICRMW_UMIN:
+        return IsWord ? InstructionOpcode::AMOMINU_W : InstructionOpcode::AMOMINU_D;
+      default:
+        return InstructionOpcode::Invalid;
+      }
+    };
+    const InstructionOpcode Selected = SelectOpcode();
+    if (Selected == InstructionOpcode::Invalid)
+      return makeUnexpected(ErrorCode::InternalError,
+                            "RISC-V atomic RMW selection reached an unhandled opcode");
+    auto SetValue = Instruction.operandAt(1)->setRegister(SelectedValue);
+    if (!SetValue)
+      return std::unexpected(SetValue.error());
+    auto SetAddress = Instruction.operandAt(2)->setRegister(*Address);
+    if (!SetAddress)
+      return std::unexpected(SetAddress.error());
+    return Instruction.setOpcode(targetOpcode(Selected));
+  }
+
+  [[nodiscard]] Expected<void> selectIntegerExtension(MachineInstr &Instruction,
+                                                      MachineFunction &Function,
+                                                      const TargetDescription &Target,
+                                                      GenericOpcode Generic,
+                                                      std::optional<LLT> Type) const {
+    const std::optional<LLT> SourceType = operandType(Function, Instruction, 1);
+    const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+    const MachineOperand *SourceOperand = Instruction.operandAt(1);
+    const Register *Destination =
+        DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+    const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>();
+    if (!Type || !SourceType || Destination == nullptr || Source == nullptr)
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "RISC-V integer extension has invalid register operands");
+    const Register DestinationRegister = *Destination;
+    if (Generic == GenericOpcode::G_ANYEXT)
+      return Instruction.setOpcode(opcode(GenericOpcode::COPY));
+
+    const std::uint32_t DestinationBits = Type->scalarBitWidth();
+    const std::uint32_t SourceBits = SourceType->scalarBitWidth();
+    if (Generic == GenericOpcode::G_SEXT && DestinationBits == 64 && SourceBits == 32) {
+      auto ImmediateResult = appendImmediate(Instruction, 0);
+      if (!ImmediateResult)
+        return std::unexpected(ImmediateResult.error());
+      return Instruction.setOpcode(targetOpcode(InstructionOpcode::ADDIW));
+    }
+
+    const std::uint32_t ContainerBits = DestinationBits <= 32 ? 32 : 64;
+    const std::int64_t Shift = static_cast<std::int64_t>(ContainerBits - SourceBits);
+    const InstructionOpcode ShiftLeft =
+        ContainerBits == 32 ? InstructionOpcode::SLLIW : InstructionOpcode::SLLI;
+    const InstructionOpcode ShiftRight =
+        Generic == GenericOpcode::G_SEXT
+            ? (ContainerBits == 32 ? InstructionOpcode::SRAIW : InstructionOpcode::SRAI)
+            : (ContainerBits == 32 ? InstructionOpcode::SRLIW : InstructionOpcode::SRLI);
+    auto Temporary = createTemporary(Function, *Type, Target);
+    if (!Temporary)
+      return std::unexpected(Temporary.error());
+    auto SetDestination = Instruction.operandAt(0)->setRegister(*Temporary);
+    if (!SetDestination)
+      return std::unexpected(SetDestination.error());
+    auto SetOpcode = Instruction.setOpcode(targetOpcode(ShiftLeft));
+    if (!SetOpcode)
+      return std::unexpected(SetOpcode.error());
+    auto ShiftImmediate = appendImmediate(Instruction, Shift);
+    if (!ShiftImmediate)
+      return std::unexpected(ShiftImmediate.error());
+
+    MachineIRBuilder Builder(Function, Target);
+    auto Shifted = buildImmediateAfter(Builder, Instruction, ShiftRight, DestinationRegister,
+                                       *Temporary, Shift);
+    if (!Shifted)
+      return std::unexpected(Shifted.error());
+    return {};
+  }
+
+  [[nodiscard]] Expected<void> selectSignExtendInRegister(MachineInstr &Instruction,
+                                                          MachineFunction &Function,
+                                                          const TargetDescription &Target,
+                                                          std::optional<LLT> Type) const {
+    const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+    const MachineOperand *SourceOperand = Instruction.operandAt(1);
+    const Register *Destination =
+        DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+    const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>();
+    std::optional<std::int64_t> Width;
+    if (const MachineOperand *WidthOperand = Instruction.operandAt(2); WidthOperand != nullptr) {
+      auto ParsedWidth = signedImmediate(*WidthOperand);
+      if (ParsedWidth)
+        Width = *ParsedWidth;
+    }
+    if (!Type || Destination == nullptr || Source == nullptr || !Width || *Width <= 0 ||
+        static_cast<std::uint64_t>(*Width) > Type->scalarBitWidth())
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "RISC-V sign-extension-in-register has invalid operands");
+    const Register DestinationRegister = *Destination;
+    if (static_cast<std::uint64_t>(*Width) == Type->scalarBitWidth()) {
+      auto EraseWidth = Instruction.eraseOperand(2);
+      if (!EraseWidth)
+        return std::unexpected(EraseWidth.error());
+      return Instruction.setOpcode(opcode(GenericOpcode::COPY));
+    }
+
+    const std::uint32_t TypeBits = Type->scalarBitWidth();
+    if (TypeBits == 64 && *Width == 32) {
+      auto ImmediateResult = replaceImmediate(Instruction, 2, 0);
+      if (!ImmediateResult)
+        return std::unexpected(ImmediateResult.error());
+      return Instruction.setOpcode(targetOpcode(InstructionOpcode::ADDIW));
+    }
+    const std::uint32_t ContainerBits = TypeBits <= 32 ? 32 : 64;
+    const std::int64_t Shift = static_cast<std::int64_t>(ContainerBits) - *Width;
+    const InstructionOpcode ShiftLeft =
+        ContainerBits == 32 ? InstructionOpcode::SLLIW : InstructionOpcode::SLLI;
+    const InstructionOpcode ShiftRight =
+        ContainerBits == 32 ? InstructionOpcode::SRAIW : InstructionOpcode::SRAI;
+    auto Temporary = createTemporary(Function, *Type, Target);
+    if (!Temporary)
+      return std::unexpected(Temporary.error());
+    auto SetDestination = Instruction.operandAt(0)->setRegister(*Temporary);
+    if (!SetDestination)
+      return std::unexpected(SetDestination.error());
+    auto SetOpcode = Instruction.setOpcode(targetOpcode(ShiftLeft));
+    if (!SetOpcode)
+      return std::unexpected(SetOpcode.error());
+    auto ShiftImmediate = replaceImmediate(Instruction, 2, Shift);
+    if (!ShiftImmediate)
+      return std::unexpected(ShiftImmediate.error());
+
+    MachineIRBuilder Builder(Function, Target);
+    auto Shifted = buildImmediateAfter(Builder, Instruction, ShiftRight, DestinationRegister,
+                                       *Temporary, Shift);
+    if (!Shifted)
+      return std::unexpected(Shifted.error());
+    return {};
+  }
+
+  [[nodiscard]] Expected<void> selectIntegerCompare(MachineInstr &Instruction,
+                                                    MachineFunction &Function,
+                                                    const TargetDescription &Target) const {
+    const MachineOperand *PredicateMO = Instruction.operandAt(1);
+    const auto *Predicate =
+        PredicateMO == nullptr ? nullptr : PredicateMO->getIf<PredicateOperand>();
+    const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+    const MachineOperand *LeftOperand = Instruction.operandAt(2);
+    const MachineOperand *RightOperand = Instruction.operandAt(3);
+    const Register *Destination =
+        DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+    const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf<Register>();
+    const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf<Register>();
+    const std::optional<LLT> LeftType = operandType(Function, Instruction, 2);
+    if (Predicate == nullptr || Predicate->integerPredicate() == nullptr ||
+        Destination == nullptr || Left == nullptr || Right == nullptr || !LeftType)
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "RISC-V integer comparison has invalid operands");
+    const IntegerPredicate Condition = *Predicate->integerPredicate();
+    const Register DestinationRegister = *Destination;
+    const bool SignedOperands = Condition == IntegerPredicate::SignedGreaterThan ||
+                                Condition == IntegerPredicate::SignedGreaterEqual ||
+                                Condition == IntegerPredicate::SignedLessThan ||
+                                Condition == IntegerPredicate::SignedLessEqual;
+    auto LeftRegister =
+        normalizeIntegerBefore(Instruction, Function, Target, *Left, *LeftType, SignedOperands);
+    auto RightRegister =
+        normalizeIntegerBefore(Instruction, Function, Target, *Right, *LeftType, SignedOperands);
+    if (!LeftRegister || !RightRegister)
+      return std::unexpected(!LeftRegister ? LeftRegister.error() : RightRegister.error());
+    auto ErasePredicate = Instruction.eraseOperand(1);
+    if (!ErasePredicate)
+      return std::unexpected(ErasePredicate.error());
+
+    const auto SetSources = [&Instruction](Register NewLeft, Register NewRight) -> Expected<void> {
+      auto SetLeft = Instruction.operandAt(1)->setRegister(NewLeft);
+      if (!SetLeft)
+        return std::unexpected(SetLeft.error());
+      return Instruction.operandAt(2)->setRegister(NewRight);
+    };
+    const auto StrictOpcode = [](IntegerPredicate Value) -> std::optional<InstructionOpcode> {
+      switch (Value) {
+      case IntegerPredicate::SignedLessThan:
+        return InstructionOpcode::SLT;
+      case IntegerPredicate::UnsignedLessThan:
+        return InstructionOpcode::SLTU;
+      default:
+        return std::nullopt;
+      }
+    };
+
+    IntegerPredicate StrictCondition = Condition;
+    bool Swap = false;
+    bool Invert = false;
+    switch (Condition) {
+    case IntegerPredicate::Equal:
+    case IntegerPredicate::NotEqual:
+      break;
+    case IntegerPredicate::SignedGreaterThan:
+      StrictCondition = IntegerPredicate::SignedLessThan;
+      Swap = true;
+      break;
+    case IntegerPredicate::SignedGreaterEqual:
+      StrictCondition = IntegerPredicate::SignedLessThan;
+      Invert = true;
+      break;
+    case IntegerPredicate::SignedLessThan:
+      StrictCondition = IntegerPredicate::SignedLessThan;
+      break;
+    case IntegerPredicate::SignedLessEqual:
+      StrictCondition = IntegerPredicate::SignedLessThan;
+      Swap = true;
+      Invert = true;
+      break;
+    case IntegerPredicate::UnsignedGreaterThan:
+      StrictCondition = IntegerPredicate::UnsignedLessThan;
+      Swap = true;
+      break;
+    case IntegerPredicate::UnsignedGreaterEqual:
+      StrictCondition = IntegerPredicate::UnsignedLessThan;
+      Invert = true;
+      break;
+    case IntegerPredicate::UnsignedLessThan:
+      StrictCondition = IntegerPredicate::UnsignedLessThan;
+      break;
+    case IntegerPredicate::UnsignedLessEqual:
+      StrictCondition = IntegerPredicate::UnsignedLessThan;
+      Swap = true;
+      Invert = true;
+      break;
+    case IntegerPredicate::Count:
+      return makeUnexpected(ErrorCode::Unsupported,
+                            "RISC-V integer comparison predicate is invalid");
+    }
+
+    MachineIRBuilder Builder(Function, Target);
+    if (Condition == IntegerPredicate::Equal || Condition == IntegerPredicate::NotEqual) {
+      auto Temporary = createTemporary(Function, *LeftType, Target);
+      if (!Temporary)
+        return std::unexpected(Temporary.error());
+      auto SetDestination = Instruction.operandAt(0)->setRegister(*Temporary);
+      if (!SetDestination)
+        return std::unexpected(SetDestination.error());
+      auto Set = SetSources(*LeftRegister, *RightRegister);
+      if (!Set)
+        return std::unexpected(Set.error());
+      auto SetOpcode = Instruction.setOpcode(targetOpcode(InstructionOpcode::XOR));
+      if (!SetOpcode)
+        return std::unexpected(SetOpcode.error());
+      if (Condition == IntegerPredicate::Equal) {
+        auto Result = buildImmediateAfter(Builder, Instruction, InstructionOpcode::SLTIU,
+                                          DestinationRegister, *Temporary, 1);
+        if (!Result)
+          return std::unexpected(Result.error());
+      } else {
+        auto Zero = physicalRegister(PhysicalRegisterID::X0);
+        if (!Zero)
+          return std::unexpected(Zero.error());
+        auto Result =
+            buildBinaryAfter(Builder, Instruction, InstructionOpcode::SLTU, DestinationRegister,
+                             Register::fromPhysical(*Zero), *Temporary);
+        if (!Result)
+          return std::unexpected(Result.error());
+      }
+      return {};
+    }
+
+    const std::optional<InstructionOpcode> OpcodeValue = StrictOpcode(StrictCondition);
+    if (!OpcodeValue)
+      return makeUnexpected(ErrorCode::Unsupported,
+                            "RISC-V integer comparison predicate is invalid");
+    if (Invert) {
+      auto Temporary = createTemporary(Function, *LeftType, Target);
+      if (!Temporary)
+        return std::unexpected(Temporary.error());
+      auto SetDestination = Instruction.operandAt(0)->setRegister(*Temporary);
+      if (!SetDestination)
+        return std::unexpected(SetDestination.error());
+      const Register Base = *Temporary;
+      if (Swap) {
+        auto Set = SetSources(*RightRegister, *LeftRegister);
+        if (!Set)
+          return std::unexpected(Set.error());
+      } else {
+        auto Set = SetSources(*LeftRegister, *RightRegister);
+        if (!Set)
+          return std::unexpected(Set.error());
+      }
+      auto SetOpcode = Instruction.setOpcode(targetOpcode(*OpcodeValue));
+      if (!SetOpcode)
+        return std::unexpected(SetOpcode.error());
+      auto Result = buildImmediateAfter(Builder, Instruction, InstructionOpcode::XORI,
+                                        DestinationRegister, Base, 1);
+      if (!Result)
+        return std::unexpected(Result.error());
+      return {};
+    }
+    if (Swap) {
+      auto Set = SetSources(*RightRegister, *LeftRegister);
+      if (!Set)
+        return std::unexpected(Set.error());
+    } else {
+      auto Set = SetSources(*LeftRegister, *RightRegister);
+      if (!Set)
+        return std::unexpected(Set.error());
+    }
+    return Instruction.setOpcode(targetOpcode(*OpcodeValue));
+  }
+
+  [[nodiscard]] Expected<void> selectFloatingCompare(MachineInstr &Instruction,
+                                                     MachineFunction &Function,
+                                                     const TargetDescription &Target) const {
+    const MachineOperand *PredicateMO = Instruction.operandAt(1);
+    const auto *Predicate =
+        PredicateMO == nullptr ? nullptr : PredicateMO->getIf<PredicateOperand>();
+    const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+    const MachineOperand *LeftOperand = Instruction.operandAt(2);
+    const MachineOperand *RightOperand = Instruction.operandAt(3);
+    const Register *Destination =
+        DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+    const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf<Register>();
+    const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf<Register>();
+    const std::optional<LLT> FloatType = operandType(Function, Instruction, 2);
+    const std::optional<LLT> ResultType = resultType(Function, Instruction);
+    if (Predicate == nullptr || Predicate->floatingPredicate() == nullptr ||
+        Destination == nullptr || Left == nullptr || Right == nullptr || !FloatType || !ResultType)
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "RISC-V floating comparison has invalid operands");
+    const FloatingPredicate Condition = *Predicate->floatingPredicate();
+    const Register DestinationRegister = *Destination;
+    const Register LeftRegister = *Left;
+    const Register RightRegister = *Right;
+    const InstructionOpcode EqualOpcode =
+        FloatType->scalarBitWidth() == 32 ? InstructionOpcode::FEQ_S : InstructionOpcode::FEQ_D;
+    const InstructionOpcode LessOpcode =
+        FloatType->scalarBitWidth() == 32 ? InstructionOpcode::FLT_S : InstructionOpcode::FLT_D;
+    const InstructionOpcode LessEqualOpcode =
+        FloatType->scalarBitWidth() == 32 ? InstructionOpcode::FLE_S : InstructionOpcode::FLE_D;
+
+    auto ErasePredicate = Instruction.eraseOperand(1);
+    if (!ErasePredicate)
+      return std::unexpected(ErasePredicate.error());
+    const auto SetBase = [&Instruction](InstructionOpcode OpcodeValue, Register Def,
+                                        Register NewLeft, Register NewRight) -> Expected<void> {
+      auto SetDestination = Instruction.operandAt(0)->setRegister(Def);
+      if (!SetDestination)
+        return std::unexpected(SetDestination.error());
+      auto SetLeft = Instruction.operandAt(1)->setRegister(NewLeft);
+      if (!SetLeft)
+        return std::unexpected(SetLeft.error());
+      auto SetRight = Instruction.operandAt(2)->setRegister(NewRight);
+      if (!SetRight)
+        return std::unexpected(SetRight.error());
+      return Instruction.setOpcode(targetOpcode(OpcodeValue));
+    };
+    const auto BuildInverted = [&](InstructionOpcode BaseOpcode, Register BaseLeft,
+                                   Register BaseRight) -> Expected<void> {
+      auto Temporary = createTemporary(Function, *ResultType, Target);
+      if (!Temporary)
+        return std::unexpected(Temporary.error());
+      auto Base = SetBase(BaseOpcode, *Temporary, BaseLeft, BaseRight);
+      if (!Base)
+        return std::unexpected(Base.error());
+      MachineIRBuilder Builder(Function, Target);
+      auto Invert = buildImmediateAfter(Builder, Instruction, InstructionOpcode::XORI,
+                                        DestinationRegister, *Temporary, 1);
+      if (!Invert)
+        return std::unexpected(Invert.error());
+      return {};
+    };
+    const auto BuildSelfAnd = [&](bool Invert) -> Expected<void> {
+      auto First = createTemporary(Function, *ResultType, Target);
+      auto Second = createTemporary(Function, *ResultType, Target);
+      if (!First || !Second)
+        return std::unexpected((!First ? First.error() : Second.error()));
+      auto Base = SetBase(EqualOpcode, *First, LeftRegister, LeftRegister);
+      if (!Base)
+        return std::unexpected(Base.error());
+      MachineIRBuilder Builder(Function, Target);
+      auto SecondInstruction = buildBinaryAfter(Builder, Instruction, EqualOpcode, *Second,
+                                                RightRegister, RightRegister);
+      if (!SecondInstruction)
+        return std::unexpected(SecondInstruction.error());
+      Register CombinedDestination = DestinationRegister;
+      if (Invert) {
+        auto Temporary = createTemporary(Function, *ResultType, Target);
+        if (!Temporary)
+          return std::unexpected(Temporary.error());
+        CombinedDestination = *Temporary;
+      }
+      auto Combined = buildBinaryAfter(Builder, **SecondInstruction, InstructionOpcode::AND,
+                                       CombinedDestination, *First, *Second);
+      if (!Combined)
+        return std::unexpected(Combined.error());
+      if (!Invert)
+        return {};
+      auto Inverted = buildImmediateAfter(Builder, **Combined, InstructionOpcode::XORI,
+                                          DestinationRegister, CombinedDestination, 1);
+      if (!Inverted)
+        return std::unexpected(Inverted.error());
+      return {};
+    };
+    const auto BuildOrderedNotEqual = [&](bool Invert) -> Expected<void> {
+      auto First = createTemporary(Function, *ResultType, Target);
+      auto Second = createTemporary(Function, *ResultType, Target);
+      if (!First || !Second)
+        return std::unexpected((!First ? First.error() : Second.error()));
+      auto Base = SetBase(LessOpcode, *First, LeftRegister, RightRegister);
+      if (!Base)
+        return std::unexpected(Base.error());
+      MachineIRBuilder Builder(Function, Target);
+      auto SecondInstruction =
+          buildBinaryAfter(Builder, Instruction, LessOpcode, *Second, RightRegister, LeftRegister);
+      if (!SecondInstruction)
+        return std::unexpected(SecondInstruction.error());
+      Register CombinedDestination = DestinationRegister;
+      if (Invert) {
+        auto Temporary = createTemporary(Function, *ResultType, Target);
+        if (!Temporary)
+          return std::unexpected(Temporary.error());
+        CombinedDestination = *Temporary;
+      }
+      auto Combined = buildBinaryAfter(Builder, **SecondInstruction, InstructionOpcode::OR,
+                                       CombinedDestination, *First, *Second);
+      if (!Combined)
+        return std::unexpected(Combined.error());
+      if (!Invert)
+        return {};
+      auto Inverted = buildImmediateAfter(Builder, **Combined, InstructionOpcode::XORI,
+                                          DestinationRegister, CombinedDestination, 1);
+      if (!Inverted)
+        return std::unexpected(Inverted.error());
+      return {};
+    };
+
+    switch (Condition) {
+    case FloatingPredicate::AlwaysFalse:
+    case FloatingPredicate::AlwaysTrue: {
+      auto EraseRight = Instruction.eraseOperand(2);
+      if (!EraseRight)
+        return std::unexpected(EraseRight.error());
+      auto EraseLeft = Instruction.eraseOperand(1);
+      if (!EraseLeft)
+        return std::unexpected(EraseLeft.error());
+      auto Zero = physicalRegister(PhysicalRegisterID::X0);
+      if (!Zero)
+        return std::unexpected(Zero.error());
+      auto InsertZero = insertPhysicalRegister(Instruction, 1, *Zero, RegisterOperandFlag::Use);
+      if (!InsertZero)
+        return std::unexpected(InsertZero.error());
+      auto Immediate =
+          appendImmediate(Instruction, Condition == FloatingPredicate::AlwaysTrue ? 1 : 0);
+      if (!Immediate)
+        return std::unexpected(Immediate.error());
+      return Instruction.setOpcode(targetOpcode(InstructionOpcode::ADDI));
+    }
+    case FloatingPredicate::OrderedEqual:
+      return SetBase(EqualOpcode, DestinationRegister, LeftRegister, RightRegister);
+    case FloatingPredicate::OrderedGreaterThan:
+      return SetBase(LessOpcode, DestinationRegister, RightRegister, LeftRegister);
+    case FloatingPredicate::OrderedGreaterEqual:
+      return SetBase(LessEqualOpcode, DestinationRegister, RightRegister, LeftRegister);
+    case FloatingPredicate::OrderedLessThan:
+      return SetBase(LessOpcode, DestinationRegister, LeftRegister, RightRegister);
+    case FloatingPredicate::OrderedLessEqual:
+      return SetBase(LessEqualOpcode, DestinationRegister, LeftRegister, RightRegister);
+    case FloatingPredicate::OrderedNotEqual:
+      return BuildOrderedNotEqual(false);
+    case FloatingPredicate::Ordered:
+      return BuildSelfAnd(false);
+    case FloatingPredicate::Unordered:
+      return BuildSelfAnd(true);
+    case FloatingPredicate::UnorderedEqual:
+      return BuildOrderedNotEqual(true);
+    case FloatingPredicate::UnorderedGreaterThan:
+      return BuildInverted(LessEqualOpcode, LeftRegister, RightRegister);
+    case FloatingPredicate::UnorderedGreaterEqual:
+      return BuildInverted(LessOpcode, LeftRegister, RightRegister);
+    case FloatingPredicate::UnorderedLessThan:
+      return BuildInverted(LessEqualOpcode, RightRegister, LeftRegister);
+    case FloatingPredicate::UnorderedLessEqual:
+      return BuildInverted(LessOpcode, RightRegister, LeftRegister);
+    case FloatingPredicate::UnorderedNotEqual:
+      return BuildInverted(EqualOpcode, LeftRegister, RightRegister);
+    case FloatingPredicate::Count:
+      return makeUnexpected(ErrorCode::Unsupported,
+                            "RISC-V floating comparison predicate is invalid");
+    }
+    return makeUnexpected(ErrorCode::Unsupported,
+                          "RISC-V floating comparison predicate is invalid");
+  }
+
+  [[nodiscard]] Expected<void> selectFloatToInteger(MachineInstr &Instruction,
+                                                    MachineFunction &Function,
+                                                    std::optional<LLT> Type, bool Unsigned) const {
+    const std::optional<LLT> Source = operandType(Function, Instruction, 1);
+    if (!Type || !Source)
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "RISC-V floating-to-integer conversion has invalid types");
+    auto Rounding = appendImmediate(Instruction, 1);
+    if (!Rounding)
+      return std::unexpected(Rounding.error());
+    const bool IsWord = Type->scalarBitWidth() <= 32;
+    if (Source->scalarBitWidth() == 32) {
+      return Instruction.setOpcode(targetOpcode(
+          IsWord ? (Unsigned ? InstructionOpcode::FCVT_WU_S : InstructionOpcode::FCVT_W_S)
+                 : (Unsigned ? InstructionOpcode::FCVT_LU_S : InstructionOpcode::FCVT_L_S)));
+    }
+    return Instruction.setOpcode(targetOpcode(
+        IsWord ? (Unsigned ? InstructionOpcode::FCVT_WU_D : InstructionOpcode::FCVT_W_D)
+               : (Unsigned ? InstructionOpcode::FCVT_LU_D : InstructionOpcode::FCVT_L_D)));
+  }
+
+  [[nodiscard]] Expected<void> selectIntegerToFloat(MachineInstr &Instruction,
+                                                    MachineFunction &Function,
+                                                    const TargetDescription &Target,
+                                                    std::optional<LLT> Type, bool Unsigned) const {
+    const std::optional<LLT> Source = operandType(Function, Instruction, 1);
+    const MachineOperand *SourceOperand = Instruction.operandAt(1);
+    const Register *SourceRegister =
+        SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>();
+    if (!Type || !Source || SourceRegister == nullptr)
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "RISC-V integer-to-floating conversion has invalid types");
+    auto Normalized =
+        normalizeIntegerBefore(Instruction, Function, Target, *SourceRegister, *Source, !Unsigned);
+    if (!Normalized)
+      return std::unexpected(Normalized.error());
+    auto SetSource = Instruction.operandAt(1)->setRegister(*Normalized);
+    if (!SetSource)
+      return std::unexpected(SetSource.error());
+    auto Rounding = appendImmediate(Instruction, 0);
+    if (!Rounding)
+      return std::unexpected(Rounding.error());
+    const bool IsWord = Source->scalarBitWidth() <= 32;
+    if (Type->scalarBitWidth() == 32) {
+      return Instruction.setOpcode(targetOpcode(
+          IsWord ? (Unsigned ? InstructionOpcode::FCVT_S_WU : InstructionOpcode::FCVT_S_W)
+                 : (Unsigned ? InstructionOpcode::FCVT_S_LU : InstructionOpcode::FCVT_S_L)));
+    }
+    return Instruction.setOpcode(targetOpcode(
+        IsWord ? (Unsigned ? InstructionOpcode::FCVT_D_WU : InstructionOpcode::FCVT_D_W)
+               : (Unsigned ? InstructionOpcode::FCVT_D_LU : InstructionOpcode::FCVT_D_L)));
+  }
+
+  [[nodiscard]] Expected<void> selectFloatingMinMax(MachineInstr &Instruction,
+                                                    std::optional<LLT> Type,
+                                                    InstructionOpcode Single,
+                                                    InstructionOpcode Double) const {
+    if (Type && Type->isFloat() && Type->scalarBitWidth() == 32)
+      return Instruction.setOpcode(targetOpcode(Single));
+    if (Type && Type->isFloat() && Type->scalarBitWidth() == 64)
+      return Instruction.setOpcode(targetOpcode(Double));
+    return makeUnexpected(ErrorCode::Unsupported,
+                          "RISC-V cannot select this floating min/max type");
+  }
+
+  [[nodiscard]] Expected<void> selectFence(MachineInstr &Instruction) const {
+    const auto Ordering = signedImmediate(*Instruction.operandAt(0));
+    if (!Ordering)
+      return std::unexpected(Ordering.error());
+    std::int64_t Pred = 0;
+    std::int64_t Succ = 0;
+    switch (*Ordering) {
+    case 4:
+      Pred = 1;
+      Succ = 3;
+      break;
+    case 5:
+      Pred = 3;
+      Succ = 2;
+      break;
+    case 6:
+    case 7:
+      Pred = 3;
+      Succ = 3;
+      break;
+    default:
+      return makeUnexpected(ErrorCode::Unsupported, "RISC-V fence ordering is unsupported");
+    }
+    auto SetPred = replaceImmediate(Instruction, 0, Pred);
+    if (!SetPred)
+      return std::unexpected(SetPred.error());
+    auto SetSucc = replaceImmediate(Instruction, 1, Succ);
+    if (!SetSucc)
+      return std::unexpected(SetSucc.error());
+    return Instruction.setOpcode(targetOpcode(InstructionOpcode::FENCE));
+  }
+
+  [[nodiscard]] Expected<void> selectIndirectBranch(MachineInstr &Instruction) const {
+    auto Zero = physicalRegister(PhysicalRegisterID::X0);
+    if (!Zero)
+      return std::unexpected(Zero.error());
+    auto Inserted = insertPhysicalRegister(Instruction, 0, *Zero, RegisterOperandFlag::Define);
+    if (!Inserted)
+      return std::unexpected(Inserted.error());
+    auto Immediate = appendImmediate(Instruction, 0);
+    if (!Immediate)
+      return std::unexpected(Immediate.error());
+    return Instruction.setOpcode(targetOpcode(InstructionOpcode::JALR));
+  }
+
+  [[nodiscard]] Expected<void> selectTrap(MachineInstr &Instruction) const {
+    while (Instruction.operandCount() != 0) {
+      auto Erased = Instruction.eraseOperand(Instruction.operandCount() - 1U);
+      if (!Erased)
+        return std::unexpected(Erased.error());
+    }
+    return Instruction.setOpcode(targetOpcode(InstructionOpcode::EBREAK));
+  }
+
+  [[nodiscard]] Expected<void> selectBitcast(MachineInstr &Instruction,
+                                             MachineFunction &Function) const {
+    const std::optional<LLT> DestinationType = resultType(Function, Instruction);
+    const std::optional<LLT> SourceType = operandType(Function, Instruction, 1);
+    if (!DestinationType || !SourceType)
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "RISC-V bitcast has invalid register types");
+    if (DestinationType->isInteger() && SourceType->isFloat()) {
+      return Instruction.setOpcode(targetOpcode(SourceType->scalarBitWidth() == 32
+                                                    ? InstructionOpcode::FMV_X_W
+                                                    : InstructionOpcode::FMV_X_D));
+    }
+    if (DestinationType->isFloat() && SourceType->isInteger()) {
+      return Instruction.setOpcode(targetOpcode(DestinationType->scalarBitWidth() == 32
+                                                    ? InstructionOpcode::FMV_W_X
+                                                    : InstructionOpcode::FMV_D_X));
+    }
+    return makeUnexpected(ErrorCode::Unsupported, "RISC-V bitcast pair is unsupported");
+  }
+
+  [[nodiscard]] Expected<void> selectIntegerBinary(MachineInstr &Instruction,
+                                                   std::optional<LLT> Type,
+                                                   InstructionOpcode DoubleWord,
+                                                   InstructionOpcode Word) const {
+    if (!Type || !isSupportedIntegerType(*Type))
+      return makeUnexpected(ErrorCode::Unsupported,
+                            "RISC-V integer instruction requires a supported scalar integer");
+    const bool IsWord = Type->isInteger() && Type->scalarBitWidth() <= 32;
+    return Instruction.setOpcode(targetOpcode(IsWord ? Word : DoubleWord));
+  }
+
+  [[nodiscard]] Expected<void>
+  selectNormalizedIntegerBinary(MachineInstr &Instruction, MachineFunction &Function,
+                                const TargetDescription &Target, std::optional<LLT> Type,
+                                InstructionOpcode DoubleWord, InstructionOpcode Word, bool Signed,
+                                bool NormalizeRight) const {
+    const MachineOperand *LeftOperand = Instruction.operandAt(1);
+    const MachineOperand *RightOperand = Instruction.operandAt(2);
+    const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf<Register>();
+    const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf<Register>();
+    if (!Type || !isSupportedIntegerScalar(*Type) || Left == nullptr || Right == nullptr) {
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "RISC-V normalized integer operation has invalid operands");
+    }
+    auto NormalizedLeft =
+        normalizeIntegerBefore(Instruction, Function, Target, *Left, *Type, Signed);
+    if (!NormalizedLeft)
+      return std::unexpected(NormalizedLeft.error());
+    Register SelectedRight = *Right;
+    if (NormalizeRight) {
+      auto NormalizedRight =
+          normalizeIntegerBefore(Instruction, Function, Target, *Right, *Type, Signed);
+      if (!NormalizedRight)
+        return std::unexpected(NormalizedRight.error());
+      SelectedRight = *NormalizedRight;
+    }
+    auto SetLeft = Instruction.operandAt(1)->setRegister(*NormalizedLeft);
+    if (!SetLeft)
+      return std::unexpected(SetLeft.error());
+    auto SetRight = Instruction.operandAt(2)->setRegister(SelectedRight);
+    if (!SetRight)
+      return std::unexpected(SetRight.error());
+    const bool IsWord = Type->scalarBitWidth() <= 32;
+    return Instruction.setOpcode(targetOpcode(IsWord ? Word : DoubleWord));
+  }
+
+  [[nodiscard]] Expected<void> selectLoad(MachineInstr &Instruction,
+                                          std::optional<LLT> Type) const {
+    if (!Type)
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "load result has no virtual-register type");
+    const auto Bytes = memoryAccessBytes(Instruction);
+    if (!Bytes)
+      return makeUnexpected(ErrorCode::Unsupported,
+                            "RISC-V load size does not match its result type");
+    if (Instruction.memOperands().front()->isAtomic()) {
+      if (!isSupportedIntegerType(*Type) || (*Bytes != 4 && *Bytes != 8))
+        return makeUnexpected(ErrorCode::Unsupported,
+                              "RISC-V atomic load requires a 32-bit or 64-bit GPR value");
+      return Instruction.setOpcode(
+          targetOpcode(*Bytes == 4 ? InstructionOpcode::LR_W : InstructionOpcode::LR_D));
+    }
+    auto AppendOffset = appendImmediate(Instruction, 0);
+    if (!AppendOffset)
+      return std::unexpected(AppendOffset.error());
+    if (isSupportedIntegerType(*Type)) {
+      const InstructionOpcode Selected = *Bytes == 1   ? InstructionOpcode::LB
+                                         : *Bytes == 2 ? InstructionOpcode::LH
+                                         : *Bytes == 4 ? InstructionOpcode::LW
+                                         : *Bytes == 8 ? InstructionOpcode::LD
+                                                       : InstructionOpcode::Invalid;
+      if (Selected == InstructionOpcode::Invalid)
+        return makeUnexpected(ErrorCode::Unsupported, "RISC-V integer load width is unsupported");
+      return Instruction.setOpcode(targetOpcode(Selected));
+    }
+    if (Type->isFloat() && Type->scalarBitWidth() == 32)
+      return Instruction.setOpcode(targetOpcode(InstructionOpcode::FLW));
+    if (Type->isFloat() && Type->scalarBitWidth() == 64)
+      return Instruction.setOpcode(targetOpcode(InstructionOpcode::FLD));
+    return makeUnexpected(ErrorCode::Unsupported, "RISC-V cannot select this generic load type");
+  }
+
+  [[nodiscard]] Expected<void> selectExtLoad(MachineInstr &Instruction, std::optional<LLT> Type,
+                                             bool Signed) const {
+    if (!Type || !isSupportedIntegerType(*Type))
+      return makeUnexpected(ErrorCode::Unsupported,
+                            "RISC-V extension load requires an integer result");
+    const auto Bytes = memoryAccessBytes(Instruction);
+    if (!Bytes)
+      return makeUnexpected(ErrorCode::Unsupported, "RISC-V extension load has an invalid width");
+    const InstructionOpcode Selected =
+        *Bytes == 1   ? (Signed ? InstructionOpcode::LB : InstructionOpcode::LBU)
+        : *Bytes == 2 ? (Signed ? InstructionOpcode::LH : InstructionOpcode::LHU)
+        : *Bytes == 4 ? (Signed ? InstructionOpcode::LW : InstructionOpcode::LWU)
+                      : InstructionOpcode::Invalid;
+    if (Selected == InstructionOpcode::Invalid)
+      return makeUnexpected(ErrorCode::Unsupported, "RISC-V extension load width is unsupported");
+    auto Immediate = appendImmediate(Instruction, 0);
+    if (!Immediate)
+      return std::unexpected(Immediate.error());
+    return Instruction.setOpcode(targetOpcode(Selected));
+  }
+
+  [[nodiscard]] Expected<void> selectFloatExtLoad(MachineInstr &Instruction,
+                                                  MachineFunction &Function,
+                                                  const TargetDescription &Target) const {
+    const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+    const Register *Destination =
+        DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+    if (Destination == nullptr)
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "RISC-V floating extension load has no destination register");
+    const Register DestinationRegister = *Destination;
+    auto F32 = LLT::makeFloat(FloatSemantics::IEEESingle);
+    if (!F32)
+      return std::unexpected(F32.error());
+    auto Temporary = createTemporary(Function, *F32, Target);
+    if (!Temporary)
+      return std::unexpected(Temporary.error());
+    auto SetDestination = Instruction.operandAt(0)->setRegister(*Temporary);
+    if (!SetDestination)
+      return std::unexpected(SetDestination.error());
+    auto Immediate = appendImmediate(Instruction, 0);
+    if (!Immediate)
+      return std::unexpected(Immediate.error());
+    auto SetOpcode = Instruction.setOpcode(targetOpcode(InstructionOpcode::FLW));
+    if (!SetOpcode)
+      return std::unexpected(SetOpcode.error());
+    MachineIRBuilder Builder(Function, Target);
+    auto Conversion = buildImmediateAfter(Builder, Instruction, InstructionOpcode::FCVT_D_S,
+                                          DestinationRegister, *Temporary, 0);
+    if (!Conversion)
+      return std::unexpected(Conversion.error());
+    return {};
+  }
+
+  [[nodiscard]] Expected<void> selectFloatTruncStore(MachineInstr &Instruction,
+                                                     MachineFunction &Function,
+                                                     const TargetDescription &Target) const {
+    const MachineOperand *ValueOperand = Instruction.operandAt(0);
+    const Register *Value = ValueOperand == nullptr ? nullptr : ValueOperand->getIf<Register>();
+    if (Value == nullptr)
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "RISC-V floating truncation store has no value register");
+    const Register ValueRegister = *Value;
+    auto F32 = LLT::makeFloat(FloatSemantics::IEEESingle);
+    if (!F32)
+      return std::unexpected(F32.error());
+    auto Temporary = createTemporary(Function, *F32, Target);
+    if (!Temporary)
+      return std::unexpected(Temporary.error());
+    MachineIRBuilder Builder(Function, Target);
+    auto Point = Builder.setInsertPointBefore(Instruction);
+    if (!Point)
+      return std::unexpected(Point.error());
+    auto Conversion = Builder.buildInstr(targetOpcode(InstructionOpcode::FCVT_S_D));
+    if (!Conversion)
+      return std::unexpected(Conversion.error());
+    auto Def = Conversion->addDef(*Temporary);
+    if (!Def)
+      return std::unexpected(Def.error());
+    auto Use = Conversion->addUse(ValueRegister);
+    if (!Use)
+      return std::unexpected(Use.error());
+    auto Rounding = Conversion->addImmediate(0);
+    if (!Rounding)
+      return std::unexpected(Rounding.error());
+    auto Built = Conversion->build();
+    if (!Built)
+      return std::unexpected(Built.error());
+    auto SetValue = Instruction.operandAt(0)->setRegister(*Temporary);
+    if (!SetValue)
+      return std::unexpected(SetValue.error());
+    auto Immediate = appendImmediate(Instruction, 0);
+    if (!Immediate)
+      return std::unexpected(Immediate.error());
+    return Instruction.setOpcode(targetOpcode(InstructionOpcode::FSW));
+  }
+
+  [[nodiscard]] Expected<void> selectStore(MachineInstr &Instruction,
+                                           MachineFunction &Function) const {
+    const MachineOperand *Value = Instruction.operandAt(0);
+    const std::optional<LLT> Type =
+        Value == nullptr ? std::nullopt : registerType(Function, *Value);
+    if (!Type)
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "store value has no virtual-register type");
+    const auto Bytes = memoryAccessBytes(Instruction);
+    if (!Bytes)
+      return makeUnexpected(ErrorCode::Unsupported,
+                            "RISC-V store size does not match its value type");
+    if (Instruction.memOperands().front()->isAtomic()) {
+      if (!isSupportedIntegerType(*Type) || (*Bytes != 4 && *Bytes != 8))
+        return makeUnexpected(ErrorCode::Unsupported,
+                              "RISC-V atomic store requires a 32-bit or 64-bit GPR value");
+      auto Zero = physicalRegister(PhysicalRegisterID::X0);
+      if (!Zero)
+        return std::unexpected(Zero.error());
+      auto Destination = insertPhysicalRegister(Instruction, 0, *Zero, RegisterOperandFlag::Define);
+      if (!Destination)
+        return std::unexpected(Destination.error());
+      return Instruction.setOpcode(
+          targetOpcode(*Bytes == 4 ? InstructionOpcode::AMOSWAP_W : InstructionOpcode::AMOSWAP_D));
+    }
+    auto AppendOffset = appendImmediate(Instruction, 0);
+    if (!AppendOffset)
+      return std::unexpected(AppendOffset.error());
+    if (isSupportedIntegerType(*Type)) {
+      const InstructionOpcode Selected = *Bytes == 1   ? InstructionOpcode::SB
+                                         : *Bytes == 2 ? InstructionOpcode::SH
+                                         : *Bytes == 4 ? InstructionOpcode::SW
+                                         : *Bytes == 8 ? InstructionOpcode::SD
+                                                       : InstructionOpcode::Invalid;
+      if (Selected == InstructionOpcode::Invalid)
+        return makeUnexpected(ErrorCode::Unsupported, "RISC-V integer store width is unsupported");
+      return Instruction.setOpcode(targetOpcode(Selected));
+    }
+    if (Type->isFloat() && Type->scalarBitWidth() == 32)
+      return Instruction.setOpcode(targetOpcode(InstructionOpcode::FSW));
+    if (Type->isFloat() && Type->scalarBitWidth() == 64)
+      return Instruction.setOpcode(targetOpcode(InstructionOpcode::FSD));
+    return makeUnexpected(ErrorCode::Unsupported, "RISC-V cannot select this generic store type");
+  }
+
+  [[nodiscard]] Expected<void> selectBranch(MachineInstr &Instruction) const {
+    auto Zero = physicalRegister(PhysicalRegisterID::X0);
+    if (!Zero)
+      return std::unexpected(Zero.error());
+    auto Inserted = insertPhysicalRegister(Instruction, 0, *Zero, RegisterOperandFlag::Define);
+    if (!Inserted)
+      return std::unexpected(Inserted.error());
+    return Instruction.setOpcode(targetOpcode(InstructionOpcode::JAL));
+  }
+
+  [[nodiscard]] Expected<void> selectConditionalBranch(MachineInstr &Instruction) const {
+    auto Zero = physicalRegister(PhysicalRegisterID::X0);
+    if (!Zero)
+      return std::unexpected(Zero.error());
+    auto Inserted = insertPhysicalRegister(Instruction, 1, *Zero, RegisterOperandFlag::Use);
+    if (!Inserted)
+      return std::unexpected(Inserted.error());
+    return Instruction.setOpcode(targetOpcode(InstructionOpcode::BNE));
+  }
+
+  [[nodiscard]] Expected<void> selectIntegerConstant(MachineInstr &Instruction,
+                                                     MachineFunction &Function,
+                                                     const TargetDescription &Target,
+                                                     std::optional<LLT> Type) const {
+    if (!Type || !isSupportedIntegerType(*Type)) {
+      return makeUnexpected(ErrorCode::Unsupported,
+                            "RISC-V constant selection requires an i1-i64 or pointer result");
+    }
+    const MachineOperand *ImmediateOperand = Instruction.operandAt(1);
+    if (ImmediateOperand == nullptr)
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "RISC-V generic constant has no immediate operand");
+    auto ImmediateValue = signedImmediate(*ImmediateOperand);
+    if (!ImmediateValue)
+      return std::unexpected(ImmediateValue.error());
+    const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+    const Register *DestinationValue =
+        DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+    if (DestinationValue == nullptr || !DestinationValue->isVirtual()) {
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "RISC-V generic constant must define a virtual register");
+    }
+    const Register Destination = *DestinationValue;
+    const std::int64_t Value = *ImmediateValue;
+    const std::uint32_t ImmediateTargetFlags = ImmediateOperand->targetFlags();
+    const bool IsWord = Type->isInteger() && Type->scalarBitWidth() <= 32;
+    const InstructionOpcode AddImmediate =
+        IsWord ? InstructionOpcode::ADDIW : InstructionOpcode::ADDI;
+    const InstructionOpcode ShiftLeftImmediate =
+        IsWord ? InstructionOpcode::SLLIW : InstructionOpcode::SLLI;
+    if (fitsSigned(Value, 12)) {
+      auto Zero = physicalRegister(PhysicalRegisterID::X0);
+      if (!Zero)
+        return std::unexpected(Zero.error());
+      auto Source = insertPhysicalRegister(Instruction, 1, *Zero, RegisterOperandFlag::Use);
+      if (!Source)
+        return std::unexpected(Source.error());
+      auto EraseImmediate = Instruction.eraseOperand(2);
+      if (!EraseImmediate)
+        return std::unexpected(EraseImmediate.error());
+      auto NormalizeImmediate =
+          Instruction.insertOperand(2, MachineOperand::immediate(Value, ImmediateTargetFlags));
+      if (!NormalizeImmediate)
+        return std::unexpected(NormalizeImmediate.error());
+      return Instruction.setOpcode(targetOpcode(AddImmediate));
+    }
+
+    // Recursively decompose the value into signed 12-bit chunks.  The
+    // quotient is computed from an unsigned bit pattern with explicit sign
+    // extension so both INT64_MIN and INT64_MAX remain well-defined.
+    std::uint64_t Remaining = static_cast<std::uint64_t>(Value);
+    std::vector<std::int64_t> Digits;
+    while (!fitsSigned(std::bit_cast<std::int64_t>(Remaining), 12)) {
+      const std::uint64_t LowBits = Remaining & 0xFFFU;
+      std::int64_t Digit = static_cast<std::int64_t>(LowBits);
+      if (Digit >= 0x800)
+        Digit -= 0x1000;
+      Digits.push_back(Digit);
+
+      const std::uint64_t Adjusted = Remaining - static_cast<std::uint64_t>(Digit);
+      std::uint64_t Quotient = Adjusted >> 12U;
+      if ((Remaining & (std::uint64_t{1} << 63U)) != 0)
+        Quotient |= ~std::uint64_t{0} << 52U;
+      Remaining = Quotient;
+    }
+    const std::int64_t Base = std::bit_cast<std::int64_t>(Remaining);
+
+    auto Temporary = createTemporary(Function, *Type, Target);
+    if (!Temporary)
+      return std::unexpected(Temporary.error());
+    auto *Parent = Instruction.parent();
+    if (Parent == nullptr)
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "RISC-V constant selection received a detached instruction");
+    auto SetOpcode = Instruction.setOpcode(targetOpcode(AddImmediate));
+    if (!SetOpcode)
+      return std::unexpected(SetOpcode.error());
+    auto SetDestination = Instruction.operandAt(0)->setRegister(*Temporary);
+    if (!SetDestination)
+      return std::unexpected(SetDestination.error());
+    auto EraseImmediate = Instruction.eraseOperand(1);
+    if (!EraseImmediate)
+      return std::unexpected(EraseImmediate.error());
+    auto Zero = physicalRegister(PhysicalRegisterID::X0);
+    if (!Zero)
+      return std::unexpected(Zero.error());
+    auto AddSource = insertPhysicalRegister(Instruction, 1, *Zero, RegisterOperandFlag::Use);
+    if (!AddSource)
+      return std::unexpected(AddSource.error());
+    auto AddBase = appendImmediate(Instruction, Base);
+    if (!AddBase)
+      return std::unexpected(AddBase.error());
+
+    MachineIRBuilder Builder(Function, Target);
+    auto InsertPoint = Builder.setInsertPointAfter(Instruction);
+    if (!InsertPoint)
+      return std::unexpected(InsertPoint.error());
+
+    Register Current = *Temporary;
+    for (std::size_t Reverse = Digits.size(); Reverse != 0; --Reverse) {
+      const std::size_t Index = Reverse - 1U;
+      auto ShiftResult = createTemporary(Function, *Type, Target);
+      if (!ShiftResult)
+        return std::unexpected(ShiftResult.error());
+      auto Shift = Builder.buildInstr(targetOpcode(ShiftLeftImmediate));
+      if (!Shift)
+        return std::unexpected(Shift.error());
+      auto ShiftDef = Shift->addDef(*ShiftResult);
+      if (!ShiftDef)
+        return std::unexpected(ShiftDef.error());
+      auto ShiftUse = Shift->addUse(Current);
+      if (!ShiftUse)
+        return std::unexpected(ShiftUse.error());
+      auto ShiftAmount = Shift->addImmediate(12);
+      if (!ShiftAmount)
+        return std::unexpected(ShiftAmount.error());
+      auto BuiltShift = Shift->build();
+      if (!BuiltShift)
+        return std::unexpected(BuiltShift.error());
+
+      const bool IsFinal = Index == 0;
+      Register AddDestination;
+      if (IsFinal) {
+        AddDestination = Destination;
+      } else {
+        auto AddTemporary = createTemporary(Function, *Type, Target);
+        if (!AddTemporary)
+          return std::unexpected(AddTemporary.error());
+        AddDestination = *AddTemporary;
+      }
+      auto Add = Builder.setInsertPointAfter(**BuiltShift);
+      if (!Add)
+        return std::unexpected(Add.error());
+      auto AddInstruction = Builder.buildInstr(targetOpcode(AddImmediate));
+      if (!AddInstruction)
+        return std::unexpected(AddInstruction.error());
+      auto AddDef = AddInstruction->addDef(AddDestination);
+      if (!AddDef)
+        return std::unexpected(AddDef.error());
+      auto AddUse = AddInstruction->addUse(*ShiftResult);
+      if (!AddUse)
+        return std::unexpected(AddUse.error());
+      auto AddImmediateResult = AddInstruction->addImmediate(Digits[Index]);
+      if (!AddImmediateResult)
+        return std::unexpected(AddImmediateResult.error());
+      auto BuiltAdd = AddInstruction->build();
+      if (!BuiltAdd)
+        return std::unexpected(BuiltAdd.error());
+      Current = AddDestination;
+    }
+    return {};
+  }
+
+  [[nodiscard]] Expected<void> selectFloatingConstant(MachineInstr &Instruction,
+                                                      MachineFunction &Function,
+                                                      const TargetDescription &Target,
+                                                      std::optional<LLT> Type) const {
+    if (!Type || !isRVFloatType(*Type))
+      return makeUnexpected(ErrorCode::Unsupported, "RISC-V floating constant requires f32 or f64");
+    const MachineOperand *Immediate = Instruction.operandAt(1);
+    const auto *Floating =
+        Immediate == nullptr ? nullptr : Immediate->getIf<FloatingImmediateOperand>();
+    const std::optional<FloatSemantics> Semantics = Type->floatSemantics();
+    if (Floating == nullptr || !Semantics || Floating->Bits.semantics() != *Semantics)
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "RISC-V floating constant has mismatched bit semantics");
+    const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+    const Register *Destination =
+        DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+    if (Destination == nullptr || !Destination->isVirtual())
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "RISC-V floating constant must define a virtual register");
+    const Register DestinationRegister = *Destination;
+
+    const std::uint32_t BitWidth = Type->scalarBitWidth();
+    const std::uint64_t Bits = Floating->Bits.words().empty() ? 0 : Floating->Bits.words().front();
+    auto Integer = IntegerImmediate::fromUnsigned(64, Bits);
+    if (!Integer)
+      return std::unexpected(Integer.error());
+    auto IntegerType = LLT::makeInteger(64);
+    if (!IntegerType)
+      return std::unexpected(IntegerType.error());
+    auto Temporary = createTemporary(Function, *IntegerType, Target);
+    if (!Temporary)
+      return std::unexpected(Temporary.error());
+
+    auto SetOpcode = Instruction.setOpcode(opcode(GenericOpcode::G_CONSTANT));
+    if (!SetOpcode)
+      return std::unexpected(SetOpcode.error());
+    auto SetDestination = Instruction.operandAt(0)->setRegister(*Temporary);
+    if (!SetDestination)
+      return std::unexpected(SetDestination.error());
+    auto EraseImmediate = Instruction.eraseOperand(1);
+    if (!EraseImmediate)
+      return std::unexpected(EraseImmediate.error());
+    auto AddInteger = Instruction.addOperand(MachineOperand::integerImmediate(*Integer));
+    if (!AddInteger)
+      return std::unexpected(AddInteger.error());
+    auto IntegerSelection = selectIntegerConstant(Instruction, Function, Target, *IntegerType);
+    if (!IntegerSelection)
+      return std::unexpected(IntegerSelection.error());
+
+    MachineInstr *IntegerDefinition = nullptr;
+    for (MachineOperand &DefOperand : Function.registerInfo().defOperands(*Temporary)) {
+      IntegerDefinition = DefOperand.parent();
+      break;
+    }
+    if (IntegerDefinition == nullptr)
+      return makeUnexpected(ErrorCode::InternalError,
+                            "RISC-V floating constant lost its integer materialization");
+    MachineIRBuilder Builder(Function, Target);
+    auto InsertPoint = Builder.setInsertPointAfter(*IntegerDefinition);
+    if (!InsertPoint)
+      return std::unexpected(InsertPoint.error());
+    const InstructionOpcode Move =
+        BitWidth == 32 ? InstructionOpcode::FMV_W_X : InstructionOpcode::FMV_D_X;
+    auto MoveInstruction = Builder.buildInstr(targetOpcode(Move));
+    if (!MoveInstruction)
+      return std::unexpected(MoveInstruction.error());
+    auto MoveDef = MoveInstruction->addDef(DestinationRegister);
+    if (!MoveDef)
+      return std::unexpected(MoveDef.error());
+    auto MoveUse = MoveInstruction->addUse(*Temporary);
+    if (!MoveUse)
+      return std::unexpected(MoveUse.error());
+    auto BuiltMove = MoveInstruction->build();
+    if (!BuiltMove)
+      return std::unexpected(BuiltMove.error());
+    return {};
+  }
+
+  [[nodiscard]] Expected<void> selectRoundingBinary(MachineInstr &Instruction,
+                                                    std::optional<LLT> Type,
+                                                    InstructionOpcode Single,
+                                                    InstructionOpcode Double) const {
+    auto RoundingMode = appendImmediate(Instruction, 0);
+    if (!RoundingMode)
+      return std::unexpected(RoundingMode.error());
+    if (Type && Type->isFloat() && Type->scalarBitWidth() == 32)
+      return Instruction.setOpcode(targetOpcode(Single));
+    if (Type && Type->isFloat() && Type->scalarBitWidth() == 64)
+      return Instruction.setOpcode(targetOpcode(Double));
+    return makeUnexpected(ErrorCode::Unsupported, "RISC-V cannot select this floating binary type");
+  }
+
+  [[nodiscard]] Expected<void> selectRoundingTernary(MachineInstr &Instruction,
+                                                     std::optional<LLT> Type,
+                                                     InstructionOpcode Single,
+                                                     InstructionOpcode Double) const {
+    return selectRoundingBinary(Instruction, Type, Single, Double);
+  }
+
+  [[nodiscard]] Expected<void> selectRoundingUnary(MachineInstr &Instruction,
+                                                   std::optional<LLT> Type,
+                                                   InstructionOpcode Single,
+                                                   InstructionOpcode Double) const {
+    return selectRoundingBinary(Instruction, Type, Single, Double);
+  }
+
+  [[nodiscard]] Expected<void> selectSignInjection(MachineInstr &Instruction,
+                                                   std::optional<LLT> Type,
+                                                   InstructionOpcode Single,
+                                                   InstructionOpcode Double) const {
+    const MachineOperand *Source = Instruction.operandAt(1);
+    const Register *SourceReg = Source == nullptr ? nullptr : Source->getIf<Register>();
+    if (SourceReg == nullptr)
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "RISC-V floating sign operation requires a register source");
+    auto Duplicate = insertVirtualUse(Instruction, 2, *SourceReg);
+    if (!Duplicate)
+      return std::unexpected(Duplicate.error());
+    if (Type && Type->isFloat() && Type->scalarBitWidth() == 32)
+      return Instruction.setOpcode(targetOpcode(Single));
+    if (Type && Type->isFloat() && Type->scalarBitWidth() == 64)
+      return Instruction.setOpcode(targetOpcode(Double));
+    return makeUnexpected(ErrorCode::Unsupported, "RISC-V cannot select this floating unary type");
+  }
+  const RISCVSubtarget &Subtarget;
+  const RISCVRegisterBankInfo &RegisterBanks;
+};
+
+} // namespace
+
+InstructionSelector *createRISCVInstructionSelector(const RISCVSubtarget &Subtarget,
+                                                    const RISCVRegisterBankInfo &RegisterBanks) {
+  return new RISCVInstructionSelector(Subtarget, RegisterBanks);
+}
+
+} // namespace rcc::riscv
diff --git a/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp b/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
new file mode 100644
index 0000000..e9d72fd
--- /dev/null
+++ b/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
@@ -0,0 +1,3818 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "RISCVLegalizerInfo.h"
+
+#include "RISCVCallLowering.h"
+#include "RISCVImmediateMaterialization.h"
+#include "rcc/CodeGen/GenericOpcodes.h"
+#include "rcc/CodeGen/LowLevelType.h"
+#include "rcc/CodeGen/MachineBasicBlock.h"
+#include "rcc/CodeGen/MachineFunction.h"
+#include "rcc/CodeGen/MachineIRBuilder.h"
+#include "rcc/CodeGen/MachineInstr.h"
+#include "rcc/CodeGen/MachineMemOperand.h"
+#include "rcc/CodeGen/MachineOperand.h"
+#include "rcc/CodeGen/MachineRegisterInfo.h"
+#include "rcc/CodeGen/TargetDescription.h"
+#include "rcc/Target/RISCV/RISCVSubtarget.h"
+#include "rcc/Target/RISCV/RISCVTarget.h"
+
+#include <algorithm>
+#include <array>
+#include <bit>
+#include <cstdint>
+#include <limits>
+#include <optional>
+#include <ranges>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace rcc::riscv {
+namespace {
+
+[[nodiscard]] Opcode targetOpcode(InstructionOpcode Value) noexcept {
+  return Opcode::fromValue(static_cast<std::uint32_t>(Value));
+}
+
+[[nodiscard]] std::optional<LLT> registerType(const MachineFunction &Function,
+                                              const MachineOperand &Operand) {
+  const Register *Reg = Operand.getIf<Register>();
+  if (Reg == nullptr || !Reg->isVirtual())
+    return std::nullopt;
+  auto Info = Function.registerInfo().virtualRegisterInfo(*Reg);
+  if (!Info)
+    return std::nullopt;
+  return Info->get().Type;
+}
+
+[[nodiscard]] std::optional<LLT> resultType(const MachineFunction &Function,
+                                            const MachineInstr &Instruction) {
+  const MachineOperand *Result = Instruction.operandAt(0);
+  return Result == nullptr ? std::nullopt : registerType(Function, *Result);
+}
+
+[[nodiscard]] std::optional<LLT> operandType(const MachineFunction &Function,
+                                             const MachineInstr &Instruction, std::size_t Index) {
+  const MachineOperand *Operand = Instruction.operandAt(Index);
+  return Operand == nullptr ? std::nullopt : registerType(Function, *Operand);
+}
+
+[[nodiscard]] bool isRVFloatType(LLT Type) noexcept {
+  if (!Type.isFloat())
+    return false;
+  const std::optional<FloatSemantics> Semantics = Type.floatSemantics();
+  return Semantics &&
+         (*Semantics == FloatSemantics::IEEESingle || *Semantics == FloatSemantics::IEEEDouble);
+}
+
+[[nodiscard]] bool isSupportedIntegerType(LLT Type) noexcept {
+  return ((Type.isInteger() || Type.isAnyScalar()) && Type.scalarBitWidth() != 0 &&
+          Type.scalarBitWidth() <= 64) ||
+         (Type.isPointer() && Type.scalarBitWidth() == 64);
+}
+
+[[nodiscard]] bool isSupportedIntegerScalar(LLT Type) noexcept {
+  return (Type.isInteger() || Type.isAnyScalar()) && Type.scalarBitWidth() != 0 &&
+         Type.scalarBitWidth() <= 64;
+}
+
+[[nodiscard]] bool isNativeIntegerScalar(LLT Type) noexcept {
+  return (Type.isInteger() || Type.isAnyScalar()) &&
+         (Type.scalarBitWidth() == 32 || Type.scalarBitWidth() == 64);
+}
+
+[[nodiscard]] bool isScalarGPRType(LLT Type) noexcept {
+  return ((Type.isInteger() || Type.isAnyScalar()) && Type.scalarBitWidth() != 0 &&
+          Type.scalarBitWidth() <= 64) ||
+         (Type.isPointer() && Type.scalarBitWidth() == 64);
+}
+
+[[nodiscard]] bool isI1Type(LLT Type) noexcept {
+  return (Type.isInteger() || Type.isAnyScalar()) && Type.scalarBitWidth() == 1;
+}
+
+[[nodiscard]] bool isScalarIntegerExtension(LLT Destination, LLT Source) noexcept {
+  return (Destination.isInteger() || Destination.isAnyScalar()) &&
+         (Source.isInteger() || Source.isAnyScalar()) && Destination.scalarBitWidth() != 0 &&
+         Source.scalarBitWidth() != 0 && Source.scalarBitWidth() < Destination.scalarBitWidth() &&
+         Destination.scalarBitWidth() <= 64;
+}
+
+[[nodiscard]] Expected<IntegerImmediate> extendIntegerImmediate(const IntegerImmediate &Value,
+                                                                std::uint32_t DestinationWidth,
+                                                                bool SignExtend) {
+  if (DestinationWidth == 0 || DestinationWidth > 64)
+    return makeUnexpected(ErrorCode::Unsupported,
+                          "RISC-V constant extension requires a destination no wider than 64 bits");
+
+  std::uint64_t Raw = Value.words().empty() ? 0 : Value.words().front();
+  const std::uint32_t SourceWidth = Value.bitWidth();
+  if (SignExtend && Value.isNegative() && SourceWidth < 64)
+    Raw |= ~std::uint64_t{0} << SourceWidth;
+  if (DestinationWidth < 64)
+    Raw &= (std::uint64_t{1} << DestinationWidth) - 1U;
+  return IntegerImmediate::create(DestinationWidth, std::span<const std::uint64_t>(&Raw, 1));
+}
+
+[[nodiscard]] Expected<void>
+foldConstantExtension(MachineInstr &Instruction, MachineFunction &Function, GenericOpcode Generic) {
+  const MachineOperand *SourceOperand = Instruction.operandAt(1);
+  const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+  const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>();
+  const Register *Destination =
+      DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+  if (Source == nullptr || Destination == nullptr || !Source->isVirtual() ||
+      !Destination->isVirtual()) {
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "RISC-V constant extension requires virtual-register operands");
+  }
+  const Register SourceRegister = *Source;
+  const Register DestinationRegister = *Destination;
+  const MachineOperand *SourceDefinition = Function.registerInfo().getUniqueVRegDef(SourceRegister);
+  const MachineInstr *SourceDef =
+      SourceDefinition == nullptr ? nullptr : SourceDefinition->parent();
+  if (SourceDef == nullptr || SourceDef->opcode() != opcode(GenericOpcode::G_CONSTANT))
+    return makeUnexpected(ErrorCode::Unsupported,
+                          "RISC-V constant extension requires a single constant definition");
+  const MachineOperand *ConstantOperand = SourceDef->operandAt(1);
+  const auto *Integer =
+      ConstantOperand == nullptr ? nullptr : ConstantOperand->getIf<IntegerImmediateOperand>();
+  if (Integer == nullptr)
+    return makeUnexpected(ErrorCode::Unsupported,
+                          "RISC-V constant extension requires an integer constant");
+  const auto DestinationInfo = Function.registerInfo().virtualRegisterInfo(*Destination);
+  if (!DestinationInfo || !DestinationInfo->get().Type)
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "RISC-V constant extension destination has no type");
+  const std::uint32_t DestinationWidth = DestinationInfo->get().Type->scalarBitWidth();
+  const bool SignExtend = Generic != GenericOpcode::G_ZEXT;
+  auto Extended = extendIntegerImmediate(Integer->Value, DestinationWidth, SignExtend);
+  if (!Extended)
+    return std::unexpected(Extended.error());
+
+  if (Function.registerInfo().hasOneUse(SourceRegister)) {
+    MachineInstr *Def = const_cast<MachineInstr *>(SourceDef);
+    if (auto Replaced = Def->operandAt(0)->setRegister(DestinationRegister); !Replaced)
+      return std::unexpected(Replaced.error());
+    auto Erased = Def->eraseOperand(1);
+    if (!Erased)
+      return std::unexpected(Erased.error());
+    auto Added = Def->addOperand(MachineOperand::integerImmediate(std::move(*Extended)));
+    if (!Added)
+      return std::unexpected(Added.error());
+    MachineBasicBlock *Parent = Instruction.parent();
+    if (Parent == nullptr)
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "RISC-V constant extension is detached during folding");
+    return Parent->removeInstruction(Instruction);
+  }
+
+  auto SetOpcode = Instruction.setOpcode(opcode(GenericOpcode::G_CONSTANT));
+  if (!SetOpcode)
+    return std::unexpected(SetOpcode.error());
+  while (Instruction.operandCount() > 1) {
+    auto Erased = Instruction.eraseOperand(1);
+    if (!Erased)
+      return std::unexpected(Erased.error());
+  }
+  auto Added = Instruction.addOperand(MachineOperand::integerImmediate(std::move(*Extended)));
+  if (!Added)
+    return std::unexpected(Added.error());
+
+  if (Function.registerInfo().useEmpty(SourceRegister)) {
+    MachineInstr *Def = const_cast<MachineInstr *>(SourceDef);
+    MachineBasicBlock *Parent = Def->parent();
+    if (Parent == nullptr)
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "RISC-V constant extension source definition is detached");
+    auto Removed = Parent->removeInstruction(*Def);
+    if (!Removed)
+      return std::unexpected(Removed.error());
+    // Keep the source register record as a tombstone-free historical vreg until
+    // the legalizer worklist transaction has completed.
+  }
+  return {};
+}
+
+struct WideConstantUnmergePlan final {
+  MachineInstr *Extension = nullptr;
+  MachineInstr *Unmerge = nullptr;
+  std::vector<Register> Outputs;
+  bool SignExtend = false;
+};
+
+[[nodiscard]] MachineInstr *singleRegisterUse(const MachineFunction &Function, Register Reg) {
+  MachineInstr *UseInstruction = nullptr;
+  auto &RegisterInfo = const_cast<MachineRegisterInfo &>(Function.registerInfo());
+  for (MachineOperand &Operand : RegisterInfo.useOperands(Reg)) {
+    if (Operand.parent() == nullptr)
+      continue;
+    if (UseInstruction != nullptr)
+      return nullptr;
+    UseInstruction = Operand.parent();
+  }
+  return UseInstruction;
+}
+
+[[nodiscard]] std::optional<WideConstantUnmergePlan>
+wideConstantUnmergePlan(const MachineInstr &Constant, const MachineFunction &Function) {
+  const MachineOperand *ResultOperand = Constant.operandAt(0);
+  const Register *Result = ResultOperand == nullptr ? nullptr : ResultOperand->getIf<Register>();
+  if (Result == nullptr || !Result->isVirtual())
+    return std::nullopt;
+
+  MachineInstr *FirstUse = singleRegisterUse(Function, *Result);
+  if (FirstUse == nullptr)
+    return std::nullopt;
+  MachineInstr *Extension = nullptr;
+  MachineInstr *Unmerge = FirstUse;
+  if (FirstUse->opcode() == opcode(GenericOpcode::G_ANYEXT) ||
+      FirstUse->opcode() == opcode(GenericOpcode::G_SEXT) ||
+      FirstUse->opcode() == opcode(GenericOpcode::G_ZEXT)) {
+    Extension = FirstUse;
+    const MachineOperand *ExtendedOperand = Extension->operandAt(0);
+    const Register *Extended =
+        ExtendedOperand == nullptr ? nullptr : ExtendedOperand->getIf<Register>();
+    if (Extended == nullptr || !Extended->isVirtual())
+      return std::nullopt;
+    Unmerge = singleRegisterUse(Function, *Extended);
+    if (Unmerge == nullptr)
+      return std::nullopt;
+  }
+  if (Unmerge->opcode() != opcode(GenericOpcode::G_UNMERGE_VALUES) || Unmerge->operandCount() < 3)
+    return std::nullopt;
+
+  WideConstantUnmergePlan Plan;
+  Plan.Extension = Extension;
+  Plan.Unmerge = Unmerge;
+  Plan.SignExtend = Extension != nullptr && Extension->opcode() != opcode(GenericOpcode::G_ZEXT);
+  for (std::size_t Index = 0; Index + 1U < Unmerge->operandCount(); ++Index) {
+    const MachineOperand *Operand = Unmerge->operandAt(Index);
+    const Register *Reg = Operand == nullptr ? nullptr : Operand->getIf<Register>();
+    if (Reg == nullptr || !Reg->isVirtual() || !Operand->isDef())
+      return std::nullopt;
+    const auto Info = Function.registerInfo().virtualRegisterInfo(*Reg);
+    if (!Info || !Info->get().Type || Info->get().Type->scalarBitWidth() == 0 ||
+        Info->get().Type->scalarBitWidth() > 64 ||
+        (!Info->get().Type->isInteger() && !Info->get().Type->isAnyScalar()))
+      return std::nullopt;
+    Plan.Outputs.push_back(*Reg);
+  }
+  const MachineOperand *SourceOperand = Unmerge->operandAt(Unmerge->operandCount() - 1U);
+  const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>();
+  const Register *ExpectedSource = Result;
+  if (Extension != nullptr) {
+    const MachineOperand *ExtendedOperand = Extension->operandAt(0);
+    ExpectedSource = ExtendedOperand == nullptr ? nullptr : ExtendedOperand->getIf<Register>();
+  }
+  if (Source == nullptr || ExpectedSource == nullptr || *Source != *ExpectedSource)
+    return std::nullopt;
+  return Plan;
+}
+
+[[nodiscard]] Expected<IntegerImmediate> wideConstantWord(const IntegerImmediate &Value,
+                                                          std::size_t WordIndex,
+                                                          std::uint32_t Width, bool SignExtend) {
+  std::uint64_t Raw = 0;
+  if (WordIndex < Value.words().size())
+    Raw = Value.words()[WordIndex];
+  else if (SignExtend && Value.isNegative())
+    Raw = std::numeric_limits<std::uint64_t>::max();
+  const std::size_t WordBitOffset = WordIndex * 64U;
+  if (SignExtend && Value.isNegative() && WordBitOffset < Value.bitWidth()) {
+    const std::uint32_t Remaining = Value.bitWidth() - static_cast<std::uint32_t>(WordBitOffset);
+    if (Remaining < 64)
+      Raw |= ~std::uint64_t{0} << Remaining;
+  }
+  if (Width < 64)
+    Raw &= (std::uint64_t{1} << Width) - 1U;
+  return IntegerImmediate::create(Width, std::span<const std::uint64_t>(&Raw, 1));
+}
+
+[[nodiscard]] Expected<void> foldWideConstant(MachineInstr &Instruction, MachineFunction &Function,
+                                              const TargetDescription &Target) {
+  const auto Plan = wideConstantUnmergePlan(Instruction, Function);
+  if (!Plan)
+    return makeUnexpected(ErrorCode::Unsupported,
+                          "RISC-V wide constant requires a G_UNMERGE_VALUES use");
+  const MachineOperand *ImmediateOperand = Instruction.operandAt(1);
+  const auto *Integer =
+      ImmediateOperand == nullptr ? nullptr : ImmediateOperand->getIf<IntegerImmediateOperand>();
+  if (Integer == nullptr)
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "RISC-V wide constant has no integer immediate");
+
+  std::vector<IntegerImmediate> Words;
+  Words.reserve(Plan->Outputs.size());
+  for (std::size_t Index = 0; Index < Plan->Outputs.size(); ++Index) {
+    const auto Info = Function.registerInfo().virtualRegisterInfo(Plan->Outputs[Index]);
+    if (!Info || !Info->get().Type)
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "RISC-V wide constant output has no virtual-register type");
+    auto Word = wideConstantWord(Integer->Value, Index, Info->get().Type->scalarBitWidth(),
+                                 Plan->SignExtend);
+    if (!Word)
+      return std::unexpected(Word.error());
+    Words.push_back(std::move(*Word));
+  }
+
+  std::vector<std::size_t> Representatives(Plan->Outputs.size());
+  for (std::size_t Index = 0; Index < Representatives.size(); ++Index) {
+    Representatives[Index] = Index;
+    for (std::size_t Previous = 0; Previous < Index; ++Previous) {
+      if (Words[Previous] == Words[Index]) {
+        Representatives[Index] = Previous;
+        break;
+      }
+    }
+  }
+
+  const Register FirstOutput = Plan->Outputs.front();
+  auto Retargeted = Instruction.operandAt(0)->setRegister(FirstOutput);
+  if (!Retargeted)
+    return std::unexpected(Retargeted.error());
+  auto SetOpcode = Instruction.setOpcode(opcode(GenericOpcode::G_CONSTANT));
+  if (!SetOpcode)
+    return std::unexpected(SetOpcode.error());
+  while (Instruction.operandCount() > 1) {
+    auto Erased = Instruction.eraseOperand(1);
+    if (!Erased)
+      return std::unexpected(Erased.error());
+  }
+  auto Added = Instruction.addOperand(MachineOperand::integerImmediate(std::move(Words.front())));
+  if (!Added)
+    return std::unexpected(Added.error());
+
+  MachineBasicBlock *Block = Instruction.parent();
+  if (Block == nullptr)
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "RISC-V wide constant is detached during legalization");
+  if (Plan->Unmerge != &Instruction) {
+    auto RemovedUnmerge = Plan->Unmerge->parent()->removeInstruction(*Plan->Unmerge);
+    if (!RemovedUnmerge)
+      return std::unexpected(RemovedUnmerge.error());
+  }
+  if (Plan->Extension != nullptr) {
+    auto RemovedExtension = Plan->Extension->parent()->removeInstruction(*Plan->Extension);
+    if (!RemovedExtension)
+      return std::unexpected(RemovedExtension.error());
+  }
+
+  for (std::size_t Index = 1; Index < Plan->Outputs.size(); ++Index) {
+    if (Representatives[Index] == Index)
+      continue;
+    auto Replaced = Function.registerInfo().replaceRegWith(Plan->Outputs[Index],
+                                                           Plan->Outputs[Representatives[Index]]);
+    if (!Replaced)
+      return std::unexpected(Replaced.error());
+  }
+
+  MachineInstr *Last = &Instruction;
+  for (std::size_t Index = 1; Index < Plan->Outputs.size(); ++Index) {
+    if (Representatives[Index] != Index)
+      continue;
+    MachineIRBuilder Builder(Function, Target);
+    auto Point = Builder.setInsertPointAfter(*Last);
+    if (!Point)
+      return std::unexpected(Point.error());
+    auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_CONSTANT));
+    if (!Pending)
+      return std::unexpected(Pending.error());
+    auto Def = Pending->addDef(Plan->Outputs[Index]);
+    if (!Def)
+      return std::unexpected(Def.error());
+    auto Value = Pending->addIntegerImmediate(std::move(Words[Index]));
+    if (!Value)
+      return std::unexpected(Value.error());
+    auto Built = Pending->build();
+    if (!Built)
+      return std::unexpected(Built.error());
+    Last = *Built;
+  }
+  return {};
+}
+
+[[nodiscard]] Expected<Register> buildIntegerConstantBefore(MachineIRBuilder &Builder,
+                                                            MachineInstr &Anchor,
+                                                            std::uint32_t Width,
+                                                            std::uint64_t Value);
+
+[[nodiscard]] Expected<Register> buildBinaryBefore(MachineIRBuilder &Builder, MachineInstr &Anchor,
+                                                   GenericOpcode Generic, LLT Type, Register Left,
+                                                   Register Right);
+
+[[nodiscard]] Expected<Register> buildUnaryBefore(MachineIRBuilder &Builder, MachineInstr &Anchor,
+                                                  GenericOpcode Generic, LLT Type, Register Source);
+
+[[nodiscard]] Expected<void> eraseInstruction(MachineInstr &Instruction);
+
+[[nodiscard]] Expected<void> rewriteUnary(MachineInstr &Instruction, GenericOpcode Generic,
+                                          Register Destination, Register Source);
+
+struct NarrowAnyExtBinaryPlan final {
+  MachineInstr *Binary = nullptr;
+  GenericOpcode Generic = GenericOpcode::Invalid;
+  std::array<Register, 2> WideInputs{};
+  std::vector<MachineInstr *> Artifacts;
+  std::vector<MachineInstr *> Constants;
+  std::uint32_t SourceWidth = 0;
+};
+
+[[nodiscard]] std::optional<NarrowAnyExtBinaryPlan>
+narrowAnyExtBinaryPlan(const MachineInstr &Extension, const MachineFunction &Function) {
+  const MachineOperand *DestinationOperand = Extension.operandAt(0);
+  const MachineOperand *SourceOperand = Extension.operandAt(1);
+  const Register *Destination =
+      DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+  const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>();
+  if (Destination == nullptr || Source == nullptr || !Destination->isVirtual() ||
+      !Source->isVirtual() || singleRegisterUse(Function, *Source) != &Extension)
+    return std::nullopt;
+  const auto DestinationInfo = Function.registerInfo().virtualRegisterInfo(*Destination);
+  const auto SourceInfo = Function.registerInfo().virtualRegisterInfo(*Source);
+  if (!DestinationInfo || !SourceInfo || !DestinationInfo->get().Type || !SourceInfo->get().Type)
+    return std::nullopt;
+  const LLT DestinationType = *DestinationInfo->get().Type;
+  const LLT SourceType = *SourceInfo->get().Type;
+  if (!isScalarIntegerExtension(DestinationType, SourceType) ||
+      DestinationType.scalarBitWidth() != 64)
+    return std::nullopt;
+
+  const MachineOperand *SourceDefinition = Function.registerInfo().getUniqueVRegDef(*Source);
+  MachineInstr *Binary = SourceDefinition == nullptr
+                             ? nullptr
+                             : const_cast<MachineInstr *>(SourceDefinition->parent());
+  if (Binary == nullptr || Binary->operandCount() != 3)
+    return std::nullopt;
+  const GenericOpcode Generic = static_cast<GenericOpcode>(Binary->opcode().value());
+  if (Generic != GenericOpcode::G_ADD && Generic != GenericOpcode::G_SUB &&
+      Generic != GenericOpcode::G_MUL && Generic != GenericOpcode::G_AND &&
+      Generic != GenericOpcode::G_OR && Generic != GenericOpcode::G_XOR &&
+      Generic != GenericOpcode::G_SHL)
+    return std::nullopt;
+
+  NarrowAnyExtBinaryPlan Plan;
+  Plan.Binary = Binary;
+  Plan.Generic = Generic;
+  Plan.SourceWidth = SourceType.scalarBitWidth();
+  const auto AddArtifact = [&Plan](MachineInstr *Artifact) {
+    if (std::ranges::find(Plan.Artifacts, Artifact) == Plan.Artifacts.end())
+      Plan.Artifacts.push_back(Artifact);
+  };
+  const auto TruncatedWideSource = [&](MachineInstr *Truncation,
+                                       MachineInstr *OnlyUse) -> std::optional<Register> {
+    if (Truncation == nullptr || Truncation->opcode() != opcode(GenericOpcode::G_TRUNC) ||
+        Truncation->operandCount() != 2)
+      return std::nullopt;
+    const MachineOperand *NarrowOperand = Truncation->operandAt(0);
+    const Register *Narrow = NarrowOperand == nullptr ? nullptr : NarrowOperand->getIf<Register>();
+    const MachineOperand *WideOperand = Truncation->operandAt(1);
+    const Register *Wide = WideOperand == nullptr ? nullptr : WideOperand->getIf<Register>();
+    const std::optional<LLT> WideType =
+        WideOperand == nullptr ? std::nullopt : registerType(Function, *WideOperand);
+    if (Narrow == nullptr || Wide == nullptr || !Wide->isVirtual() || !WideType ||
+        *WideType != DestinationType || singleRegisterUse(Function, *Narrow) != OnlyUse)
+      return std::nullopt;
+    return *Wide;
+  };
+  for (std::size_t Index = 0; Index != Plan.WideInputs.size(); ++Index) {
+    const MachineOperand *Operand = Binary->operandAt(Index + 1U);
+    const Register *Reg = Operand == nullptr ? nullptr : Operand->getIf<Register>();
+    if (Reg == nullptr || !Reg->isVirtual())
+      return std::nullopt;
+    const auto Info = Function.registerInfo().virtualRegisterInfo(*Reg);
+    if (!Info || !Info->get().Type)
+      return std::nullopt;
+    if (*Info->get().Type == DestinationType) {
+      Plan.WideInputs[Index] = *Reg;
+      continue;
+    }
+    if (*Info->get().Type != SourceType)
+      return std::nullopt;
+
+    const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(*Reg);
+    MachineInstr *Def =
+        Definition == nullptr ? nullptr : const_cast<MachineInstr *>(Definition->parent());
+    if (auto Wide = TruncatedWideSource(Def, Binary)) {
+      Plan.WideInputs[Index] = *Wide;
+      AddArtifact(Def);
+      continue;
+    }
+    if (Def != nullptr &&
+        (Def->opcode() == opcode(GenericOpcode::COPY) ||
+         Def->opcode() == opcode(GenericOpcode::G_ANYEXT)) &&
+        Def->operandCount() == 2 && singleRegisterUse(Function, *Reg) == Binary) {
+      const MachineOperand *NarrowOperand = Def->operandAt(1);
+      const Register *Narrow =
+          NarrowOperand == nullptr ? nullptr : NarrowOperand->getIf<Register>();
+      const MachineOperand *NarrowDefinition =
+          Narrow == nullptr ? nullptr : Function.registerInfo().getUniqueVRegDef(*Narrow);
+      MachineInstr *Truncation = NarrowDefinition == nullptr
+                                     ? nullptr
+                                     : const_cast<MachineInstr *>(NarrowDefinition->parent());
+      if (auto Wide = TruncatedWideSource(Truncation, Def)) {
+        Plan.WideInputs[Index] = *Wide;
+        AddArtifact(Def);
+        AddArtifact(Truncation);
+        continue;
+      }
+    }
+    if (Def != nullptr && Def->opcode() == opcode(GenericOpcode::G_CONSTANT) &&
+        singleRegisterUse(Function, *Reg) == Binary) {
+      Plan.WideInputs[Index] = *Reg;
+      Plan.Constants.push_back(Def);
+      continue;
+    }
+    return std::nullopt;
+  }
+  return Plan;
+}
+
+[[nodiscard]] Expected<void> widenConstantToRV64(MachineInstr &Instruction,
+                                                 MachineFunction &Function) {
+  const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+  const Register *Destination =
+      DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+  const MachineOperand *ImmediateOperand = Instruction.operandAt(1);
+  const auto *Immediate =
+      ImmediateOperand == nullptr ? nullptr : ImmediateOperand->getIf<IntegerImmediateOperand>();
+  if (Destination == nullptr || !Destination->isVirtual() || Immediate == nullptr)
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "RISC-V narrow constant legalization has invalid operands");
+  auto Type = LLT::makeAnyScalar(64);
+  if (!Type)
+    return std::unexpected(Type.error());
+  auto SetType = Function.registerInfo().setType(*Destination, *Type);
+  if (!SetType)
+    return std::unexpected(SetType.error());
+  auto Extended = extendIntegerImmediate(Immediate->Value, 64, true);
+  if (!Extended)
+    return std::unexpected(Extended.error());
+  auto Erased = Instruction.eraseOperand(1);
+  if (!Erased)
+    return std::unexpected(Erased.error());
+  auto Added = Instruction.addOperand(MachineOperand::integerImmediate(std::move(*Extended)));
+  if (!Added)
+    return std::unexpected(Added.error());
+  return {};
+}
+
+[[nodiscard]] Expected<Register> widenAtomicCompareExchangeSource(MachineInstr &Instruction,
+                                                                  MachineFunction &Function,
+                                                                  MachineIRBuilder &Builder,
+                                                                  Register Source, LLT WideType) {
+  const auto SourceInfo = Function.registerInfo().virtualRegisterInfo(Source);
+  if (!SourceInfo || !SourceInfo->get().Type)
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "RISC-V compare-exchange source has no scalar type");
+  if (SourceInfo->get().Type->scalarBitWidth() == WideType.scalarBitWidth())
+    return Source;
+
+  const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(Source);
+  MachineInstr *Def =
+      Definition == nullptr ? nullptr : const_cast<MachineInstr *>(Definition->parent());
+  if (Def != nullptr && Def->opcode() == opcode(GenericOpcode::G_CONSTANT) &&
+      singleRegisterUse(Function, Source) == &Instruction) {
+    auto Widened = widenConstantToRV64(*Def, Function);
+    if (!Widened)
+      return std::unexpected(Widened.error());
+    return Source;
+  }
+  return buildUnaryBefore(Builder, Instruction, GenericOpcode::G_ANYEXT, WideType, Source);
+}
+
+[[nodiscard]] Expected<void> lowerAtomicCompareExchange(MachineInstr &Instruction,
+                                                        MachineFunction &Function,
+                                                        const TargetDescription &Target) {
+  const std::optional<LLT> OriginalType = resultType(Function, Instruction);
+  const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+  const MachineOperand *CompareOperand = Instruction.operandAt(2);
+  const MachineOperand *NewValueOperand = Instruction.operandAt(3);
+  const Register *Destination =
+      DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+  const Register *Compare = CompareOperand == nullptr ? nullptr : CompareOperand->getIf<Register>();
+  const Register *NewValue =
+      NewValueOperand == nullptr ? nullptr : NewValueOperand->getIf<Register>();
+  if (!OriginalType || Destination == nullptr || Compare == nullptr || NewValue == nullptr ||
+      OriginalType->scalarBitWidth() >= 64)
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "RISC-V narrow compare-exchange has invalid operands");
+  const Register OriginalDestination = *Destination;
+
+  auto WideType = LLT::makeAnyScalar(64);
+  if (!WideType)
+    return std::unexpected(WideType.error());
+  MachineIRBuilder Builder(Function, Target);
+  auto WideCompare =
+      widenAtomicCompareExchangeSource(Instruction, Function, Builder, *Compare, *WideType);
+  if (!WideCompare)
+    return std::unexpected(WideCompare.error());
+  auto WideNewValue =
+      widenAtomicCompareExchangeSource(Instruction, Function, Builder, *NewValue, *WideType);
+  if (!WideNewValue)
+    return std::unexpected(WideNewValue.error());
+
+  Register WideDestination;
+  MachineInstr *FoldedExtension = singleRegisterUse(Function, OriginalDestination);
+  if (FoldedExtension != nullptr && FoldedExtension->opcode() == opcode(GenericOpcode::G_ANYEXT) &&
+      operandType(Function, *FoldedExtension, 0)
+          .transform([](LLT Type) { return Type.scalarBitWidth() == 64; })
+          .value_or(false)) {
+    const MachineOperand *WideDestinationOperand = FoldedExtension->operandAt(0);
+    const Register *FoldedDestination =
+        WideDestinationOperand == nullptr ? nullptr : WideDestinationOperand->getIf<Register>();
+    if (FoldedDestination == nullptr)
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "RISC-V compare-exchange extension has no destination");
+    WideDestination = *FoldedDestination;
+  } else {
+    auto Created = Function.registerInfo().createVirtualRegister(*WideType);
+    if (!Created)
+      return std::unexpected(Created.error());
+    WideDestination = *Created;
+    FoldedExtension = nullptr;
+  }
+
+  if (auto Replaced = Instruction.operandAt(0)->setRegister(WideDestination); !Replaced)
+    return std::unexpected(Replaced.error());
+  if (auto Replaced = Instruction.operandAt(2)->setRegister(*WideCompare); !Replaced)
+    return std::unexpected(Replaced.error());
+  if (auto Replaced = Instruction.operandAt(3)->setRegister(*WideNewValue); !Replaced)
+    return std::unexpected(Replaced.error());
+
+  if (FoldedExtension != nullptr)
+    return eraseInstruction(*FoldedExtension);
+
+  auto Point = Builder.setInsertPointAfter(Instruction);
+  if (!Point)
+    return std::unexpected(Point.error());
+  auto Truncation = Builder.buildInstr(opcode(GenericOpcode::G_TRUNC));
+  if (!Truncation)
+    return std::unexpected(Truncation.error());
+  if (auto Added = Truncation->addDef(OriginalDestination); !Added)
+    return std::unexpected(Added.error());
+  if (auto Added = Truncation->addUse(WideDestination); !Added)
+    return std::unexpected(Added.error());
+  auto Built = Truncation->build();
+  if (!Built)
+    return std::unexpected(Built.error());
+  return {};
+}
+
+[[nodiscard]] Expected<void>
+lowerAtomicCompareExchangeWithSuccess(MachineInstr &Instruction, MachineFunction &Function,
+                                      const TargetDescription &Target) {
+  const MachineOperand *OldValueOperand = Instruction.operandAt(0);
+  const MachineOperand *SuccessOperand = Instruction.operandAt(1);
+  const MachineOperand *AddressOperand = Instruction.operandAt(2);
+  const MachineOperand *CompareOperand = Instruction.operandAt(3);
+  const MachineOperand *NewValueOperand = Instruction.operandAt(4);
+  const Register *OldValue =
+      OldValueOperand == nullptr ? nullptr : OldValueOperand->getIf<Register>();
+  const Register *Success = SuccessOperand == nullptr ? nullptr : SuccessOperand->getIf<Register>();
+  const Register *Address = AddressOperand == nullptr ? nullptr : AddressOperand->getIf<Register>();
+  const Register *Compare = CompareOperand == nullptr ? nullptr : CompareOperand->getIf<Register>();
+  const Register *NewValue =
+      NewValueOperand == nullptr ? nullptr : NewValueOperand->getIf<Register>();
+  const std::optional<LLT> ValueType = resultType(Function, Instruction);
+  if (OldValue == nullptr || Success == nullptr || Address == nullptr || Compare == nullptr ||
+      NewValue == nullptr || !ValueType || Instruction.memOperands().size() != 1)
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "RISC-V compare-exchange-with-success has invalid operands");
+
+  auto AtomicResult = Function.registerInfo().createVirtualRegister(*ValueType);
+  if (!AtomicResult)
+    return std::unexpected(AtomicResult.error());
+  MachineIRBuilder Builder(Function, Target);
+  auto Point = Builder.setInsertPointBefore(Instruction);
+  if (!Point)
+    return std::unexpected(Point.error());
+  auto Atomic = Builder.buildInstr(opcode(GenericOpcode::G_ATOMIC_CMPXCHG));
+  if (!Atomic)
+    return std::unexpected(Atomic.error());
+  if (auto Added = Atomic->addDef(*AtomicResult); !Added)
+    return std::unexpected(Added.error());
+  if (auto Added = Atomic->addUse(*Address); !Added)
+    return std::unexpected(Added.error());
+  if (auto Added = Atomic->addUse(*Compare); !Added)
+    return std::unexpected(Added.error());
+  if (auto Added = Atomic->addUse(*NewValue); !Added)
+    return std::unexpected(Added.error());
+  if (auto Added = Atomic->addMemOperand(*Instruction.memOperands().front()); !Added)
+    return std::unexpected(Added.error());
+  auto BuiltAtomic = Atomic->build();
+  if (!BuiltAtomic)
+    return std::unexpected(BuiltAtomic.error());
+
+  Point = Builder.setInsertPointBefore(Instruction);
+  if (!Point)
+    return std::unexpected(Point.error());
+  auto CompareResult = Builder.buildInstr(opcode(GenericOpcode::G_ICMP));
+  if (!CompareResult)
+    return std::unexpected(CompareResult.error());
+  if (auto Added = CompareResult->addDef(*Success); !Added)
+    return std::unexpected(Added.error());
+  if (auto Added = CompareResult->addIntegerPredicate(IntegerPredicate::Equal); !Added)
+    return std::unexpected(Added.error());
+  if (auto Added = CompareResult->addUse(*AtomicResult); !Added)
+    return std::unexpected(Added.error());
+  if (auto Added = CompareResult->addUse(*Compare); !Added)
+    return std::unexpected(Added.error());
+  auto BuiltCompare = CompareResult->build();
+  if (!BuiltCompare)
+    return std::unexpected(BuiltCompare.error());
+
+  while (!Instruction.memOperands().empty()) {
+    auto Erased = Instruction.eraseMemOperand(0);
+    if (!Erased)
+      return std::unexpected(Erased.error());
+  }
+  return rewriteUnary(Instruction, GenericOpcode::COPY, *OldValue, *AtomicResult);
+}
+
+[[nodiscard]] Expected<Register> widenIntegerCompareSource(MachineInstr &Instruction,
+                                                           MachineFunction &Function,
+                                                           MachineIRBuilder &Builder,
+                                                           Register Source, LLT WideType) {
+  const auto SourceInfo = Function.registerInfo().virtualRegisterInfo(Source);
+  if (!SourceInfo || !SourceInfo->get().Type)
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "RISC-V integer compare source has no scalar type");
+  const LLT SourceType = *SourceInfo->get().Type;
+  if (SourceType.scalarBitWidth() == WideType.scalarBitWidth())
+    return Source;
+
+  const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(Source);
+  MachineInstr *Def =
+      Definition == nullptr ? nullptr : const_cast<MachineInstr *>(Definition->parent());
+  std::optional<Register> ExistingExtension;
+  bool AmbiguousExtension = false;
+  auto &RegisterInfo = Function.registerInfo();
+  for (MachineOperand &Use : RegisterInfo.useOperands(Source)) {
+    MachineInstr *User = Use.parent();
+    if (User == nullptr || User == &Instruction || User->parent() == nullptr ||
+        (User->opcode() != opcode(GenericOpcode::G_ANYEXT) &&
+         User->opcode() != opcode(GenericOpcode::G_SEXT))) {
+      continue;
+    }
+    const MachineOperand *ExtendedOperand = User->operandAt(0);
+    const Register *Extended =
+        ExtendedOperand == nullptr ? nullptr : ExtendedOperand->getIf<Register>();
+    const std::optional<LLT> ExtendedType =
+        ExtendedOperand == nullptr ? std::nullopt : registerType(Function, *ExtendedOperand);
+    if (Extended == nullptr || !ExtendedType || *ExtendedType != WideType ||
+        (Def == nullptr || (Def->opcode() != opcode(GenericOpcode::G_CONSTANT) &&
+                            User->opcode() != opcode(GenericOpcode::G_SEXT)))) {
+      continue;
+    }
+    if (ExistingExtension) {
+      AmbiguousExtension = true;
+      break;
+    }
+    ExistingExtension = *Extended;
+  }
+  if (ExistingExtension && !AmbiguousExtension)
+    return *ExistingExtension;
+  if (Def != nullptr && Def->opcode() == opcode(GenericOpcode::G_CONSTANT) &&
+      singleRegisterUse(Function, Source) == &Instruction) {
+    auto Widened = widenConstantToRV64(*Def, Function);
+    if (!Widened)
+      return std::unexpected(Widened.error());
+    return Source;
+  }
+
+  if (Def != nullptr && Def->opcode() == opcode(GenericOpcode::G_TRUNC) &&
+      Def->operandCount() == 2) {
+    const MachineOperand *WideOperand = Def->operandAt(1);
+    const Register *WideSource = WideOperand == nullptr ? nullptr : WideOperand->getIf<Register>();
+    const std::optional<LLT> ExistingWideType =
+        WideOperand == nullptr ? std::nullopt : registerType(Function, *WideOperand);
+    if (WideSource != nullptr && ExistingWideType && *ExistingWideType == WideType) {
+      auto Result = Function.registerInfo().createVirtualRegister(WideType);
+      if (!Result)
+        return std::unexpected(Result.error());
+      auto Point = Builder.setInsertPointBefore(Instruction);
+      if (!Point)
+        return std::unexpected(Point.error());
+      auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_SEXT_INREG));
+      if (!Pending)
+        return std::unexpected(Pending.error());
+      if (auto Added = Pending->addDef(*Result); !Added)
+        return std::unexpected(Added.error());
+      if (auto Added = Pending->addUse(*WideSource); !Added)
+        return std::unexpected(Added.error());
+      if (auto Added =
+              Pending->addImmediate(static_cast<std::int64_t>(SourceType.scalarBitWidth()));
+          !Added) {
+        return std::unexpected(Added.error());
+      }
+      auto Built = Pending->build();
+      if (!Built)
+        return std::unexpected(Built.error());
+      return *Result;
+    }
+  }
+  return buildUnaryBefore(Builder, Instruction, GenericOpcode::G_SEXT, WideType, Source);
+}
+
+[[nodiscard]] Expected<void> lowerIntegerCompare(MachineInstr &Instruction,
+                                                 MachineFunction &Function,
+                                                 const TargetDescription &Target) {
+  const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+  const MachineOperand *LeftOperand = Instruction.operandAt(2);
+  const MachineOperand *RightOperand = Instruction.operandAt(3);
+  const Register *Destination =
+      DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+  const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf<Register>();
+  const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf<Register>();
+  if (Destination == nullptr || Left == nullptr || Right == nullptr)
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "RISC-V integer compare has invalid register operands");
+  const Register OriginalDestination = *Destination;
+  const Register OriginalLeft = *Left;
+  const Register OriginalRight = *Right;
+
+  auto WideType = LLT::makeAnyScalar(64);
+  if (!WideType)
+    return std::unexpected(WideType.error());
+  MachineIRBuilder Builder(Function, Target);
+  auto WideLeft =
+      widenIntegerCompareSource(Instruction, Function, Builder, OriginalLeft, *WideType);
+  if (!WideLeft)
+    return std::unexpected(WideLeft.error());
+  auto WideRight =
+      widenIntegerCompareSource(Instruction, Function, Builder, OriginalRight, *WideType);
+  if (!WideRight)
+    return std::unexpected(WideRight.error());
+  if (auto Replaced = Instruction.operandAt(2)->setRegister(*WideLeft); !Replaced)
+    return std::unexpected(Replaced.error());
+  if (auto Replaced = Instruction.operandAt(3)->setRegister(*WideRight); !Replaced)
+    return std::unexpected(Replaced.error());
+
+  const auto DestinationInfo = Function.registerInfo().virtualRegisterInfo(OriginalDestination);
+  if (!DestinationInfo || !DestinationInfo->get().Type)
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "RISC-V integer compare destination has no scalar type");
+  if (DestinationInfo->get().Type->scalarBitWidth() == 64)
+    return {};
+
+  MachineInstr *FoldedExtension = singleRegisterUse(Function, OriginalDestination);
+  if (FoldedExtension != nullptr &&
+      (FoldedExtension->opcode() == opcode(GenericOpcode::G_ANYEXT) ||
+       FoldedExtension->opcode() == opcode(GenericOpcode::G_SEXT) ||
+       FoldedExtension->opcode() == opcode(GenericOpcode::G_ZEXT)) &&
+      operandType(Function, *FoldedExtension, 0)
+          .transform([](LLT Type) { return Type.scalarBitWidth() == 64; })
+          .value_or(false)) {
+    const MachineOperand *ExtendedOperand = FoldedExtension->operandAt(0);
+    const Register *Extended =
+        ExtendedOperand == nullptr ? nullptr : ExtendedOperand->getIf<Register>();
+    if (Extended == nullptr)
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "RISC-V integer compare extension has no destination");
+    const Register WideDestination = *Extended;
+    if (auto Replaced = Instruction.operandAt(0)->setRegister(WideDestination); !Replaced)
+      return std::unexpected(Replaced.error());
+    return eraseInstruction(*FoldedExtension);
+  }
+
+  auto WideDestination = Function.registerInfo().createVirtualRegister(*WideType);
+  if (!WideDestination)
+    return std::unexpected(WideDestination.error());
+  if (auto Replaced = Instruction.operandAt(0)->setRegister(*WideDestination); !Replaced)
+    return std::unexpected(Replaced.error());
+  if (Function.registerInfo().useEmpty(OriginalDestination))
+    return {};
+  auto Point = Builder.setInsertPointAfter(Instruction);
+  if (!Point)
+    return std::unexpected(Point.error());
+  auto Truncation = Builder.buildInstr(opcode(GenericOpcode::G_TRUNC));
+  if (!Truncation)
+    return std::unexpected(Truncation.error());
+  if (auto Added = Truncation->addDef(OriginalDestination); !Added)
+    return std::unexpected(Added.error());
+  if (auto Added = Truncation->addUse(*WideDestination); !Added)
+    return std::unexpected(Added.error());
+  auto Built = Truncation->build();
+  if (!Built)
+    return std::unexpected(Built.error());
+  return {};
+}
+
+[[nodiscard]] Expected<void> foldNarrowAnyExtBinary(MachineInstr &Extension,
+                                                    MachineFunction &Function,
+                                                    const TargetDescription &Target) {
+  const auto Plan = narrowAnyExtBinaryPlan(Extension, Function);
+  if (!Plan)
+    return makeUnexpected(ErrorCode::Unsupported,
+                          "RISC-V narrow any-extension does not have a legal binary chain");
+  for (MachineInstr *Constant : Plan->Constants) {
+    auto Widened = widenConstantToRV64(*Constant, Function);
+    if (!Widened)
+      return std::unexpected(Widened.error());
+  }
+  for (std::size_t Index = 0; Index != Plan->WideInputs.size(); ++Index) {
+    auto Replaced = Plan->Binary->operandAt(Index + 1U)->setRegister(Plan->WideInputs[Index]);
+    if (!Replaced)
+      return std::unexpected(Replaced.error());
+  }
+  for (MachineInstr *Artifact : Plan->Artifacts) {
+    if (Artifact->parent() == nullptr)
+      continue;
+    auto Removed = Artifact->parent()->removeInstruction(*Artifact);
+    if (!Removed)
+      return std::unexpected(Removed.error());
+  }
+
+  if (Plan->Generic == GenericOpcode::G_SHL) {
+    Register Amount = Plan->WideInputs[1];
+    if (Plan->SourceWidth < 32 || Plan->SourceWidth > 32) {
+      const std::uint64_t Mask = Plan->SourceWidth == 64
+                                     ? std::numeric_limits<std::uint64_t>::max()
+                                     : (std::uint64_t{1} << Plan->SourceWidth) - 1U;
+      auto WideType = LLT::makeAnyScalar(64);
+      if (!WideType)
+        return std::unexpected(WideType.error());
+      MachineIRBuilder Builder(Function, Target);
+      auto MaskValue = buildIntegerConstantBefore(Builder, Extension, 64, Mask);
+      if (!MaskValue)
+        return std::unexpected(MaskValue.error());
+      auto MaskedAmount = buildBinaryBefore(Builder, Extension, GenericOpcode::G_AND, *WideType,
+                                            Amount, *MaskValue);
+      if (!MaskedAmount)
+        return std::unexpected(MaskedAmount.error());
+      Amount = *MaskedAmount;
+    }
+    auto SetOpcode =
+        Extension.setOpcode(Plan->SourceWidth == 32 ? targetOpcode(InstructionOpcode::G_SLLW)
+                                                    : opcode(GenericOpcode::G_SHL));
+    if (!SetOpcode)
+      return std::unexpected(SetOpcode.error());
+    while (Extension.operandCount() > 1) {
+      auto Erased = Extension.eraseOperand(1);
+      if (!Erased)
+        return std::unexpected(Erased.error());
+    }
+    for (Register Input : {Plan->WideInputs[0], Amount}) {
+      auto Operand =
+          MachineOperand::registerOperand(Input, RegisterOperandFlags{RegisterOperandFlag::Use});
+      if (!Operand)
+        return std::unexpected(Operand.error());
+      auto Added = Extension.addOperand(std::move(*Operand));
+      if (!Added)
+        return std::unexpected(Added.error());
+    }
+    MachineBasicBlock *BinaryBlock = Plan->Binary->parent();
+    if (BinaryBlock == nullptr)
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "RISC-V narrow shift is detached during legalization");
+    return BinaryBlock->removeInstruction(*Plan->Binary);
+  }
+
+  if (Plan->SourceWidth == 32 &&
+      (Plan->Generic == GenericOpcode::G_ADD || Plan->Generic == GenericOpcode::G_SUB)) {
+    auto WideType = LLT::makeAnyScalar(64);
+    if (!WideType)
+      return std::unexpected(WideType.error());
+    auto Temporary = Function.registerInfo().createVirtualRegister(*WideType);
+    if (!Temporary)
+      return std::unexpected(Temporary.error());
+    auto Retargeted = Plan->Binary->operandAt(0)->setRegister(*Temporary);
+    if (!Retargeted)
+      return std::unexpected(Retargeted.error());
+    auto SetOpcode = Extension.setOpcode(opcode(GenericOpcode::G_SEXT_INREG));
+    if (!SetOpcode)
+      return std::unexpected(SetOpcode.error());
+    auto ReplacedSource = Extension.operandAt(1)->setRegister(*Temporary);
+    if (!ReplacedSource)
+      return std::unexpected(ReplacedSource.error());
+    auto Width = Extension.addOperand(MachineOperand::immediate(32));
+    if (!Width)
+      return std::unexpected(Width.error());
+    return {};
+  }
+
+  auto SetOpcode = Extension.setOpcode(opcode(Plan->Generic));
+  if (!SetOpcode)
+    return std::unexpected(SetOpcode.error());
+  while (Extension.operandCount() > 1) {
+    auto Erased = Extension.eraseOperand(1);
+    if (!Erased)
+      return std::unexpected(Erased.error());
+  }
+  for (Register Input : Plan->WideInputs) {
+    auto Operand =
+        MachineOperand::registerOperand(Input, RegisterOperandFlags{RegisterOperandFlag::Use});
+    if (!Operand)
+      return std::unexpected(Operand.error());
+    auto Added = Extension.addOperand(std::move(*Operand));
+    if (!Added)
+      return std::unexpected(Added.error());
+  }
+  MachineBasicBlock *BinaryBlock = Plan->Binary->parent();
+  if (BinaryBlock == nullptr)
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "RISC-V narrow binary instruction is detached during legalization");
+  auto Removed = BinaryBlock->removeInstruction(*Plan->Binary);
+  if (!Removed)
+    return std::unexpected(Removed.error());
+  (void)Target;
+  return {};
+}
+
+[[nodiscard]] Expected<void> lowerConstantToPool(MachineInstr &Instruction,
+                                                 MachineFunction &Function,
+                                                 const TargetDescription &Target) {
+  const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+  const Register *Destination =
+      DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+  const MachineOperand *ImmediateOperand = Instruction.operandAt(1);
+  const auto *Immediate =
+      ImmediateOperand == nullptr ? nullptr : ImmediateOperand->getIf<IntegerImmediateOperand>();
+  if (Destination == nullptr || !Destination->isVirtual() || Immediate == nullptr)
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "RISC-V constant-pool legalization has invalid operands");
+  const auto DestinationInfo = Function.registerInfo().virtualRegisterInfo(*Destination);
+  if (!DestinationInfo || !DestinationInfo->get().Type ||
+      DestinationInfo->get().Type->scalarBitWidth() != 64)
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "RISC-V constant-pool legalization requires a 64-bit destination");
+  const LLT DestinationType = *DestinationInfo->get().Type;
+  auto Value = GenericConstantData::integer(Immediate->Value);
+  if (!Value)
+    return std::unexpected(Value.error());
+  auto Align = Alignment::create(8);
+  if (!Align)
+    return std::unexpected(Align.error());
+  MachineConstantPoolEntryDesc Description;
+  Description.Value = std::move(*Value);
+  Description.Size = 8;
+  Description.Align = *Align;
+  auto PoolIndex = Function.constantPool().addEntry(std::move(Description));
+  if (!PoolIndex)
+    return std::unexpected(PoolIndex.error());
+
+  auto PointerType = LLT::makePointer(0, 64);
+  if (!PointerType)
+    return std::unexpected(PointerType.error());
+  auto Address = Function.registerInfo().createVirtualRegister(*PointerType);
+  if (!Address)
+    return std::unexpected(Address.error());
+  MachineIRBuilder Builder(Function, Target);
+  auto Point = Builder.setInsertPointBefore(Instruction);
+  if (!Point)
+    return std::unexpected(Point.error());
+  auto AddressInstruction = Builder.buildInstr(opcode(GenericOpcode::G_CONSTANT_POOL));
+  if (!AddressInstruction)
+    return std::unexpected(AddressInstruction.error());
+  auto AddressDef = AddressInstruction->addDef(*Address);
+  if (!AddressDef)
+    return std::unexpected(AddressDef.error());
+  auto AddressPool = AddressInstruction->addConstantPool(*PoolIndex);
+  if (!AddressPool)
+    return std::unexpected(AddressPool.error());
+  auto BuiltAddress = AddressInstruction->build();
+  if (!BuiltAddress)
+    return std::unexpected(BuiltAddress.error());
+
+  MachineMemOperandDesc MemoryDescription;
+  MemoryDescription.Accesses = {MemoryAccessFlag::Load};
+  MemoryDescription.Size = *MemorySize::fixed(8);
+  MemoryDescription.MemoryType = DestinationType;
+  MemoryDescription.Align = *Align;
+  MemoryDescription.Base = MemoryBase::constantPoolPseudo();
+  auto Memory = Function.createMemOperand(std::move(MemoryDescription));
+  if (!Memory)
+    return std::unexpected(Memory.error());
+  auto SetOpcode = Instruction.setOpcode(opcode(GenericOpcode::G_LOAD));
+  if (!SetOpcode)
+    return std::unexpected(SetOpcode.error());
+  while (Instruction.operandCount() > 1) {
+    auto Erased = Instruction.eraseOperand(1);
+    if (!Erased)
+      return std::unexpected(Erased.error());
+  }
+  auto AddressUse =
+      MachineOperand::registerOperand(*Address, RegisterOperandFlags{RegisterOperandFlag::Use});
+  if (!AddressUse)
+    return std::unexpected(AddressUse.error());
+  auto AddedAddress = Instruction.addOperand(std::move(*AddressUse));
+  if (!AddedAddress)
+    return std::unexpected(AddedAddress.error());
+  auto AddedMemory = Instruction.addMemOperand(**Memory);
+  if (!AddedMemory)
+    return std::unexpected(AddedMemory.error());
+  return {};
+}
+
+[[nodiscard]] bool isSupportedNonAtomicMemoryAccess(const MachineInstr &Instruction) noexcept {
+  if (Instruction.memOperands().size() != 1)
+    return false;
+  const MachineMemOperand &Memory = *Instruction.memOperands().front();
+  const MachineMemOperandDesc &Description = Memory.description();
+  return !Memory.isAtomic() && Description.AddressSpace == 0 && Description.Size.isKnown() &&
+         !Description.Size.isScalable() && Description.Size.minimumBytes() != 0;
+}
+
+[[nodiscard]] bool isSupportedAtomicMemoryAccess(const MachineInstr &Instruction,
+                                                 AtomicOperation Operation,
+                                                 std::uint64_t Bytes) noexcept {
+  if (Instruction.memOperands().size() != 1 || (Bytes != 4 && Bytes != 8))
+    return false;
+  const MachineMemOperand &Memory = *Instruction.memOperands().front();
+  const MachineMemOperandDesc &Description = Memory.description();
+  return Description.AtomicKind == Operation && Description.AddressSpace == 0 &&
+         Description.Size.isKnown() && !Description.Size.isScalable() &&
+         Description.Size.minimumBytes() == Bytes && Description.Align.bytes() >= Bytes &&
+         Description.SyncScope.value() == 0;
+}
+
+[[nodiscard]] bool isSupportedAtomicContainerAccess(const MachineInstr &Instruction,
+                                                    AtomicOperation Operation,
+                                                    std::uint64_t ContainerBytes) noexcept {
+  if (Instruction.memOperands().size() != 1 || ContainerBytes == 0)
+    return false;
+  const MachineMemOperandDesc &Description = Instruction.memOperands().front()->description();
+  if (Description.AtomicKind != Operation || Description.AddressSpace != 0 ||
+      !Description.Size.isKnown() || Description.Size.isScalable() ||
+      Description.SyncScope.value() != 0)
+    return false;
+  const std::uint64_t MemoryBytes = Description.Size.minimumBytes();
+  return (MemoryBytes == 1 || MemoryBytes == 2 || MemoryBytes == 4 || MemoryBytes == 8) &&
+         MemoryBytes <= ContainerBytes && Description.Align.bytes() >= MemoryBytes;
+}
+
+[[nodiscard]] std::optional<std::uint64_t>
+memoryAccessBytes(const MachineInstr &Instruction) noexcept {
+  if (Instruction.memOperands().size() != 1)
+    return std::nullopt;
+  const MachineMemOperandDesc &Description = Instruction.memOperands().front()->description();
+  if (!Description.Size.isKnown() || Description.Size.isScalable())
+    return std::nullopt;
+  return Description.Size.minimumBytes();
+}
+
+[[nodiscard]] bool memoryMatchesType(const MachineInstr &Instruction, LLT Type) noexcept {
+  const auto Bytes = memoryAccessBytes(Instruction);
+  if (!Bytes || (!isSupportedIntegerType(Type) && !isRVFloatType(Type)))
+    return false;
+  if (isRVFloatType(Type))
+    return Type.scalarBitWidth() == 32 ? *Bytes == 4 : *Bytes == 8;
+  const std::uint64_t ValueBytes = (Type.scalarBitWidth() + 7U) / 8U;
+  return (*Bytes == 1 || *Bytes == 2 || *Bytes == 4 || *Bytes == 8) && *Bytes <= ValueBytes;
+}
+
+[[nodiscard]] Expected<std::int64_t> signedImmediate(const MachineOperand &Operand) {
+  if (const std::int64_t *Value = Operand.getIf<std::int64_t>())
+    return *Value;
+  const auto *Integer = Operand.getIf<IntegerImmediateOperand>();
+  if (Integer == nullptr || Integer->Value.bitWidth() == 0 || Integer->Value.bitWidth() > 64)
+    return makeUnexpected(ErrorCode::Unsupported,
+                          "RISC-V constant selection requires an immediate no wider than 64 bits");
+  std::uint64_t Raw = Integer->Value.words().empty() ? 0 : Integer->Value.words().front();
+  const std::uint32_t Width = Integer->Value.bitWidth();
+  if (Width < 64 && (Raw & (std::uint64_t{1} << (Width - 1U))) != 0)
+    Raw |= ~std::uint64_t{0} << Width;
+  return std::bit_cast<std::int64_t>(Raw);
+}
+
+[[nodiscard]] Expected<void> replaceRegisterUses(MachineFunction &Function, Register OldReg,
+                                                 Register NewReg) {
+  std::vector<MachineOperand *> Uses;
+  for (MachineOperand &Operand : Function.registerInfo().useOperands(OldReg))
+    Uses.push_back(&Operand);
+  for (MachineOperand *Operand : Uses) {
+    auto Replaced = Operand->setRegister(NewReg);
+    if (!Replaced)
+      return std::unexpected(Replaced.error());
+  }
+  return {};
+}
+
+[[nodiscard]] Expected<void> eraseInstruction(MachineInstr &Instruction) {
+  MachineBasicBlock *Block = Instruction.parent();
+  if (Block == nullptr)
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "RISC-V legalizer cannot erase a detached instruction");
+  return Block->removeInstruction(Instruction);
+}
+
+[[nodiscard]] bool allDefsUnused(const MachineInstr &Instruction, const MachineFunction &Function) {
+  for (const MachineOperand &Operand : Instruction.operands()) {
+    const Register *Reg = Operand.getIf<Register>();
+    if (Reg != nullptr && Operand.isDef() && Reg->isVirtual() &&
+        !Function.registerInfo().useEmpty(*Reg))
+      return false;
+  }
+  return true;
+}
+
+void eraseDeadArtifacts(std::span<MachineInstr *const> Candidates,
+                        const MachineFunction &Function) {
+  bool Changed = false;
+  do {
+    Changed = false;
+    for (auto It = Candidates.rbegin(); It != Candidates.rend(); ++It) {
+      MachineInstr *Instruction = *It;
+      if (Instruction == nullptr || Instruction->parent() == nullptr ||
+          !allDefsUnused(*Instruction, Function))
+        continue;
+      const Opcode Value = Instruction->opcode();
+      if (Value != opcode(GenericOpcode::G_TRUNC) && Value != opcode(GenericOpcode::G_ANYEXT) &&
+          Value != opcode(GenericOpcode::G_SEXT) && Value != opcode(GenericOpcode::G_ZEXT) &&
+          Value != opcode(GenericOpcode::G_MERGE_VALUES) &&
+          Value != opcode(GenericOpcode::G_UNMERGE_VALUES) && Value != opcode(GenericOpcode::COPY))
+        continue;
+      if (eraseInstruction(*Instruction))
+        Changed = true;
+    }
+  } while (Changed);
+}
+
+[[nodiscard]] Expected<Register> buildIntegerConstantBefore(MachineIRBuilder &Builder,
+                                                            MachineInstr &Anchor,
+                                                            std::uint32_t Width,
+                                                            std::uint64_t Value) {
+  auto Type = LLT::makeAnyScalar(Width);
+  if (!Type)
+    return std::unexpected(Type.error());
+  auto Result = Builder.function().registerInfo().createVirtualRegister(*Type);
+  if (!Result)
+    return std::unexpected(Result.error());
+  auto Immediate = IntegerImmediate::fromUnsigned(Width, Value);
+  if (!Immediate)
+    return std::unexpected(Immediate.error());
+  auto Point = Builder.setInsertPointBefore(Anchor);
+  if (!Point)
+    return std::unexpected(Point.error());
+  auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_CONSTANT));
+  if (!Pending)
+    return std::unexpected(Pending.error());
+  if (auto Added = Pending->addDef(*Result); !Added)
+    return std::unexpected(Added.error());
+  if (auto Added = Pending->addIntegerImmediate(std::move(*Immediate)); !Added)
+    return std::unexpected(Added.error());
+  auto Built = Pending->build();
+  if (!Built)
+    return std::unexpected(Built.error());
+  return *Result;
+}
+
+[[nodiscard]] Expected<Register> buildUnaryBefore(MachineIRBuilder &Builder, MachineInstr &Anchor,
+                                                  GenericOpcode Generic, LLT Type,
+                                                  Register Source) {
+  auto Result = Builder.function().registerInfo().createVirtualRegister(Type);
+  if (!Result)
+    return std::unexpected(Result.error());
+  auto Point = Builder.setInsertPointBefore(Anchor);
+  if (!Point)
+    return std::unexpected(Point.error());
+  auto Pending = Builder.buildInstr(opcode(Generic));
+  if (!Pending)
+    return std::unexpected(Pending.error());
+  if (auto Added = Pending->addDef(*Result); !Added)
+    return std::unexpected(Added.error());
+  if (auto Added = Pending->addUse(Source); !Added)
+    return std::unexpected(Added.error());
+  auto Built = Pending->build();
+  if (!Built)
+    return std::unexpected(Built.error());
+  return *Result;
+}
+
+[[nodiscard]] Expected<Register> buildBinaryBefore(MachineIRBuilder &Builder, MachineInstr &Anchor,
+                                                   GenericOpcode Generic, LLT Type, Register Left,
+                                                   Register Right) {
+  auto Result = Builder.function().registerInfo().createVirtualRegister(Type);
+  if (!Result)
+    return std::unexpected(Result.error());
+  auto Point = Builder.setInsertPointBefore(Anchor);
+  if (!Point)
+    return std::unexpected(Point.error());
+  auto Pending = Builder.buildInstr(opcode(Generic));
+  if (!Pending)
+    return std::unexpected(Pending.error());
+  if (auto Added = Pending->addDef(*Result); !Added)
+    return std::unexpected(Added.error());
+  if (auto Added = Pending->addUse(Left); !Added)
+    return std::unexpected(Added.error());
+  if (auto Added = Pending->addUse(Right); !Added)
+    return std::unexpected(Added.error());
+  auto Built = Pending->build();
+  if (!Built)
+    return std::unexpected(Built.error());
+  return *Result;
+}
+
+[[nodiscard]] Expected<void> rewriteUnary(MachineInstr &Instruction, GenericOpcode Generic,
+                                          Register Destination, Register Source) {
+  auto SetOpcode = Instruction.setOpcode(opcode(Generic));
+  if (!SetOpcode)
+    return std::unexpected(SetOpcode.error());
+  while (Instruction.operandCount() != 0) {
+    auto Erased = Instruction.eraseOperand(Instruction.operandCount() - 1U);
+    if (!Erased)
+      return std::unexpected(Erased.error());
+  }
+  auto Def = MachineOperand::registerOperand(Destination,
+                                             RegisterOperandFlags{RegisterOperandFlag::Define});
+  if (!Def)
+    return std::unexpected(Def.error());
+  if (auto Added = Instruction.addOperand(std::move(*Def)); !Added)
+    return std::unexpected(Added.error());
+  auto Use =
+      MachineOperand::registerOperand(Source, RegisterOperandFlags{RegisterOperandFlag::Use});
+  if (!Use)
+    return std::unexpected(Use.error());
+  return Instruction.addOperand(std::move(*Use));
+}
+
+[[nodiscard]] Expected<void> widenNarrowMultiply(MachineInstr &Instruction,
+                                                 MachineFunction &Function,
+                                                 const TargetDescription &Target) {
+  const std::optional<LLT> Type = resultType(Function, Instruction);
+  const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+  const MachineOperand *LeftOperand = Instruction.operandAt(1);
+  const MachineOperand *RightOperand = Instruction.operandAt(2);
+  const Register *Destination =
+      DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+  const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf<Register>();
+  const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf<Register>();
+  if (!Type || Destination == nullptr || Left == nullptr || Right == nullptr ||
+      Type->scalarBitWidth() >= 64) {
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "RISC-V narrow multiply has invalid operands");
+  }
+
+  MachineInstr *Extension = singleRegisterUse(Function, *Destination);
+  if (Extension != nullptr && Extension->opcode() == opcode(GenericOpcode::G_ANYEXT)) {
+    const auto Plan = narrowAnyExtBinaryPlan(*Extension, Function);
+    if (Plan && Plan->Binary == &Instruction)
+      return foldNarrowAnyExtBinary(*Extension, Function, Target);
+  }
+
+  auto WideType = LLT::makeAnyScalar(64);
+  if (!WideType)
+    return std::unexpected(WideType.error());
+  MachineIRBuilder Builder(Function, Target);
+  auto WideLeft = buildUnaryBefore(Builder, Instruction, GenericOpcode::G_ANYEXT, *WideType, *Left);
+  if (!WideLeft)
+    return std::unexpected(WideLeft.error());
+  auto WideRight =
+      buildUnaryBefore(Builder, Instruction, GenericOpcode::G_ANYEXT, *WideType, *Right);
+  if (!WideRight)
+    return std::unexpected(WideRight.error());
+  auto Product = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_MUL, *WideType, *WideLeft,
+                                   *WideRight);
+  if (!Product)
+    return std::unexpected(Product.error());
+  return rewriteUnary(Instruction, GenericOpcode::G_TRUNC, *Destination, *Product);
+}
+
+[[nodiscard]] Expected<void> lowerMultiplyLibcall(MachineInstr &Instruction,
+                                                  MachineFunction &Function,
+                                                  const TargetDescription &Target,
+                                                  CallingConvention Convention,
+                                                  const TargetCallLowering &CallLowering) {
+  const std::optional<LLT> Type = resultType(Function, Instruction);
+  const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+  const MachineOperand *LeftOperand = Instruction.operandAt(1);
+  const MachineOperand *RightOperand = Instruction.operandAt(2);
+  const Register *Destination =
+      DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+  const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf<Register>();
+  const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf<Register>();
+  if (!Type || Destination == nullptr || Left == nullptr || Right == nullptr ||
+      Type->scalarBitWidth() != 64) {
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "RISC-V multiply libcall requires three s64 registers");
+  }
+  auto ABIAlignment = Alignment::create(8);
+  auto Callee = MachineCallTarget::externalSymbol("__muldi3");
+  if (!ABIAlignment || !Callee)
+    return makeUnexpected(ErrorCode::InternalError,
+                          "RISC-V multiply libcall ABI facts are unavailable");
+  const auto Value = [Type = *Type, Alignment = *ABIAlignment](Register Reg, std::uint32_t Number) {
+    return ABIValue{
+        .Value = Reg,
+        .Type = Type,
+        .OriginalValue = Number,
+        .FragmentIndex = 0,
+        .FragmentCount = 1,
+        .FragmentOffsetBytes = 0,
+        .OriginalAlignment = Alignment,
+        .Extension = ABIExtensionKind::None,
+        .Flags = {},
+        .ByValSizeBytes = 0,
+        .ByValAlignment = Alignment,
+    };
+  };
+
+  CallLoweringRequest Request;
+  Request.Convention = callingConventionID(Convention);
+  Request.Callee = std::move(*Callee);
+  Request.Arguments = {Value(*Left, 0), Value(*Right, 1)};
+  Request.Results = {Value(*Destination, 0)};
+  Request.FixedArgumentCount = Request.Arguments.size();
+
+  MachineIRBuilder Builder(Function, Target);
+  auto Point = Builder.setInsertPointBefore(Instruction);
+  if (!Point)
+    return std::unexpected(Point.error());
+  auto Lowered = CallLowering.lowerCall(Builder, Request);
+  if (!Lowered)
+    return std::unexpected(Lowered.error());
+  return eraseInstruction(Instruction);
+}
+
+[[nodiscard]] Expected<void> rewriteBinary(MachineInstr &Instruction, GenericOpcode Generic,
+                                           Register Destination, Register Left, Register Right) {
+  auto Rewritten = rewriteUnary(Instruction, Generic, Destination, Left);
+  if (!Rewritten)
+    return std::unexpected(Rewritten.error());
+  auto Use = MachineOperand::registerOperand(Right, RegisterOperandFlags{RegisterOperandFlag::Use});
+  if (!Use)
+    return std::unexpected(Use.error());
+  return Instruction.addOperand(std::move(*Use));
+}
+
+[[nodiscard]] bool isKnownZeroExtended(const MachineFunction &Function, Register Reg,
+                                       std::uint32_t ActiveBits, unsigned Depth = 0) {
+  if (Depth > 8 || !Reg.isVirtual())
+    return false;
+  const auto Info = Function.registerInfo().virtualRegisterInfo(Reg);
+  if (!Info || !Info->get().Type)
+    return false;
+  const std::uint32_t Width = Info->get().Type->scalarBitWidth();
+  if (Width <= ActiveBits)
+    return true;
+  const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(Reg);
+  const MachineInstr *Def = Definition == nullptr ? nullptr : Definition->parent();
+  if (Def == nullptr)
+    return false;
+  if (Def->opcode() == opcode(GenericOpcode::G_ASSERT_ZEXT)) {
+    const MachineOperand *BitsOperand = Def->operandAt(2);
+    auto Bits = BitsOperand == nullptr
+                    ? Expected<std::int64_t>(std::unexpected(
+                          Error(ErrorCode::InvalidArgument, "missing asserted extension width")))
+                    : signedImmediate(*BitsOperand);
+    return Bits && *Bits >= 0 && static_cast<std::uint64_t>(*Bits) <= ActiveBits;
+  }
+  if (Def->opcode() == opcode(GenericOpcode::G_ZEXT)) {
+    const MachineOperand *SourceOperand = Def->operandAt(1);
+    const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>();
+    if (Source == nullptr)
+      return false;
+    const auto SourceInfo = Function.registerInfo().virtualRegisterInfo(*Source);
+    return SourceInfo && SourceInfo->get().Type &&
+           SourceInfo->get().Type->scalarBitWidth() <= ActiveBits;
+  }
+  if (Def->opcode() == opcode(GenericOpcode::G_TRUNC)) {
+    const MachineOperand *SourceOperand = Def->operandAt(1);
+    const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>();
+    return Source != nullptr && isKnownZeroExtended(Function, *Source, ActiveBits, Depth + 1U);
+  }
+  return false;
+}
+
+[[nodiscard]] Expected<Register> zeroExtendToRV64(MachineIRBuilder &Builder, MachineInstr &Anchor,
+                                                  MachineFunction &Function, Register Source,
+                                                  std::vector<MachineInstr *> &DeadCandidates) {
+  const auto Info = Function.registerInfo().virtualRegisterInfo(Source);
+  if (!Info || !Info->get().Type)
+    return makeUnexpected(ErrorCode::VerificationError, "RISC-V merge source has no scalar type");
+  const LLT SourceType = *Info->get().Type;
+  if (SourceType.scalarBitWidth() == 64)
+    return Source;
+  const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(Source);
+  MachineInstr *Def =
+      Definition == nullptr ? nullptr : const_cast<MachineInstr *>(Definition->parent());
+  if (Def != nullptr && Def->opcode() == opcode(GenericOpcode::G_TRUNC)) {
+    const MachineOperand *WideOperand = Def->operandAt(1);
+    const Register *Wide = WideOperand == nullptr ? nullptr : WideOperand->getIf<Register>();
+    if (Wide != nullptr) {
+      const auto WideInfo = Function.registerInfo().virtualRegisterInfo(*Wide);
+      if (WideInfo && WideInfo->get().Type && WideInfo->get().Type->scalarBitWidth() == 64 &&
+          isKnownZeroExtended(Function, *Wide, SourceType.scalarBitWidth())) {
+        DeadCandidates.push_back(Def);
+        return *Wide;
+      }
+    }
+  }
+  auto WideType = LLT::makeAnyScalar(64);
+  if (!WideType)
+    return std::unexpected(WideType.error());
+  return buildUnaryBefore(Builder, Anchor, GenericOpcode::G_ZEXT, *WideType, Source);
+}
+
+[[nodiscard]] MachineInstr *singleUseInstruction(const MachineFunction &Function, Register Reg) {
+  MachineInstr *Result = nullptr;
+  auto &RegisterInfo = const_cast<MachineRegisterInfo &>(Function.registerInfo());
+  for (MachineOperand &Operand : RegisterInfo.useOperands(Reg)) {
+    MachineInstr *Instruction = Operand.parent();
+    if (Instruction == nullptr || Instruction->parent() == nullptr)
+      continue;
+    if (Result != nullptr && Result != Instruction)
+      return nullptr;
+    Result = Instruction;
+  }
+  return Result;
+}
+
+[[nodiscard]] Expected<void> lowerWideMergeValues(MachineInstr &Instruction,
+                                                  MachineFunction &Function) {
+  const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+  const Register *Destination =
+      DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+  const MachineOperand *LowOperand = Instruction.operandAt(1);
+  const Register *Low = LowOperand == nullptr ? nullptr : LowOperand->getIf<Register>();
+  if (Destination == nullptr || Low == nullptr)
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "RISC-V wide merge has invalid register operands");
+
+  std::vector<MachineInstr *> Truncations;
+  auto &RegisterInfo = Function.registerInfo();
+  std::vector<MachineOperand *> Uses;
+  for (MachineOperand &Use : RegisterInfo.useOperands(*Destination))
+    Uses.push_back(&Use);
+  for (MachineOperand *Use : Uses) {
+    MachineInstr *Trunc = Use->parent();
+    if (Trunc == nullptr || Trunc->parent() == nullptr ||
+        Trunc->opcode() != opcode(GenericOpcode::G_TRUNC) || Trunc->operandAt(1) != Use)
+      return makeUnexpected(ErrorCode::Unsupported,
+                            "RISC-V wide merge is only representable through low-part truncation");
+    const MachineOperand *TruncResultOperand = Trunc->operandAt(0);
+    const Register *TruncResult =
+        TruncResultOperand == nullptr ? nullptr : TruncResultOperand->getIf<Register>();
+    const auto LowInfo = RegisterInfo.virtualRegisterInfo(*Low);
+    const auto ResultInfo =
+        TruncResult == nullptr
+            ? Expected<std::reference_wrapper<const VirtualRegisterInfo>>(
+                  std::unexpected(Error(ErrorCode::InvalidArgument, "missing truncation result")))
+            : RegisterInfo.virtualRegisterInfo(*TruncResult);
+    if (!LowInfo || !LowInfo->get().Type || !ResultInfo || !ResultInfo->get().Type ||
+        ResultInfo->get().Type->scalarBitWidth() > LowInfo->get().Type->scalarBitWidth())
+      return makeUnexpected(ErrorCode::Unsupported,
+                            "RISC-V wide merge truncation spans more than its low source");
+    if (*ResultInfo->get().Type == *LowInfo->get().Type) {
+      auto Replaced = replaceRegisterUses(Function, *TruncResult, *Low);
+      if (!Replaced)
+        return std::unexpected(Replaced.error());
+      Truncations.push_back(Trunc);
+    } else {
+      auto Replaced = Use->setRegister(*Low);
+      if (!Replaced)
+        return std::unexpected(Replaced.error());
+    }
+  }
+  for (MachineInstr *Trunc : Truncations) {
+    auto Erased = eraseInstruction(*Trunc);
+    if (!Erased)
+      return std::unexpected(Erased.error());
+  }
+  return eraseInstruction(Instruction);
+}
+
+[[nodiscard]] Expected<void> lowerMergeValues(MachineInstr &Instruction, MachineFunction &Function,
+                                              const TargetDescription &Target) {
+  const std::optional<LLT> DestinationType = resultType(Function, Instruction);
+  if (!DestinationType || DestinationType->scalarBitWidth() == 0)
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "RISC-V merge destination has no scalar type");
+  if (DestinationType->scalarBitWidth() > 64)
+    return lowerWideMergeValues(Instruction, Function);
+
+  const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+  const Register *Destination =
+      DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+  if (Destination == nullptr || Instruction.operandCount() < 3)
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "RISC-V merge has invalid register operands");
+
+  Register Result = *Destination;
+  MachineInstr *FoldedExtension = nullptr;
+  if (DestinationType->scalarBitWidth() < 64) {
+    MachineInstr *Use = singleUseInstruction(Function, *Destination);
+    if (Use != nullptr && Use->opcode() == opcode(GenericOpcode::G_ZEXT) &&
+        operandType(Function, *Use, 0)
+            .transform([](LLT Type) { return Type.scalarBitWidth() == 64; })
+            .value_or(false)) {
+      const MachineOperand *ExtendedOperand = Use->operandAt(0);
+      const Register *Extended =
+          ExtendedOperand == nullptr ? nullptr : ExtendedOperand->getIf<Register>();
+      if (Extended != nullptr) {
+        Result = *Extended;
+        FoldedExtension = Use;
+      }
+    }
+  }
+
+  auto WideType = LLT::makeAnyScalar(64);
+  if (!WideType)
+    return std::unexpected(WideType.error());
+  MachineIRBuilder Builder(Function, Target);
+  std::vector<MachineInstr *> DeadCandidates;
+  std::vector<Register> Terms;
+  Terms.reserve(Instruction.operandCount() - 1U);
+  for (std::size_t Index = 1; Index < Instruction.operandCount(); ++Index) {
+    const MachineOperand *Operand = Instruction.operandAt(Index);
+    const Register *Source = Operand == nullptr ? nullptr : Operand->getIf<Register>();
+    if (Source == nullptr)
+      return makeUnexpected(ErrorCode::VerificationError, "RISC-V merge source is not a register");
+    auto WideSource = zeroExtendToRV64(Builder, Instruction, Function, *Source, DeadCandidates);
+    if (!WideSource)
+      return std::unexpected(WideSource.error());
+    if (Index == 1) {
+      Terms.push_back(*WideSource);
+      continue;
+    }
+    const std::optional<LLT> SourceType = registerType(Function, *Operand);
+    if (!SourceType)
+      return makeUnexpected(ErrorCode::VerificationError, "RISC-V merge source has no scalar type");
+    const std::uint64_t Shift =
+        static_cast<std::uint64_t>(SourceType->scalarBitWidth()) * (Index - 1U);
+    auto Amount = buildIntegerConstantBefore(Builder, Instruction, 64, Shift);
+    if (!Amount)
+      return std::unexpected(Amount.error());
+    auto Shifted = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_SHL, *WideType,
+                                     *WideSource, *Amount);
+    if (!Shifted)
+      return std::unexpected(Shifted.error());
+    Terms.push_back(*Shifted);
+  }
+
+  Register Accumulator = Terms.front();
+  for (std::size_t Index = 1; Index + 1U < Terms.size(); ++Index) {
+    auto Combined = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_OR, *WideType,
+                                      Accumulator, Terms[Index]);
+    if (!Combined)
+      return std::unexpected(Combined.error());
+    Accumulator = *Combined;
+  }
+  if (DestinationType->scalarBitWidth() == 64 || FoldedExtension != nullptr) {
+    auto Rewritten =
+        rewriteBinary(Instruction, GenericOpcode::G_OR, Result, Accumulator, Terms.back());
+    if (!Rewritten)
+      return std::unexpected(Rewritten.error());
+  } else {
+    auto Combined = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_OR, *WideType,
+                                      Accumulator, Terms.back());
+    if (!Combined)
+      return std::unexpected(Combined.error());
+    auto Rewritten = rewriteUnary(Instruction, GenericOpcode::G_TRUNC, *Destination, *Combined);
+    if (!Rewritten)
+      return std::unexpected(Rewritten.error());
+  }
+  if (FoldedExtension != nullptr) {
+    auto Erased = eraseInstruction(*FoldedExtension);
+    if (!Erased)
+      return std::unexpected(Erased.error());
+  }
+  eraseDeadArtifacts(DeadCandidates, Function);
+  return {};
+}
+
+[[nodiscard]] Expected<void> foldWideExtendedUnmerge(MachineInstr &Instruction,
+                                                     MachineFunction &Function) {
+  const std::size_t OutputCount = Instruction.operandCount() - 1U;
+  const MachineOperand *SourceOperand = Instruction.operandAt(OutputCount);
+  const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>();
+  const MachineOperand *Definition =
+      Source == nullptr ? nullptr : Function.registerInfo().getUniqueVRegDef(*Source);
+  MachineInstr *Extension =
+      Definition == nullptr ? nullptr : const_cast<MachineInstr *>(Definition->parent());
+  if (Extension == nullptr || (Extension->opcode() != opcode(GenericOpcode::G_ZEXT) &&
+                               Extension->opcode() != opcode(GenericOpcode::G_ANYEXT) &&
+                               Extension->opcode() != opcode(GenericOpcode::G_SEXT)))
+    return makeUnexpected(ErrorCode::Unsupported,
+                          "RISC-V wide unmerge requires an extension-defined source");
+  const MachineOperand *NarrowOperand = Extension->operandAt(1);
+  const Register *Narrow = NarrowOperand == nullptr ? nullptr : NarrowOperand->getIf<Register>();
+  if (Narrow == nullptr)
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "RISC-V wide unmerge extension has no register source");
+  const auto NarrowInfo = Function.registerInfo().virtualRegisterInfo(*Narrow);
+  if (!NarrowInfo || !NarrowInfo->get().Type)
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "RISC-V wide unmerge extension source has no type");
+  const std::uint32_t NarrowWidth = NarrowInfo->get().Type->scalarBitWidth();
+
+  std::vector<MachineInstr *> DeadUsers;
+  for (std::size_t Index = 0; Index < OutputCount; ++Index) {
+    const MachineOperand *OutputOperand = Instruction.operandAt(Index);
+    const Register *Output = OutputOperand == nullptr ? nullptr : OutputOperand->getIf<Register>();
+    if (Output == nullptr || Function.registerInfo().useEmpty(*Output))
+      continue;
+    const auto OutputInfo = Function.registerInfo().virtualRegisterInfo(*Output);
+    if (!OutputInfo || !OutputInfo->get().Type)
+      return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide unmerge output has no type");
+    const std::uint64_t Offset =
+        static_cast<std::uint64_t>(Index) * OutputInfo->get().Type->scalarBitWidth();
+    if (Offset != 0)
+      return makeUnexpected(ErrorCode::Unsupported,
+                            "RISC-V wide unmerge cannot materialize a used high wide part");
+    if (OutputInfo->get().Type->scalarBitWidth() == NarrowWidth) {
+      auto Replaced = replaceRegisterUses(Function, *Output, *Narrow);
+      if (!Replaced)
+        return std::unexpected(Replaced.error());
+      continue;
+    }
+    if (OutputInfo->get().Type->scalarBitWidth() < NarrowWidth)
+      return makeUnexpected(ErrorCode::Unsupported,
+                            "RISC-V wide unmerge output unexpectedly narrows its source");
+    std::vector<MachineOperand *> Uses;
+    for (MachineOperand &Use : Function.registerInfo().useOperands(*Output))
+      Uses.push_back(&Use);
+    for (MachineOperand *Use : Uses) {
+      MachineInstr *Trunc = Use->parent();
+      if (Trunc == nullptr || Trunc->parent() == nullptr ||
+          Trunc->opcode() != opcode(GenericOpcode::G_TRUNC) || Trunc->operandAt(1) != Use)
+        return makeUnexpected(
+            ErrorCode::Unsupported,
+            "RISC-V wide unmerge result is only representable through truncation");
+      const MachineOperand *ResultOperand = Trunc->operandAt(0);
+      const Register *Result =
+          ResultOperand == nullptr ? nullptr : ResultOperand->getIf<Register>();
+      const auto ResultInfo =
+          Result == nullptr
+              ? Expected<std::reference_wrapper<const VirtualRegisterInfo>>(
+                    std::unexpected(Error(ErrorCode::InvalidArgument, "missing truncation result")))
+              : Function.registerInfo().virtualRegisterInfo(*Result);
+      if (!ResultInfo || !ResultInfo->get().Type ||
+          ResultInfo->get().Type->scalarBitWidth() > NarrowWidth)
+        return makeUnexpected(ErrorCode::Unsupported,
+                              "RISC-V wide unmerge truncation exceeds its source width");
+      if (ResultInfo->get().Type->scalarBitWidth() == NarrowWidth) {
+        auto Replaced = replaceRegisterUses(Function, *Result, *Narrow);
+        if (!Replaced)
+          return std::unexpected(Replaced.error());
+        DeadUsers.push_back(Trunc);
+      } else {
+        auto Replaced = Use->setRegister(*Narrow);
+        if (!Replaced)
+          return std::unexpected(Replaced.error());
+      }
+    }
+  }
+  for (MachineInstr *User : DeadUsers) {
+    auto Erased = eraseInstruction(*User);
+    if (!Erased)
+      return std::unexpected(Erased.error());
+  }
+  auto Erased = eraseInstruction(Instruction);
+  if (!Erased)
+    return std::unexpected(Erased.error());
+  if (Function.registerInfo().useEmpty(*Source))
+    return eraseInstruction(*Extension);
+  return {};
+}
+
+[[nodiscard]] Expected<void> lowerUnmergeValues(MachineInstr &Instruction,
+                                                MachineFunction &Function,
+                                                const TargetDescription &Target) {
+  if (Instruction.operandCount() < 3)
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "RISC-V unmerge has invalid register operands");
+  const std::size_t OutputCount = Instruction.operandCount() - 1U;
+  const std::optional<LLT> OutputType = operandType(Function, Instruction, 0);
+  const std::optional<LLT> SourceType = operandType(Function, Instruction, OutputCount);
+  if (!OutputType || !SourceType)
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "RISC-V unmerge operands have no scalar types");
+  if (SourceType->scalarBitWidth() > 64 || OutputType->scalarBitWidth() > 64)
+    return foldWideExtendedUnmerge(Instruction, Function);
+
+  const MachineOperand *SourceOperand = Instruction.operandAt(OutputCount);
+  const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>();
+  if (Source == nullptr)
+    return makeUnexpected(ErrorCode::VerificationError, "RISC-V unmerge source is not a register");
+  auto WideType = LLT::makeAnyScalar(64);
+  if (!WideType)
+    return std::unexpected(WideType.error());
+  MachineIRBuilder Builder(Function, Target);
+  std::vector<MachineInstr *> FoldedExtensions;
+  for (std::size_t Index = 0; Index < OutputCount; ++Index) {
+    const MachineOperand *OutputOperand = Instruction.operandAt(Index);
+    const Register *Output = OutputOperand == nullptr ? nullptr : OutputOperand->getIf<Register>();
+    if (Output == nullptr || Function.registerInfo().useEmpty(*Output))
+      continue;
+    Register Extracted = *Source;
+    const std::uint64_t Offset = static_cast<std::uint64_t>(Index) * OutputType->scalarBitWidth();
+    if (Offset != 0) {
+      auto Amount = buildIntegerConstantBefore(Builder, Instruction, 64, Offset);
+      if (!Amount)
+        return std::unexpected(Amount.error());
+      auto Shifted = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_LSHR, *WideType,
+                                       *Source, *Amount);
+      if (!Shifted)
+        return std::unexpected(Shifted.error());
+      Extracted = *Shifted;
+    }
+
+    MachineInstr *Extension = singleUseInstruction(Function, *Output);
+    const bool FoldZExt = Extension != nullptr &&
+                          Extension->opcode() == opcode(GenericOpcode::G_ZEXT) &&
+                          operandType(Function, *Extension, 0)
+                              .transform([](LLT Type) { return Type.scalarBitWidth() == 64; })
+                              .value_or(false);
+    if (FoldZExt) {
+      const MachineOperand *ExtendedOperand = Extension->operandAt(0);
+      const Register *Extended =
+          ExtendedOperand == nullptr ? nullptr : ExtendedOperand->getIf<Register>();
+      if (Extended == nullptr)
+        return makeUnexpected(ErrorCode::VerificationError,
+                              "RISC-V unmerge extension has no destination");
+      const std::uint64_t Mask = OutputType->scalarBitWidth() == 64
+                                     ? std::numeric_limits<std::uint64_t>::max()
+                                     : (std::uint64_t{1} << OutputType->scalarBitWidth()) - 1U;
+      auto MaskReg = buildIntegerConstantBefore(Builder, Instruction, 64, Mask);
+      if (!MaskReg)
+        return std::unexpected(MaskReg.error());
+      auto Masked = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_AND, *WideType,
+                                      Extracted, *MaskReg);
+      if (!Masked)
+        return std::unexpected(Masked.error());
+      auto Replaced = replaceRegisterUses(Function, *Extended, *Masked);
+      if (!Replaced)
+        return std::unexpected(Replaced.error());
+      FoldedExtensions.push_back(Extension);
+      continue;
+    }
+    auto Truncated =
+        buildUnaryBefore(Builder, Instruction, GenericOpcode::G_TRUNC, *OutputType, Extracted);
+    if (!Truncated)
+      return std::unexpected(Truncated.error());
+    auto Replaced = replaceRegisterUses(Function, *Output, *Truncated);
+    if (!Replaced)
+      return std::unexpected(Replaced.error());
+  }
+  for (MachineInstr *Extension : FoldedExtensions) {
+    auto Erased = eraseInstruction(*Extension);
+    if (!Erased)
+      return std::unexpected(Erased.error());
+  }
+  return eraseInstruction(Instruction);
+}
+
+struct WideValueParts final {
+  std::uint32_t BitWidth = 0;
+  std::vector<Register> Parts;
+  std::vector<MachineInstr *> Artifacts;
+};
+
+[[nodiscard]] Expected<WideValueParts>
+collectWideValueParts(Register Value, MachineFunction &Function, unsigned Depth = 0) {
+  if (Depth > 16)
+    return makeUnexpected(ErrorCode::Unsupported, "RISC-V wide-value artifact chain is too deep");
+  const auto Info = Function.registerInfo().virtualRegisterInfo(Value);
+  if (!Info || !Info->get().Type)
+    return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide value has no scalar type");
+  const LLT Type = *Info->get().Type;
+  if ((!Type.isInteger() && !Type.isAnyScalar()) || Type.scalarBitWidth() == 0)
+    return makeUnexpected(ErrorCode::Unsupported,
+                          "RISC-V wide-value decomposition requires an integer scalar");
+  if (Type.scalarBitWidth() <= 64)
+    return WideValueParts{.BitWidth = Type.scalarBitWidth(), .Parts = {Value}, .Artifacts = {}};
+
+  const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(Value);
+  MachineInstr *Def =
+      Definition == nullptr ? nullptr : const_cast<MachineInstr *>(Definition->parent());
+  if (Def == nullptr)
+    return makeUnexpected(ErrorCode::Unsupported,
+                          "RISC-V wide value has no decomposable definition");
+  if (Def->opcode() == opcode(GenericOpcode::G_MERGE_VALUES)) {
+    WideValueParts Result;
+    Result.BitWidth = Type.scalarBitWidth();
+    for (std::size_t Index = 1; Index < Def->operandCount(); ++Index) {
+      const MachineOperand *Operand = Def->operandAt(Index);
+      const Register *Part = Operand == nullptr ? nullptr : Operand->getIf<Register>();
+      const std::optional<LLT> PartType =
+          Operand == nullptr ? std::nullopt : registerType(Function, *Operand);
+      if (Part == nullptr || !PartType || PartType->scalarBitWidth() != 64)
+        return makeUnexpected(ErrorCode::Unsupported,
+                              "RISC-V wide merge must already use 64-bit limbs");
+      Result.Parts.push_back(*Part);
+    }
+    Result.Artifacts.push_back(Def);
+    return Result;
+  }
+  if (Def->opcode() == opcode(GenericOpcode::G_TRUNC)) {
+    const MachineOperand *SourceOperand = Def->operandAt(1);
+    const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>();
+    if (Source == nullptr)
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "RISC-V wide truncation has no source register");
+    auto Result = collectWideValueParts(*Source, Function, Depth + 1U);
+    if (!Result)
+      return std::unexpected(Result.error());
+    Result->BitWidth = Type.scalarBitWidth();
+    const std::size_t RequiredParts = (Type.scalarBitWidth() + 63U) / 64U;
+    if (Result->Parts.size() < RequiredParts)
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "RISC-V wide truncation source has too few limbs");
+    Result->Parts.resize(RequiredParts);
+    Result->Artifacts.push_back(Def);
+    return Result;
+  }
+  return makeUnexpected(ErrorCode::Unsupported,
+                        "RISC-V wide value is not defined by merge/trunc artifacts");
+}
+
+struct WideResultPlan final {
+  std::vector<Register> Outputs;
+  MachineInstr *Extension = nullptr;
+  MachineInstr *Unmerge = nullptr;
+};
+
+[[nodiscard]] Expected<WideResultPlan> wideResultPlan(Register Result, MachineFunction &Function,
+                                                      std::size_t RequiredParts) {
+  MachineInstr *Use = singleUseInstruction(Function, Result);
+  WideResultPlan Plan;
+  if (Use != nullptr && (Use->opcode() == opcode(GenericOpcode::G_ANYEXT) ||
+                         Use->opcode() == opcode(GenericOpcode::G_SEXT) ||
+                         Use->opcode() == opcode(GenericOpcode::G_ZEXT))) {
+    Plan.Extension = Use;
+    const MachineOperand *ExtendedOperand = Use->operandAt(0);
+    const Register *Extended =
+        ExtendedOperand == nullptr ? nullptr : ExtendedOperand->getIf<Register>();
+    Use = Extended == nullptr ? nullptr : singleUseInstruction(Function, *Extended);
+  }
+  if (Use == nullptr || Use->opcode() != opcode(GenericOpcode::G_UNMERGE_VALUES))
+    return makeUnexpected(ErrorCode::Unsupported,
+                          "RISC-V wide result must feed an unmerge artifact");
+  Plan.Unmerge = Use;
+  const std::size_t OutputCount = Use->operandCount() - 1U;
+  if (OutputCount < RequiredParts)
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "RISC-V wide result unmerge has too few outputs");
+  Plan.Outputs.reserve(RequiredParts);
+  for (std::size_t Index = 0; Index < RequiredParts; ++Index) {
+    const MachineOperand *Operand = Use->operandAt(Index);
+    const Register *Output = Operand == nullptr ? nullptr : Operand->getIf<Register>();
+    const std::optional<LLT> Type =
+        Operand == nullptr ? std::nullopt : registerType(Function, *Operand);
+    if (Output == nullptr || !Type || Type->scalarBitWidth() != 64)
+      return makeUnexpected(ErrorCode::Unsupported,
+                            "RISC-V wide result must unmerge to 64-bit limbs");
+    Plan.Outputs.push_back(*Output);
+  }
+  return Plan;
+}
+
+[[nodiscard]] Expected<Register> buildIntegerCompareBefore(MachineIRBuilder &Builder,
+                                                           MachineInstr &Anchor,
+                                                           IntegerPredicate Predicate,
+                                                           Register Left, Register Right) {
+  auto Type = LLT::makeAnyScalar(64);
+  if (!Type)
+    return std::unexpected(Type.error());
+  auto Result = Builder.function().registerInfo().createVirtualRegister(*Type);
+  if (!Result)
+    return std::unexpected(Result.error());
+  auto Point = Builder.setInsertPointBefore(Anchor);
+  if (!Point)
+    return std::unexpected(Point.error());
+  auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_ICMP));
+  if (!Pending)
+    return std::unexpected(Pending.error());
+  if (auto Added = Pending->addDef(*Result); !Added)
+    return std::unexpected(Added.error());
+  if (auto Added = Pending->addIntegerPredicate(Predicate); !Added)
+    return std::unexpected(Added.error());
+  if (auto Added = Pending->addUse(Left); !Added)
+    return std::unexpected(Added.error());
+  if (auto Added = Pending->addUse(Right); !Added)
+    return std::unexpected(Added.error());
+  auto Built = Pending->build();
+  if (!Built)
+    return std::unexpected(Built.error());
+  return *Result;
+}
+
+[[nodiscard]] Expected<Register> buildSelectBefore(MachineIRBuilder &Builder, MachineInstr &Anchor,
+                                                   LLT Type, Register Condition, Register TrueValue,
+                                                   Register FalseValue) {
+  auto Result = Builder.function().registerInfo().createVirtualRegister(Type);
+  if (!Result)
+    return std::unexpected(Result.error());
+  auto Point = Builder.setInsertPointBefore(Anchor);
+  if (!Point)
+    return std::unexpected(Point.error());
+  auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_SELECT));
+  if (!Pending)
+    return std::unexpected(Pending.error());
+  if (auto Added = Pending->addDef(*Result); !Added)
+    return std::unexpected(Added.error());
+  if (auto Added = Pending->addUse(Condition); !Added)
+    return std::unexpected(Added.error());
+  if (auto Added = Pending->addUse(TrueValue); !Added)
+    return std::unexpected(Added.error());
+  if (auto Added = Pending->addUse(FalseValue); !Added)
+    return std::unexpected(Added.error());
+  auto Built = Pending->build();
+  if (!Built)
+    return std::unexpected(Built.error());
+  return *Result;
+}
+
+struct TwoLimbShiftResult final {
+  Register Low;
+  Register High;
+};
+
+[[nodiscard]] Expected<TwoLimbShiftResult>
+emitTwoLimbLeftShiftBody(MachineIRBuilder &Builder, MachineInstr &Anchor, Register Low,
+                         Register High, Register Amount, Register AmountExcess, Register AmountLack,
+                         Register Zero, Register IsShort, Register IsZero) {
+  auto Type = LLT::makeAnyScalar(64);
+  if (!Type)
+    return std::unexpected(Type.error());
+  auto LowShort = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SHL, *Type, Low, Amount);
+  if (!LowShort)
+    return std::unexpected(LowShort.error());
+  auto CrossLow = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_LSHR, *Type, Low, AmountLack);
+  if (!CrossLow)
+    return std::unexpected(CrossLow.error());
+  auto HighShort = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SHL, *Type, High, Amount);
+  if (!HighShort)
+    return std::unexpected(HighShort.error());
+  auto ShortHigh =
+      buildBinaryBefore(Builder, Anchor, GenericOpcode::G_OR, *Type, *CrossLow, *HighShort);
+  if (!ShortHigh)
+    return std::unexpected(ShortHigh.error());
+  auto LongHigh =
+      buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SHL, *Type, Low, AmountExcess);
+  if (!LongHigh)
+    return std::unexpected(LongHigh.error());
+  auto ResultLow = buildSelectBefore(Builder, Anchor, *Type, IsShort, *LowShort, Zero);
+  if (!ResultLow)
+    return std::unexpected(ResultLow.error());
+  auto ShiftedHigh = buildSelectBefore(Builder, Anchor, *Type, IsShort, *ShortHigh, *LongHigh);
+  if (!ShiftedHigh)
+    return std::unexpected(ShiftedHigh.error());
+  auto ResultHigh = buildSelectBefore(Builder, Anchor, *Type, IsZero, High, *ShiftedHigh);
+  if (!ResultHigh)
+    return std::unexpected(ResultHigh.error());
+  return TwoLimbShiftResult{.Low = *ResultLow, .High = *ResultHigh};
+}
+
+[[nodiscard]] Expected<TwoLimbShiftResult> emitTwoLimbLeftShift(MachineIRBuilder &Builder,
+                                                                MachineInstr &Anchor, Register Low,
+                                                                Register High, Register Amount) {
+  auto Type = LLT::makeAnyScalar(64);
+  if (!Type)
+    return std::unexpected(Type.error());
+  auto Boundary = buildIntegerConstantBefore(Builder, Anchor, 64, 64);
+  if (!Boundary)
+    return std::unexpected(Boundary.error());
+  auto AmountExcess =
+      buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, Amount, *Boundary);
+  if (!AmountExcess)
+    return std::unexpected(AmountExcess.error());
+  auto AmountLack =
+      buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, *Boundary, Amount);
+  if (!AmountLack)
+    return std::unexpected(AmountLack.error());
+  auto Zero = buildIntegerConstantBefore(Builder, Anchor, 64, 0);
+  if (!Zero)
+    return std::unexpected(Zero.error());
+  auto IsShort = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan,
+                                           Amount, *Boundary);
+  if (!IsShort)
+    return std::unexpected(IsShort.error());
+  auto IsZero = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::Equal, Amount, *Zero);
+  if (!IsZero)
+    return std::unexpected(IsZero.error());
+  return emitTwoLimbLeftShiftBody(Builder, Anchor, Low, High, Amount, *AmountExcess, *AmountLack,
+                                  *Zero, *IsShort, *IsZero);
+}
+
+[[nodiscard]] Expected<Register>
+emitLowLimbOfTwoLimbLogicalRightShift(MachineIRBuilder &Builder, MachineInstr &Anchor, Register Low,
+                                      Register High, Register Amount, Register Boundary,
+                                      Register Zero) {
+  auto Type = LLT::makeAnyScalar(64);
+  if (!Type)
+    return std::unexpected(Type.error());
+  auto AmountExcess =
+      buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, Amount, Boundary);
+  if (!AmountExcess)
+    return std::unexpected(AmountExcess.error());
+  auto AmountLack =
+      buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, Boundary, Amount);
+  if (!AmountLack)
+    return std::unexpected(AmountLack.error());
+  auto IsShort = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan,
+                                           Amount, Boundary);
+  if (!IsShort)
+    return std::unexpected(IsShort.error());
+  auto IsZero = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::Equal, Amount, Zero);
+  if (!IsZero)
+    return std::unexpected(IsZero.error());
+  auto LowShort = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_LSHR, *Type, Low, Amount);
+  if (!LowShort)
+    return std::unexpected(LowShort.error());
+  auto HighShort =
+      buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SHL, *Type, High, *AmountLack);
+  if (!HighShort)
+    return std::unexpected(HighShort.error());
+  auto ShortValue =
+      buildBinaryBefore(Builder, Anchor, GenericOpcode::G_OR, *Type, *LowShort, *HighShort);
+  if (!ShortValue)
+    return std::unexpected(ShortValue.error());
+  auto LongValue =
+      buildBinaryBefore(Builder, Anchor, GenericOpcode::G_LSHR, *Type, High, *AmountExcess);
+  if (!LongValue)
+    return std::unexpected(LongValue.error());
+  auto Shifted = buildSelectBefore(Builder, Anchor, *Type, *IsShort, *ShortValue, *LongValue);
+  if (!Shifted)
+    return std::unexpected(Shifted.error());
+  return buildSelectBefore(Builder, Anchor, *Type, *IsZero, Low, *Shifted);
+}
+
+[[nodiscard]] Expected<std::vector<Register>>
+emitThreeLimbLeftShift(MachineIRBuilder &Builder, MachineInstr &Anchor,
+                       std::span<const Register> Parts, Register Amount) {
+  if (Parts.size() != 3)
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "RISC-V three-limb shift requires exactly three limbs");
+  auto Type = LLT::makeAnyScalar(64);
+  if (!Type)
+    return std::unexpected(Type.error());
+
+  auto OuterBoundary = buildIntegerConstantBefore(Builder, Anchor, 64, 128);
+  if (!OuterBoundary)
+    return std::unexpected(OuterBoundary.error());
+  auto OuterExcess =
+      buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, Amount, *OuterBoundary);
+  if (!OuterExcess)
+    return std::unexpected(OuterExcess.error());
+  auto OuterLack =
+      buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, *OuterBoundary, Amount);
+  if (!OuterLack)
+    return std::unexpected(OuterLack.error());
+  auto Zero = buildIntegerConstantBefore(Builder, Anchor, 64, 0);
+  if (!Zero)
+    return std::unexpected(Zero.error());
+  auto OuterIsShort = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan,
+                                                Amount, *OuterBoundary);
+  if (!OuterIsShort)
+    return std::unexpected(OuterIsShort.error());
+  auto OuterIsZero =
+      buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::Equal, Amount, *Zero);
+  if (!OuterIsZero)
+    return std::unexpected(OuterIsZero.error());
+
+  auto InnerBoundary = buildIntegerConstantBefore(Builder, Anchor, 64, 64);
+  if (!InnerBoundary)
+    return std::unexpected(InnerBoundary.error());
+  auto InnerExcess =
+      buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, Amount, *InnerBoundary);
+  if (!InnerExcess)
+    return std::unexpected(InnerExcess.error());
+  auto InnerLack =
+      buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, *InnerBoundary, Amount);
+  if (!InnerLack)
+    return std::unexpected(InnerLack.error());
+  auto InnerIsShort = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan,
+                                                Amount, *InnerBoundary);
+  if (!InnerIsShort)
+    return std::unexpected(InnerIsShort.error());
+  auto InnerIsZero =
+      buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::Equal, Amount, *Zero);
+  if (!InnerIsZero)
+    return std::unexpected(InnerIsZero.error());
+  auto Lower = emitTwoLimbLeftShiftBody(Builder, Anchor, Parts[0], Parts[1], Amount, *InnerExcess,
+                                        *InnerLack, *Zero, *InnerIsShort, *InnerIsZero);
+  if (!Lower)
+    return std::unexpected(Lower.error());
+
+  auto Cross = emitLowLimbOfTwoLimbLogicalRightShift(Builder, Anchor, Parts[0], Parts[1],
+                                                     *OuterLack, *InnerBoundary, *Zero);
+  if (!Cross)
+    return std::unexpected(Cross.error());
+  auto HighIsShort = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan,
+                                               Amount, *InnerBoundary);
+  if (!HighIsShort)
+    return std::unexpected(HighIsShort.error());
+  auto HighShort =
+      buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SHL, *Type, Parts[2], Amount);
+  if (!HighShort)
+    return std::unexpected(HighShort.error());
+  auto BoundedHigh = buildSelectBefore(Builder, Anchor, *Type, *HighIsShort, *HighShort, *Zero);
+  if (!BoundedHigh)
+    return std::unexpected(BoundedHigh.error());
+  auto ShortHigh =
+      buildBinaryBefore(Builder, Anchor, GenericOpcode::G_OR, *Type, *Cross, *BoundedHigh);
+  if (!ShortHigh)
+    return std::unexpected(ShortHigh.error());
+
+  auto LongIsShort = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan,
+                                               *OuterExcess, *InnerBoundary);
+  if (!LongIsShort)
+    return std::unexpected(LongIsShort.error());
+  auto LongShift =
+      buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SHL, *Type, Parts[0], *OuterExcess);
+  if (!LongShift)
+    return std::unexpected(LongShift.error());
+  auto LongHigh = buildSelectBefore(Builder, Anchor, *Type, *LongIsShort, *LongShift, *Zero);
+  if (!LongHigh)
+    return std::unexpected(LongHigh.error());
+
+  auto ResultLow = buildSelectBefore(Builder, Anchor, *Type, *OuterIsShort, Lower->Low, *Zero);
+  if (!ResultLow)
+    return std::unexpected(ResultLow.error());
+  auto ResultMiddle = buildSelectBefore(Builder, Anchor, *Type, *OuterIsShort, Lower->High, *Zero);
+  if (!ResultMiddle)
+    return std::unexpected(ResultMiddle.error());
+  auto ShiftedHigh =
+      buildSelectBefore(Builder, Anchor, *Type, *OuterIsShort, *ShortHigh, *LongHigh);
+  if (!ShiftedHigh)
+    return std::unexpected(ShiftedHigh.error());
+  auto ResultHigh = buildSelectBefore(Builder, Anchor, *Type, *OuterIsZero, Parts[2], *ShiftedHigh);
+  if (!ResultHigh)
+    return std::unexpected(ResultHigh.error());
+  return std::vector<Register>{*ResultLow, *ResultMiddle, *ResultHigh};
+}
+
+[[nodiscard]] Expected<void> lowerWideShiftLeft(MachineInstr &Instruction,
+                                                MachineFunction &Function,
+                                                const TargetDescription &Target) {
+  const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+  const MachineOperand *ValueOperand = Instruction.operandAt(1);
+  const MachineOperand *AmountOperand = Instruction.operandAt(2);
+  const Register *Destination =
+      DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+  const Register *Value = ValueOperand == nullptr ? nullptr : ValueOperand->getIf<Register>();
+  const Register *Amount = AmountOperand == nullptr ? nullptr : AmountOperand->getIf<Register>();
+  if (Destination == nullptr || Value == nullptr || Amount == nullptr)
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "RISC-V wide shift has invalid register operands");
+  auto ValueParts = collectWideValueParts(*Value, Function);
+  if (!ValueParts)
+    return std::unexpected(ValueParts.error());
+  auto AmountParts = collectWideValueParts(*Amount, Function);
+  if (!AmountParts)
+    return std::unexpected(AmountParts.error());
+  if (ValueParts->Parts.size() < 2 || AmountParts->Parts.empty())
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "RISC-V wide shift has an invalid limb decomposition");
+  auto ResultPlan = wideResultPlan(*Destination, Function, ValueParts->Parts.size());
+  if (!ResultPlan)
+    return std::unexpected(ResultPlan.error());
+
+  MachineIRBuilder Builder(Function, Target);
+  Expected<std::vector<Register>> Results =
+      ValueParts->Parts.size() == 2
+          ? emitTwoLimbLeftShift(Builder, Instruction, ValueParts->Parts[0], ValueParts->Parts[1],
+                                 AmountParts->Parts[0])
+                .transform([](TwoLimbShiftResult Result) {
+                  return std::vector<Register>{Result.Low, Result.High};
+                })
+      : ValueParts->Parts.size() == 3
+          ? emitThreeLimbLeftShift(Builder, Instruction, ValueParts->Parts, AmountParts->Parts[0])
+          : Expected<std::vector<Register>>(std::unexpected(
+                Error(ErrorCode::Unsupported,
+                      "RISC-V wide shift currently requires two or three 64-bit limbs")));
+  if (!Results)
+    return std::unexpected(Results.error());
+  for (std::size_t Index = 0; Index < Results->size(); ++Index) {
+    auto Replaced = replaceRegisterUses(Function, ResultPlan->Outputs[Index], (*Results)[Index]);
+    if (!Replaced)
+      return std::unexpected(Replaced.error());
+  }
+  if (auto Erased = eraseInstruction(*ResultPlan->Unmerge); !Erased)
+    return std::unexpected(Erased.error());
+  if (ResultPlan->Extension != nullptr) {
+    if (auto Erased = eraseInstruction(*ResultPlan->Extension); !Erased)
+      return std::unexpected(Erased.error());
+  }
+  if (auto Erased = eraseInstruction(Instruction); !Erased)
+    return std::unexpected(Erased.error());
+  for (std::size_t Index = 1; Index < AmountParts->Parts.size(); ++Index) {
+    const MachineOperand *Definition =
+        Function.registerInfo().getUniqueVRegDef(AmountParts->Parts[Index]);
+    if (Definition != nullptr && Definition->parent() != nullptr)
+      AmountParts->Artifacts.push_back(const_cast<MachineInstr *>(Definition->parent()));
+  }
+  ValueParts->Artifacts.insert(ValueParts->Artifacts.end(), AmountParts->Artifacts.begin(),
+                               AmountParts->Artifacts.end());
+  eraseDeadArtifacts(ValueParts->Artifacts, Function);
+  return {};
+}
+
+[[nodiscard]] Expected<std::vector<Register>> emitWideAdd(MachineIRBuilder &Builder,
+                                                          MachineInstr &Anchor,
+                                                          std::span<const Register> Left,
+                                                          std::span<const Register> Right) {
+  if (Left.size() != Right.size() || Left.size() < 2)
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "RISC-V wide add requires matching multi-limb operands");
+  auto Type = LLT::makeAnyScalar(64);
+  if (!Type)
+    return std::unexpected(Type.error());
+  std::vector<Register> Results;
+  Results.reserve(Left.size());
+  auto Low = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_ADD, *Type, Left[0], Right[0]);
+  if (!Low)
+    return std::unexpected(Low.error());
+  auto Carry = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan, *Low,
+                                         Right[0]);
+  if (!Carry)
+    return std::unexpected(Carry.error());
+  auto LowCopy = buildUnaryBefore(Builder, Anchor, GenericOpcode::COPY, *Type, *Low);
+  if (!LowCopy)
+    return std::unexpected(LowCopy.error());
+  Results.push_back(*LowCopy);
+  for (std::size_t Index = 1; Index < Left.size(); ++Index) {
+    auto Base =
+        buildBinaryBefore(Builder, Anchor, GenericOpcode::G_ADD, *Type, Left[Index], Right[Index]);
+    if (!Base)
+      return std::unexpected(Base.error());
+    std::optional<Register> BaseCarry;
+    if (Index + 1U < Left.size()) {
+      auto Compared = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan,
+                                                *Base, Left[Index]);
+      if (!Compared)
+        return std::unexpected(Compared.error());
+      BaseCarry = *Compared;
+    }
+    auto Sum = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_ADD, *Type, *Base, *Carry);
+    if (!Sum)
+      return std::unexpected(Sum.error());
+    if (Index + 1U < Left.size()) {
+      auto Zero = buildIntegerConstantBefore(Builder, Anchor, 64, 0);
+      if (!Zero)
+        return std::unexpected(Zero.error());
+      auto IsZero =
+          buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::Equal, *Sum, *Zero);
+      if (!IsZero)
+        return std::unexpected(IsZero.error());
+      auto CarryFromIncoming =
+          buildBinaryBefore(Builder, Anchor, GenericOpcode::G_AND, *Type, *IsZero, *Carry);
+      if (!CarryFromIncoming)
+        return std::unexpected(CarryFromIncoming.error());
+      auto NextCarry = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_OR, *Type, *BaseCarry,
+                                         *CarryFromIncoming);
+      if (!NextCarry)
+        return std::unexpected(NextCarry.error());
+      Carry = *NextCarry;
+    }
+    auto Copy = buildUnaryBefore(Builder, Anchor, GenericOpcode::COPY, *Type, *Sum);
+    if (!Copy)
+      return std::unexpected(Copy.error());
+    Results.push_back(*Copy);
+  }
+  return Results;
+}
+
+[[nodiscard]] Expected<std::vector<Register>> emitWideSub(MachineIRBuilder &Builder,
+                                                          MachineInstr &Anchor,
+                                                          std::span<const Register> Left,
+                                                          std::span<const Register> Right) {
+  if (Left.size() != Right.size() || Left.size() < 2)
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "RISC-V wide sub requires matching multi-limb operands");
+  auto Type = LLT::makeAnyScalar(64);
+  if (!Type)
+    return std::unexpected(Type.error());
+  std::vector<Register> Results;
+  Results.reserve(Left.size());
+  auto Low = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, Left[0], Right[0]);
+  if (!Low)
+    return std::unexpected(Low.error());
+  Results.push_back(*Low);
+  auto Borrow = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan,
+                                          Left[0], Right[0]);
+  if (!Borrow)
+    return std::unexpected(Borrow.error());
+  for (std::size_t Index = 1; Index < Left.size(); ++Index) {
+    auto Base =
+        buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, Left[Index], Right[Index]);
+    if (!Base)
+      return std::unexpected(Base.error());
+    std::optional<Register> BaseBorrow;
+    if (Index + 1U < Left.size()) {
+      auto Compared = buildIntegerCompareBefore(
+          Builder, Anchor, IntegerPredicate::UnsignedGreaterThan, *Base, Left[Index]);
+      if (!Compared)
+        return std::unexpected(Compared.error());
+      BaseBorrow = *Compared;
+    }
+    auto Difference =
+        buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, *Base, *Borrow);
+    if (!Difference)
+      return std::unexpected(Difference.error());
+    Results.push_back(*Difference);
+    if (Index + 1U < Left.size()) {
+      auto Zero = buildIntegerConstantBefore(Builder, Anchor, 64, 0);
+      if (!Zero)
+        return std::unexpected(Zero.error());
+      auto IsZero =
+          buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::Equal, *Base, *Zero);
+      if (!IsZero)
+        return std::unexpected(IsZero.error());
+      auto BorrowFromIncoming =
+          buildBinaryBefore(Builder, Anchor, GenericOpcode::G_AND, *Type, *IsZero, *Borrow);
+      if (!BorrowFromIncoming)
+        return std::unexpected(BorrowFromIncoming.error());
+      auto NextBorrow = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_OR, *Type, *BaseBorrow,
+                                          *BorrowFromIncoming);
+      if (!NextBorrow)
+        return std::unexpected(NextBorrow.error());
+      Borrow = *NextBorrow;
+    }
+  }
+  return Results;
+}
+
+[[nodiscard]] Expected<std::vector<Register>>
+emitWideBitwise(MachineIRBuilder &Builder, MachineInstr &Anchor, GenericOpcode Generic,
+                std::span<const Register> Left, std::span<const Register> Right) {
+  if (Left.size() != Right.size() || Left.size() < 2)
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "RISC-V wide bitwise operation requires matching multi-limb operands");
+  auto Type = LLT::makeAnyScalar(64);
+  if (!Type)
+    return std::unexpected(Type.error());
+  std::vector<Register> Results;
+  Results.reserve(Left.size());
+  for (std::size_t Index = 0; Index < Left.size(); ++Index) {
+    auto Part = buildBinaryBefore(Builder, Anchor, Generic, *Type, Left[Index], Right[Index]);
+    if (!Part)
+      return std::unexpected(Part.error());
+    Results.push_back(*Part);
+  }
+  return Results;
+}
+
+[[nodiscard]] Expected<void> lowerWideBinary(MachineInstr &Instruction, MachineFunction &Function,
+                                             const TargetDescription &Target,
+                                             GenericOpcode Generic) {
+  const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+  const MachineOperand *LeftOperand = Instruction.operandAt(1);
+  const MachineOperand *RightOperand = Instruction.operandAt(2);
+  const Register *Destination =
+      DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+  const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf<Register>();
+  const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf<Register>();
+  if (Destination == nullptr || Left == nullptr || Right == nullptr)
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "RISC-V wide binary instruction has invalid operands");
+  auto LeftParts = collectWideValueParts(*Left, Function);
+  if (!LeftParts)
+    return std::unexpected(LeftParts.error());
+  auto RightParts = collectWideValueParts(*Right, Function);
+  if (!RightParts)
+    return std::unexpected(RightParts.error());
+  if (LeftParts->Parts.size() != RightParts->Parts.size())
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "RISC-V wide binary operands have different limb counts");
+  auto ResultPlan = wideResultPlan(*Destination, Function, LeftParts->Parts.size());
+  if (!ResultPlan)
+    return std::unexpected(ResultPlan.error());
+
+  MachineIRBuilder Builder(Function, Target);
+  Expected<std::vector<Register>> Results =
+      Generic == GenericOpcode::G_ADD
+          ? emitWideAdd(Builder, Instruction, LeftParts->Parts, RightParts->Parts)
+      : Generic == GenericOpcode::G_SUB
+          ? emitWideSub(Builder, Instruction, LeftParts->Parts, RightParts->Parts)
+          : emitWideBitwise(Builder, Instruction, Generic, LeftParts->Parts, RightParts->Parts);
+  if (!Results)
+    return std::unexpected(Results.error());
+  for (std::size_t Index = 0; Index < Results->size(); ++Index) {
+    auto Replaced = replaceRegisterUses(Function, ResultPlan->Outputs[Index], (*Results)[Index]);
+    if (!Replaced)
+      return std::unexpected(Replaced.error());
+  }
+  if (auto Erased = eraseInstruction(*ResultPlan->Unmerge); !Erased)
+    return std::unexpected(Erased.error());
+  if (ResultPlan->Extension != nullptr) {
+    if (auto Erased = eraseInstruction(*ResultPlan->Extension); !Erased)
+      return std::unexpected(Erased.error());
+  }
+  if (auto Erased = eraseInstruction(Instruction); !Erased)
+    return std::unexpected(Erased.error());
+  LeftParts->Artifacts.insert(LeftParts->Artifacts.end(), RightParts->Artifacts.begin(),
+                              RightParts->Artifacts.end());
+  eraseDeadArtifacts(LeftParts->Artifacts, Function);
+  return {};
+}
+
+[[nodiscard]] bool canLowerNarrowBranchCondition(const MachineInstr &Instruction,
+                                                 const MachineFunction &Function) {
+  if (Instruction.opcode() != opcode(GenericOpcode::G_BRCOND) || Instruction.operandCount() < 2)
+    return false;
+  const MachineOperand *ConditionOperand = Instruction.operandAt(0);
+  const Register *Condition =
+      ConditionOperand == nullptr ? nullptr : ConditionOperand->getIf<Register>();
+  if (Condition == nullptr || !Condition->isVirtual() ||
+      singleRegisterUse(Function, *Condition) != &Instruction)
+    return false;
+  const auto ConditionInfo = Function.registerInfo().virtualRegisterInfo(*Condition);
+  const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(*Condition);
+  const MachineInstr *Truncation = Definition == nullptr ? nullptr : Definition->parent();
+  if (!ConditionInfo || !ConditionInfo->get().Type ||
+      ConditionInfo->get().Type->scalarBitWidth() != 1 || Truncation == nullptr ||
+      Truncation->opcode() != opcode(GenericOpcode::G_TRUNC) || Truncation->operandCount() != 2)
+    return false;
+  return operandType(Function, *Truncation, 1)
+      .transform([](LLT Type) { return isScalarGPRType(Type) && Type.scalarBitWidth() == 64; })
+      .value_or(false);
+}
+
+[[nodiscard]] Expected<void> lowerNarrowBranchCondition(MachineInstr &Instruction,
+                                                        MachineFunction &Function,
+                                                        const TargetDescription &Target) {
+  if (!canLowerNarrowBranchCondition(Instruction, Function))
+    return makeUnexpected(ErrorCode::Unsupported,
+                          "RISC-V branch condition is not a narrow truncation");
+  const MachineOperand *ConditionOperand = Instruction.operandAt(0);
+  const Register Condition = *ConditionOperand->getIf<Register>();
+  const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(Condition);
+  MachineInstr *Truncation = const_cast<MachineInstr *>(Definition->parent());
+  const Register Source = *Truncation->operandAt(1)->getIf<Register>();
+  auto Type = LLT::makeAnyScalar(64);
+  if (!Type)
+    return std::unexpected(Type.error());
+  MachineIRBuilder Builder(Function, Target);
+  auto One = buildIntegerConstantBefore(Builder, Instruction, 64, 1);
+  if (!One)
+    return std::unexpected(One.error());
+  auto Masked = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_AND, *Type, Source, *One);
+  if (!Masked)
+    return std::unexpected(Masked.error());
+  auto Replaced = Instruction.operandAt(0)->setRegister(*Masked);
+  if (!Replaced)
+    return std::unexpected(Replaced.error());
+  if (Function.registerInfo().useEmpty(Condition))
+    return eraseInstruction(*Truncation);
+  return {};
+}
+
+struct NarrowPhiIncoming final {
+  Register Value;
+  MachineBasicBlock *Block = nullptr;
+  MachineInstr *ReusableTruncation = nullptr;
+};
+
+[[nodiscard]] MachineInstr *firstInstructionAfterLeadingCopies(MachineBasicBlock &Block) {
+  MachineInstr *Instruction = Block.firstInstruction();
+  while (Instruction != nullptr && Instruction->opcode() == opcode(GenericOpcode::COPY))
+    Instruction = Instruction->next();
+  return Instruction;
+}
+
+[[nodiscard]] Expected<Register>
+buildIncomingConversion(MachineFunction &Function, const TargetDescription &Target,
+                        MachineBasicBlock &Block, MachineInstr *Anchor, GenericOpcode Generic,
+                        LLT Type, Register Source) {
+  auto Result = Function.registerInfo().createVirtualRegister(Type);
+  if (!Result)
+    return std::unexpected(Result.error());
+  MachineIRBuilder Builder(Function, Target);
+  auto Point =
+      Anchor == nullptr ? Builder.setInsertPoint(Block) : Builder.setInsertPointBefore(*Anchor);
+  if (!Point)
+    return std::unexpected(Point.error());
+  auto Pending = Builder.buildInstr(opcode(Generic));
+  if (!Pending)
+    return std::unexpected(Pending.error());
+  if (auto Added = Pending->addDef(*Result); !Added)
+    return std::unexpected(Added.error());
+  if (auto Added = Pending->addUse(Source); !Added)
+    return std::unexpected(Added.error());
+  auto Built = Pending->build();
+  if (!Built)
+    return std::unexpected(Built.error());
+  return *Result;
+}
+
+[[nodiscard]] Expected<void> lowerNarrowPhi(MachineInstr &Instruction, MachineFunction &Function,
+                                            const TargetDescription &Target) {
+  const std::optional<LLT> OriginalType = resultType(Function, Instruction);
+  const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+  const Register *Destination =
+      DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+  if (!OriginalType || Destination == nullptr || OriginalType->scalarBitWidth() >= 32 ||
+      Instruction.operandCount() < 3 || (Instruction.operandCount() % 2U) == 0)
+    return makeUnexpected(ErrorCode::VerificationError, "RISC-V narrow PHI has invalid operands");
+  auto LegalType = LLT::makeAnyScalar(32);
+  if (!LegalType)
+    return std::unexpected(LegalType.error());
+
+  std::vector<NarrowPhiIncoming> Incoming;
+  Incoming.reserve((Instruction.operandCount() - 1U) / 2U);
+  for (std::size_t Index = 1; Index < Instruction.operandCount(); Index += 2) {
+    const MachineOperand *ValueOperand = Instruction.operandAt(Index);
+    const Register *Value = ValueOperand == nullptr ? nullptr : ValueOperand->getIf<Register>();
+    const MachineOperand *BlockOperand = Instruction.operandAt(Index + 1U);
+    const auto *Block =
+        BlockOperand == nullptr ? nullptr : BlockOperand->getIf<BasicBlockOperand>();
+    if (Value == nullptr || Block == nullptr || Block->Block == nullptr)
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "RISC-V narrow PHI has an invalid incoming edge");
+    MachineInstr *Reusable = nullptr;
+    const MachineOperand *Definition =
+        Value->isVirtual() ? Function.registerInfo().getUniqueVRegDef(*Value) : nullptr;
+    MachineInstr *Def =
+        Definition == nullptr ? nullptr : const_cast<MachineInstr *>(Definition->parent());
+    if (Def != nullptr && Def->opcode() == opcode(GenericOpcode::G_TRUNC) &&
+        Def->operandCount() == 2 && singleRegisterUse(Function, *Value) == &Instruction &&
+        operandType(Function, *Def, 1)
+            .transform([](LLT Type) { return Type.scalarBitWidth() >= 32; })
+            .value_or(false)) {
+      Reusable = Def;
+    }
+    Incoming.push_back(
+        NarrowPhiIncoming{.Value = *Value, .Block = Block->Block, .ReusableTruncation = Reusable});
+  }
+  for (NarrowPhiIncoming &Entry : Incoming) {
+    if (Entry.ReusableTruncation == nullptr)
+      continue;
+    const std::size_t Count = static_cast<std::size_t>(std::ranges::count(
+        Incoming, Entry.ReusableTruncation, &NarrowPhiIncoming::ReusableTruncation));
+    if (Count != 1)
+      Entry.ReusableTruncation = nullptr;
+  }
+  for (NarrowPhiIncoming &Entry : Incoming) {
+    if (Entry.ReusableTruncation == nullptr)
+      continue;
+    MachineBasicBlock *OriginalBlock = Entry.ReusableTruncation->parent();
+    if (OriginalBlock == nullptr)
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "RISC-V narrow PHI input truncation is detached");
+    auto Removed = OriginalBlock->removeInstruction(*Entry.ReusableTruncation);
+    if (!Removed)
+      return std::unexpected(Removed.error());
+  }
+
+  std::vector<std::pair<MachineBasicBlock *, MachineInstr *>> Anchors;
+  for (const NarrowPhiIncoming &Entry : Incoming) {
+    if (std::ranges::find(Anchors, Entry.Block, &decltype(Anchors)::value_type::first) ==
+        Anchors.end()) {
+      Anchors.emplace_back(Entry.Block, firstInstructionAfterLeadingCopies(*Entry.Block));
+    }
+  }
+  std::vector<Register> LegalIncoming;
+  LegalIncoming.reserve(Incoming.size());
+  for (NarrowPhiIncoming &Entry : Incoming) {
+    const auto AnchorPosition =
+        std::ranges::find(Anchors, Entry.Block, &decltype(Anchors)::value_type::first);
+    MachineInstr *Anchor = AnchorPosition->second;
+    if (Entry.ReusableTruncation != nullptr) {
+      auto SetType = Function.registerInfo().setType(Entry.Value, *LegalType);
+      if (!SetType)
+        return std::unexpected(SetType.error());
+      auto Inserted =
+          Anchor == nullptr
+              ? Entry.Block->appendInstruction(*Entry.ReusableTruncation)
+              : Entry.Block->insertInstructionBefore(*Anchor, *Entry.ReusableTruncation);
+      if (!Inserted)
+        return std::unexpected(Inserted.error());
+      LegalIncoming.push_back(Entry.Value);
+      continue;
+    }
+    const auto ValueInfo = Function.registerInfo().virtualRegisterInfo(Entry.Value);
+    if (!ValueInfo || !ValueInfo->get().Type)
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "RISC-V narrow PHI input has no scalar type");
+    const std::uint32_t Width = ValueInfo->get().Type->scalarBitWidth();
+    const GenericOpcode Conversion = Width < 32   ? GenericOpcode::G_ANYEXT
+                                     : Width > 32 ? GenericOpcode::G_TRUNC
+                                                  : GenericOpcode::COPY;
+    auto Converted = buildIncomingConversion(Function, Target, *Entry.Block, Anchor, Conversion,
+                                             *LegalType, Entry.Value);
+    if (!Converted)
+      return std::unexpected(Converted.error());
+    LegalIncoming.push_back(*Converted);
+  }
+
+  auto LegalResult = Function.registerInfo().createVirtualRegister(*LegalType);
+  if (!LegalResult)
+    return std::unexpected(LegalResult.error());
+  MachineIRBuilder Builder(Function, Target);
+  auto Point = Builder.setInsertPointBefore(Instruction);
+  if (!Point)
+    return std::unexpected(Point.error());
+  auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_PHI));
+  if (!Pending)
+    return std::unexpected(Pending.error());
+  if (auto Added = Pending->addDef(*LegalResult); !Added)
+    return std::unexpected(Added.error());
+  for (std::size_t Index = 0; Index < Incoming.size(); ++Index) {
+    if (auto Added = Pending->addUse(LegalIncoming[Index]); !Added)
+      return std::unexpected(Added.error());
+    if (auto Added = Pending->addBasicBlock(*Incoming[Index].Block); !Added)
+      return std::unexpected(Added.error());
+  }
+  auto Built = Pending->build();
+  if (!Built)
+    return std::unexpected(Built.error());
+  auto Replaced = replaceRegisterUses(Function, *Destination, *LegalResult);
+  if (!Replaced)
+    return std::unexpected(Replaced.error());
+  return eraseInstruction(Instruction);
+}
+
+[[nodiscard]] Expected<void> lowerWidePhi(MachineInstr &Instruction, MachineFunction &Function,
+                                          const TargetDescription &Target) {
+  const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+  const Register *Destination =
+      DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+  if (Destination == nullptr || Instruction.operandCount() < 5 ||
+      (Instruction.operandCount() % 2U) == 0)
+    return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide PHI has invalid operands");
+
+  std::vector<WideValueParts> IncomingParts;
+  std::vector<MachineBasicBlock *> IncomingBlocks;
+  for (std::size_t Index = 1; Index < Instruction.operandCount(); Index += 2) {
+    const MachineOperand *ValueOperand = Instruction.operandAt(Index);
+    const Register *Value = ValueOperand == nullptr ? nullptr : ValueOperand->getIf<Register>();
+    const MachineOperand *BlockOperand = Instruction.operandAt(Index + 1U);
+    const auto *Block =
+        BlockOperand == nullptr ? nullptr : BlockOperand->getIf<BasicBlockOperand>();
+    if (Value == nullptr || Block == nullptr || Block->Block == nullptr)
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "RISC-V wide PHI has an invalid incoming edge");
+    auto Parts = collectWideValueParts(*Value, Function);
+    if (!Parts)
+      return std::unexpected(Parts.error());
+    if (!IncomingParts.empty() && Parts->Parts.size() != IncomingParts.front().Parts.size())
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "RISC-V wide PHI incoming values have different limb counts");
+    IncomingParts.push_back(std::move(*Parts));
+    IncomingBlocks.push_back(Block->Block);
+  }
+  const std::size_t PartCount = IncomingParts.front().Parts.size();
+  auto ResultPlan = wideResultPlan(*Destination, Function, PartCount);
+  if (!ResultPlan)
+    return std::unexpected(ResultPlan.error());
+  auto Type = LLT::makeAnyScalar(64);
+  if (!Type)
+    return std::unexpected(Type.error());
+  MachineIRBuilder Builder(Function, Target);
+  std::vector<Register> Results;
+  Results.reserve(PartCount);
+  for (std::size_t PartIndex = 0; PartIndex < PartCount; ++PartIndex) {
+    auto Result = Function.registerInfo().createVirtualRegister(*Type);
+    if (!Result)
+      return std::unexpected(Result.error());
+    auto Point = Builder.setInsertPointBefore(Instruction);
+    if (!Point)
+      return std::unexpected(Point.error());
+    auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_PHI));
+    if (!Pending)
+      return std::unexpected(Pending.error());
+    if (auto Added = Pending->addDef(*Result); !Added)
+      return std::unexpected(Added.error());
+    for (std::size_t IncomingIndex = 0; IncomingIndex < IncomingParts.size(); ++IncomingIndex) {
+      if (auto Added = Pending->addUse(IncomingParts[IncomingIndex].Parts[PartIndex]); !Added)
+        return std::unexpected(Added.error());
+      if (auto Added = Pending->addBasicBlock(*IncomingBlocks[IncomingIndex]); !Added)
+        return std::unexpected(Added.error());
+    }
+    auto Built = Pending->build();
+    if (!Built)
+      return std::unexpected(Built.error());
+    Results.push_back(*Result);
+  }
+  for (std::size_t Index = 0; Index < PartCount; ++Index) {
+    auto Replaced = replaceRegisterUses(Function, ResultPlan->Outputs[Index], Results[Index]);
+    if (!Replaced)
+      return std::unexpected(Replaced.error());
+  }
+  if (auto Erased = eraseInstruction(*ResultPlan->Unmerge); !Erased)
+    return std::unexpected(Erased.error());
+  if (ResultPlan->Extension != nullptr) {
+    if (auto Erased = eraseInstruction(*ResultPlan->Extension); !Erased)
+      return std::unexpected(Erased.error());
+  }
+  if (auto Erased = eraseInstruction(Instruction); !Erased)
+    return std::unexpected(Erased.error());
+  std::vector<MachineInstr *> Artifacts;
+  for (WideValueParts &Parts : IncomingParts)
+    Artifacts.insert(Artifacts.end(), Parts.Artifacts.begin(), Parts.Artifacts.end());
+  eraseDeadArtifacts(Artifacts, Function);
+  return {};
+}
+
+[[nodiscard]] bool canFoldWideMergeImmediately(const MachineInstr &Instruction,
+                                               const MachineFunction &Function) {
+  const MachineOperand *LowOperand = Instruction.operandAt(1);
+  const Register *Low = LowOperand == nullptr ? nullptr : LowOperand->getIf<Register>();
+  if (Low == nullptr)
+    return false;
+  const auto LowInfo = Function.registerInfo().virtualRegisterInfo(*Low);
+  if (!LowInfo || !LowInfo->get().Type)
+    return false;
+  const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+  const Register *Destination =
+      DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+  if (Destination == nullptr)
+    return false;
+  auto &RegisterInfo = const_cast<MachineRegisterInfo &>(Function.registerInfo());
+  for (MachineOperand &Use : RegisterInfo.useOperands(*Destination)) {
+    MachineInstr *Trunc = Use.parent();
+    if (Trunc == nullptr || Trunc->opcode() != opcode(GenericOpcode::G_TRUNC) ||
+        Trunc->operandAt(1) != &Use)
+      return false;
+    const std::optional<LLT> Result = resultType(Function, *Trunc);
+    if (!Result || Result->scalarBitWidth() > LowInfo->get().Type->scalarBitWidth())
+      return false;
+  }
+  return true;
+}
+
+struct RV64ScalarLayout final {
+  std::uint64_t StoreSize = 0;
+  std::uint64_t AllocationSize = 0;
+  std::uint64_t ABIAlignment = 0;
+};
+
+[[nodiscard]] std::optional<RV64ScalarLayout> rv64ScalarLayout(LLT Type) noexcept {
+  if (Type.isPointer()) {
+    return Type.addressSpace() == 0 && Type.scalarBitWidth() == 64
+               ? std::optional<RV64ScalarLayout>{{8, 8, 8}}
+               : std::nullopt;
+  }
+  if ((!isSupportedIntegerType(Type) && !isRVFloatType(Type)) || Type.scalarBitWidth() == 0) {
+    return std::nullopt;
+  }
+  const std::uint64_t StoreBytes = (Type.scalarBitWidth() + 7U) / 8U;
+  const std::uint64_t AllocationBytes = std::bit_ceil(StoreBytes);
+  return RV64ScalarLayout{StoreBytes, AllocationBytes, std::min<std::uint64_t>(AllocationBytes, 8)};
+}
+
+[[nodiscard]] Expected<const MachineMemOperand *>
+createMemoryAccess(MachineFunction &Function, MemoryAccessFlag Access, LLT Type,
+                   std::uint64_t SizeBytes, std::uint64_t AlignmentBytes) {
+  auto Size = MemorySize::fixed(SizeBytes);
+  auto Align = Alignment::create(AlignmentBytes);
+  if (!Size || !Align)
+    return makeUnexpected(ErrorCode::InternalError, "RISC-V memory facts are invalid");
+  MachineMemOperandDesc Description;
+  Description.Accesses = {Access};
+  Description.Size = *Size;
+  Description.MemoryType = Type;
+  Description.Align = *Align;
+  return Function.createMemOperand(std::move(Description));
+}
+
+struct VAArgPlan final {
+  Register Destination;
+  Register ListPointer;
+  LLT DestinationType;
+  LLT PointerType;
+  std::uint64_t Alignment = 0;
+  RV64ScalarLayout ValueLayout;
+};
+
+[[nodiscard]] std::optional<VAArgPlan> vaargPlan(const MachineInstr &Instruction,
+                                                 const MachineFunction &Function) {
+  if (Instruction.opcode() != opcode(GenericOpcode::G_VAARG) || Instruction.operandCount() != 3 ||
+      !Instruction.memOperands().empty()) {
+    return std::nullopt;
+  }
+  const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+  const MachineOperand *ListOperand = Instruction.operandAt(1);
+  const MachineOperand *AlignmentOperand = Instruction.operandAt(2);
+  const Register *Destination =
+      DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+  const Register *ListPointer = ListOperand == nullptr ? nullptr : ListOperand->getIf<Register>();
+  const std::optional<LLT> DestinationType = resultType(Function, Instruction);
+  const std::optional<LLT> PointerType = operandType(Function, Instruction, 1);
+  const auto AlignmentValue = AlignmentOperand == nullptr
+                                  ? Expected<std::int64_t>(std::unexpected(
+                                        Error(ErrorCode::InvalidArgument, "missing alignment")))
+                                  : signedImmediate(*AlignmentOperand);
+  const std::optional<RV64ScalarLayout> ValueLayout =
+      DestinationType ? rv64ScalarLayout(*DestinationType) : std::nullopt;
+  if (Destination == nullptr || ListPointer == nullptr || !Destination->isVirtual() ||
+      !ListPointer->isVirtual() || !DestinationOperand->isDef() || !ListOperand->isUse() ||
+      !DestinationType || !PointerType || !PointerType->isPointer() ||
+      PointerType->addressSpace() != 0 || PointerType->scalarBitWidth() != 64 || !AlignmentValue ||
+      *AlignmentValue <= 0 || !std::has_single_bit(static_cast<std::uint64_t>(*AlignmentValue)) ||
+      !ValueLayout) {
+    return std::nullopt;
+  }
+
+  return VAArgPlan{*Destination,
+                   *ListPointer,
+                   *DestinationType,
+                   *PointerType,
+                   static_cast<std::uint64_t>(*AlignmentValue),
+                   *ValueLayout};
+}
+
+[[nodiscard]] Expected<void> lowerVAArg(MachineInstr &Instruction, MachineFunction &Function,
+                                        const TargetDescription &Target) {
+  const auto Plan = vaargPlan(Instruction, Function);
+  if (!Plan) {
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "RISC-V va_arg legalization requires a scalar result, p0 list, and "
+                          "power-of-two alignment");
+  }
+
+  auto ListLoadMemory =
+      createMemoryAccess(Function, MemoryAccessFlag::Load, Plan->PointerType, 8, 8);
+  if (!ListLoadMemory)
+    return std::unexpected(ListLoadMemory.error());
+  auto ListStoreMemory =
+      createMemoryAccess(Function, MemoryAccessFlag::Store, Plan->PointerType, 8, 8);
+  if (!ListStoreMemory)
+    return std::unexpected(ListStoreMemory.error());
+
+  auto CurrentList = Function.registerInfo().createVirtualRegister(Plan->PointerType);
+  if (!CurrentList)
+    return std::unexpected(CurrentList.error());
+  MachineIRBuilder Builder(Function, Target);
+  if (auto Point = Builder.setInsertPointBefore(Instruction); !Point)
+    return std::unexpected(Point.error());
+  auto ListLoad = Builder.buildInstr(opcode(GenericOpcode::G_LOAD));
+  if (!ListLoad)
+    return std::unexpected(ListLoad.error());
+  if (auto Added = ListLoad->addDef(*CurrentList); !Added)
+    return std::unexpected(Added.error());
+  if (auto Added = ListLoad->addUse(Plan->ListPointer); !Added)
+    return std::unexpected(Added.error());
+  if (auto Added = ListLoad->addMemOperand(**ListLoadMemory); !Added)
+    return std::unexpected(Added.error());
+  if (auto Built = ListLoad->build(); !Built)
+    return std::unexpected(Built.error());
+
+  Register ValueAddress = *CurrentList;
+  if (Plan->Alignment > 1) {
+    auto AlignmentOffset =
+        buildIntegerConstantBefore(Builder, Instruction, 64, Plan->Alignment - 1U);
+    if (!AlignmentOffset)
+      return std::unexpected(AlignmentOffset.error());
+    auto BiasedAddress = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_PTR_ADD,
+                                           Plan->PointerType, ValueAddress, *AlignmentOffset);
+    if (!BiasedAddress)
+      return std::unexpected(BiasedAddress.error());
+    auto AlignmentMask =
+        buildIntegerConstantBefore(Builder, Instruction, 64, ~(Plan->Alignment - std::uint64_t{1}));
+    if (!AlignmentMask)
+      return std::unexpected(AlignmentMask.error());
+    auto AlignedAddress = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_PTRMASK,
+                                            Plan->PointerType, *BiasedAddress, *AlignmentMask);
+    if (!AlignedAddress)
+      return std::unexpected(AlignedAddress.error());
+    ValueAddress = *AlignedAddress;
+  }
+
+  auto Increment =
+      buildIntegerConstantBefore(Builder, Instruction, 64, Plan->ValueLayout.AllocationSize);
+  if (!Increment)
+    return std::unexpected(Increment.error());
+  auto NextList = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_PTR_ADD,
+                                    Plan->PointerType, ValueAddress, *Increment);
+  if (!NextList)
+    return std::unexpected(NextList.error());
+  auto ListStore = Builder.buildInstr(opcode(GenericOpcode::G_STORE));
+  if (!ListStore)
+    return std::unexpected(ListStore.error());
+  if (auto Added = ListStore->addUse(*NextList); !Added)
+    return std::unexpected(Added.error());
+  if (auto Added = ListStore->addUse(Plan->ListPointer); !Added)
+    return std::unexpected(Added.error());
+  if (auto Added = ListStore->addMemOperand(**ListStoreMemory); !Added)
+    return std::unexpected(Added.error());
+  if (auto Built = ListStore->build(); !Built)
+    return std::unexpected(Built.error());
+
+  if (!Function.registerInfo().useEmpty(Plan->Destination)) {
+    auto ValueLoadMemory =
+        createMemoryAccess(Function, MemoryAccessFlag::Load, Plan->DestinationType,
+                           Plan->ValueLayout.StoreSize, Plan->ValueLayout.ABIAlignment);
+    if (!ValueLoadMemory)
+      return std::unexpected(ValueLoadMemory.error());
+    auto ValueLoad = Builder.buildInstr(opcode(GenericOpcode::G_LOAD));
+    if (!ValueLoad)
+      return std::unexpected(ValueLoad.error());
+    if (auto Added = ValueLoad->addDef(Plan->Destination); !Added)
+      return std::unexpected(Added.error());
+    if (auto Added = ValueLoad->addUse(ValueAddress); !Added)
+      return std::unexpected(Added.error());
+    if (auto Added = ValueLoad->addMemOperand(**ValueLoadMemory); !Added)
+      return std::unexpected(Added.error());
+    if (auto Built = ValueLoad->build(); !Built)
+      return std::unexpected(Built.error());
+  }
+  return eraseInstruction(Instruction);
+}
+
+struct VACopyPlan final {
+  Register Destination;
+  Register Source;
+  LLT PointerType;
+};
+
+[[nodiscard]] std::optional<VACopyPlan> vacopyPlan(const MachineInstr &Instruction,
+                                                   const MachineFunction &Function) {
+  if (Instruction.opcode() != opcode(GenericOpcode::G_INTRINSIC_W_SIDE_EFFECTS) ||
+      Instruction.operandCount() != 3 || !Instruction.memOperands().empty()) {
+    return std::nullopt;
+  }
+  const MachineOperand *IntrinsicIDOperand = Instruction.operandAt(0);
+  const MachineOperand *DestinationOperand = Instruction.operandAt(1);
+  const MachineOperand *SourceOperand = Instruction.operandAt(2);
+  const auto *Intrinsic =
+      IntrinsicIDOperand == nullptr ? nullptr : IntrinsicIDOperand->getIf<IntrinsicOperand>();
+  const Register *Destination =
+      DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+  const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>();
+  const std::optional<LLT> DestinationType = operandType(Function, Instruction, 1);
+  const std::optional<LLT> SourceType = operandType(Function, Instruction, 2);
+  if (Intrinsic == nullptr || Intrinsic->ID != intrinsicID(GenericIntrinsic::VACopy) ||
+      Destination == nullptr || Source == nullptr || !Destination->isVirtual() ||
+      !Source->isVirtual() || !DestinationOperand->isUse() || !SourceOperand->isUse() ||
+      !DestinationType || !SourceType || *DestinationType != *SourceType ||
+      !DestinationType->isPointer() || DestinationType->addressSpace() != 0 ||
+      DestinationType->scalarBitWidth() != 64) {
+    return std::nullopt;
+  }
+  return VACopyPlan{*Destination, *Source, *DestinationType};
+}
+
+[[nodiscard]] Expected<void> lowerVACopy(MachineInstr &Instruction, MachineFunction &Function,
+                                         const TargetDescription &Target) {
+  const auto Plan = vacopyPlan(Instruction, Function);
+  if (!Plan) {
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "RISC-V va_copy legalization requires two p0 operands");
+  }
+  auto Size = MemorySize::fixed(8);
+  auto Align = Alignment::create(8);
+  if (!Size || !Align)
+    return makeUnexpected(ErrorCode::InternalError, "RISC-V pointer memory facts are invalid");
+
+  MachineMemOperandDesc LoadDescription;
+  LoadDescription.Accesses = {MemoryAccessFlag::Load};
+  LoadDescription.Size = *Size;
+  LoadDescription.MemoryType = Plan->PointerType;
+  LoadDescription.Align = *Align;
+  auto LoadMemory = Function.createMemOperand(std::move(LoadDescription));
+  if (!LoadMemory)
+    return std::unexpected(LoadMemory.error());
+
+  MachineMemOperandDesc StoreDescription;
+  StoreDescription.Accesses = {MemoryAccessFlag::Store};
+  StoreDescription.Size = *Size;
+  StoreDescription.MemoryType = Plan->PointerType;
+  StoreDescription.Align = *Align;
+  auto StoreMemory = Function.createMemOperand(std::move(StoreDescription));
+  if (!StoreMemory)
+    return std::unexpected(StoreMemory.error());
+
+  auto Value = Function.registerInfo().createVirtualRegister(Plan->PointerType);
+  if (!Value)
+    return std::unexpected(Value.error());
+  MachineIRBuilder Builder(Function, Target);
+  if (auto Point = Builder.setInsertPointBefore(Instruction); !Point)
+    return std::unexpected(Point.error());
+
+  auto Load = Builder.buildInstr(opcode(GenericOpcode::G_LOAD));
+  if (!Load)
+    return std::unexpected(Load.error());
+  if (auto Added = Load->addDef(*Value); !Added)
+    return std::unexpected(Added.error());
+  if (auto Added = Load->addUse(Plan->Source); !Added)
+    return std::unexpected(Added.error());
+  if (auto Added = Load->addMemOperand(**LoadMemory); !Added)
+    return std::unexpected(Added.error());
+  if (auto Built = Load->build(); !Built)
+    return std::unexpected(Built.error());
+
+  auto Store = Builder.buildInstr(opcode(GenericOpcode::G_STORE));
+  if (!Store)
+    return std::unexpected(Store.error());
+  if (auto Added = Store->addUse(*Value); !Added)
+    return std::unexpected(Added.error());
+  if (auto Added = Store->addUse(Plan->Destination); !Added)
+    return std::unexpected(Added.error());
+  if (auto Added = Store->addMemOperand(**StoreMemory); !Added)
+    return std::unexpected(Added.error());
+  if (auto Built = Store->build(); !Built)
+    return std::unexpected(Built.error());
+  return eraseInstruction(Instruction);
+}
+
+} // namespace
+
+Expected<LegalizationAction> RISCVLegalizerInfo::classify(const MachineInstr &Instruction,
+                                                          const MachineFunction &Function,
+                                                          const TargetDescription &) const {
+  if (!Subtarget.features().has(RISCVExtension::I)) {
+    return makeUnexpected(ErrorCode::InternalError,
+                          "RISC-V legalizer requires the base I extension");
+  }
+  if (!isGenericOpcode(Instruction.opcode()))
+    return LegalizationAction::Legal;
+
+  const GenericOpcode Generic = static_cast<GenericOpcode>(Instruction.opcode().value());
+  const std::optional<LLT> Type = resultType(Function, Instruction);
+  switch (Generic) {
+  case GenericOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
+    return vacopyPlan(Instruction, Function) ? LegalizationAction::Custom
+                                             : LegalizationAction::Unsupported;
+  case GenericOpcode::G_VAARG:
+    return vaargPlan(Instruction, Function) ? LegalizationAction::Custom
+                                            : LegalizationAction::Unsupported;
+  case GenericOpcode::G_FRAME_INDEX:
+    return Type && Type->isPointer() && Type->addressSpace() == 0 && Type->scalarBitWidth() == 64
+               ? LegalizationAction::Legal
+               : LegalizationAction::Unsupported;
+  case GenericOpcode::G_BR:
+    return LegalizationAction::Legal;
+  case GenericOpcode::G_BRCOND: {
+    const std::optional<LLT> Condition = operandType(Function, Instruction, 0);
+    if (!Condition || !isScalarGPRType(*Condition))
+      return LegalizationAction::Unsupported;
+    return canLowerNarrowBranchCondition(Instruction, Function) ? LegalizationAction::Custom
+                                                                : LegalizationAction::Legal;
+  }
+  case GenericOpcode::G_PHI: {
+    if (!Type)
+      return LegalizationAction::Unsupported;
+    for (std::size_t Index = 1; Index < Instruction.operandCount(); Index += 2) {
+      const std::optional<LLT> Incoming = operandType(Function, Instruction, Index);
+      if (!Incoming || *Incoming != *Type)
+        return LegalizationAction::Unsupported;
+    }
+    if ((Type->isInteger() || Type->isAnyScalar()) && Type->scalarBitWidth() > 64)
+      return LegalizationAction::Custom;
+    if ((Type->isInteger() || Type->isAnyScalar()) && Type->scalarBitWidth() < 32)
+      return LegalizationAction::Custom;
+    if (!isSupportedIntegerType(*Type) && !isRVFloatType(*Type))
+      return LegalizationAction::Unsupported;
+    return LegalizationAction::Legal;
+  }
+  case GenericOpcode::G_IMPLICIT_DEF:
+    return Type && (isSupportedIntegerType(*Type) || isRVFloatType(*Type))
+               ? LegalizationAction::Legal
+               : LegalizationAction::Unsupported;
+  case GenericOpcode::G_STORE:
+    if (Instruction.operandCount() == 0)
+      return LegalizationAction::Unsupported;
+    if (const std::optional<LLT> ValueType = registerType(Function, *Instruction.operandAt(0));
+        ValueType) {
+      const std::optional<LLT> AddressType = operandType(Function, Instruction, 1);
+      if (!AddressType || !AddressType->isPointer() || AddressType->addressSpace() != 0 ||
+          AddressType->scalarBitWidth() != 64) {
+        return LegalizationAction::Unsupported;
+      }
+      if (Instruction.memOperands().size() == 1 && Instruction.memOperands().front()->isAtomic()) {
+        const std::uint64_t Bytes = ValueType->scalarBitWidth() / 8U;
+        return isSupportedIntegerType(*ValueType) &&
+                       isSupportedAtomicMemoryAccess(Instruction, AtomicOperation::Store, Bytes)
+                   ? LegalizationAction::Legal
+                   : LegalizationAction::Unsupported;
+      }
+      if ((isSupportedIntegerType(*ValueType) || isRVFloatType(*ValueType)) &&
+          isSupportedNonAtomicMemoryAccess(Instruction) &&
+          memoryMatchesType(Instruction, *ValueType)) {
+        return LegalizationAction::Legal;
+      }
+    }
+    return LegalizationAction::Unsupported;
+  case GenericOpcode::G_SEXTLOAD:
+  case GenericOpcode::G_ZEXTLOAD: {
+    const auto Bytes = memoryAccessBytes(Instruction);
+    const std::optional<LLT> AddressType = operandType(Function, Instruction, 1);
+    return Type && Bytes && AddressType && isSupportedIntegerType(*Type) &&
+                   AddressType->isPointer() && AddressType->addressSpace() == 0 &&
+                   AddressType->scalarBitWidth() == 64 &&
+                   *Bytes < ((Type->scalarBitWidth() + 7U) / 8U) &&
+                   (*Bytes == 1 || *Bytes == 2 || *Bytes == 4) &&
+                   isSupportedNonAtomicMemoryAccess(Instruction)
+               ? LegalizationAction::Legal
+               : LegalizationAction::Unsupported;
+  }
+  case GenericOpcode::G_FPEXTLOAD: {
+    const auto Bytes = memoryAccessBytes(Instruction);
+    const std::optional<LLT> AddressType = operandType(Function, Instruction, 1);
+    return Type && Bytes && AddressType && Type->isFloat() && Type->scalarBitWidth() == 64 &&
+                   AddressType->isPointer() && AddressType->addressSpace() == 0 &&
+                   AddressType->scalarBitWidth() == 64 && *Bytes == 4 &&
+                   isSupportedNonAtomicMemoryAccess(Instruction)
+               ? LegalizationAction::Legal
+               : LegalizationAction::Unsupported;
+  }
+  case GenericOpcode::G_FPTRUNCSTORE: {
+    const std::optional<LLT> ValueType = operandType(Function, Instruction, 0);
+    const std::optional<LLT> AddressType = operandType(Function, Instruction, 1);
+    const auto Bytes = memoryAccessBytes(Instruction);
+    return ValueType && AddressType && Bytes && ValueType->isFloat() &&
+                   ValueType->scalarBitWidth() == 64 && AddressType->isPointer() &&
+                   AddressType->addressSpace() == 0 && AddressType->scalarBitWidth() == 64 &&
+                   *Bytes == 4 && isSupportedNonAtomicMemoryAccess(Instruction)
+               ? LegalizationAction::Legal
+               : LegalizationAction::Unsupported;
+  }
+  case GenericOpcode::G_LOAD:
+    if (Type) {
+      const std::optional<LLT> AddressType = operandType(Function, Instruction, 1);
+      if (!AddressType || !AddressType->isPointer() || AddressType->addressSpace() != 0 ||
+          AddressType->scalarBitWidth() != 64) {
+        return LegalizationAction::Unsupported;
+      }
+      if (Instruction.memOperands().size() == 1 && Instruction.memOperands().front()->isAtomic()) {
+        const std::uint64_t Bytes = Type->scalarBitWidth() / 8U;
+        return isSupportedIntegerType(*Type) &&
+                       isSupportedAtomicMemoryAccess(Instruction, AtomicOperation::Load, Bytes)
+                   ? LegalizationAction::Legal
+                   : LegalizationAction::Unsupported;
+      }
+      if ((isSupportedIntegerType(*Type) || isRVFloatType(*Type)) &&
+          isSupportedNonAtomicMemoryAccess(Instruction) && memoryMatchesType(Instruction, *Type)) {
+        return LegalizationAction::Legal;
+      }
+    }
+    return LegalizationAction::Unsupported;
+  case GenericOpcode::G_FADD:
+  case GenericOpcode::G_FSUB:
+  case GenericOpcode::G_FMUL:
+  case GenericOpcode::G_FDIV: {
+    const std::optional<LLT> Left = operandType(Function, Instruction, 1);
+    const std::optional<LLT> Right = operandType(Function, Instruction, 2);
+    return Type && Left && Right && isRVFloatType(*Type) && *Left == *Type && *Right == *Type
+               ? LegalizationAction::Legal
+               : LegalizationAction::Unsupported;
+  }
+  case GenericOpcode::G_FMA: {
+    const std::optional<LLT> First = operandType(Function, Instruction, 1);
+    const std::optional<LLT> Second = operandType(Function, Instruction, 2);
+    const std::optional<LLT> Third = operandType(Function, Instruction, 3);
+    return Type && First && Second && Third && isRVFloatType(*Type) && *First == *Type &&
+                   *Second == *Type && *Third == *Type
+               ? LegalizationAction::Legal
+               : LegalizationAction::Unsupported;
+  }
+  case GenericOpcode::G_FNEG:
+  case GenericOpcode::G_FABS:
+  case GenericOpcode::G_FSQRT: {
+    const std::optional<LLT> Source = operandType(Function, Instruction, 1);
+    return Type && Source && isRVFloatType(*Type) && *Source == *Type
+               ? LegalizationAction::Legal
+               : LegalizationAction::Unsupported;
+  }
+  case GenericOpcode::G_FCOPYSIGN: {
+    const std::optional<LLT> Left = operandType(Function, Instruction, 1);
+    const std::optional<LLT> Right = operandType(Function, Instruction, 2);
+    return Type && Left && Right && isRVFloatType(*Type) && *Left == *Type && *Right == *Type
+               ? LegalizationAction::Legal
+               : LegalizationAction::Unsupported;
+  }
+  case GenericOpcode::G_FCONSTANT: {
+    const MachineOperand *Immediate = Instruction.operandAt(1);
+    const auto *Floating =
+        Immediate == nullptr ? nullptr : Immediate->getIf<FloatingImmediateOperand>();
+    const std::optional<FloatSemantics> Semantics = Type ? Type->floatSemantics() : std::nullopt;
+    return Type && isRVFloatType(*Type) && Floating != nullptr && Semantics &&
+                   Floating->Bits.semantics() == *Semantics
+               ? LegalizationAction::Legal
+               : LegalizationAction::Unsupported;
+  }
+  case GenericOpcode::G_ASSERT_SEXT:
+  case GenericOpcode::G_ASSERT_ZEXT:
+  case GenericOpcode::G_ASSERT_ALIGN: {
+    const std::optional<LLT> Source = operandType(Function, Instruction, 1);
+    return Type && Source && *Type == *Source && isScalarGPRType(*Type) &&
+                   Instruction.operandCount() == 3
+               ? LegalizationAction::Legal
+               : LegalizationAction::Unsupported;
+  }
+  case GenericOpcode::G_FREEZE:
+  case GenericOpcode::G_CONSTANT_FOLD_BARRIER: {
+    const std::optional<LLT> Source = operandType(Function, Instruction, 1);
+    return Type && Source && *Type == *Source &&
+                   (isSupportedIntegerType(*Type) || isRVFloatType(*Type))
+               ? LegalizationAction::Legal
+               : LegalizationAction::Unsupported;
+  }
+  case GenericOpcode::G_PTRTOINT: {
+    const std::optional<LLT> Source = operandType(Function, Instruction, 1);
+    return Type && Source && Type->isInteger() && Type->scalarBitWidth() == 64 &&
+                   Source->isPointer() && Source->scalarBitWidth() == 64
+               ? LegalizationAction::Legal
+               : LegalizationAction::Unsupported;
+  }
+  case GenericOpcode::G_INTTOPTR: {
+    const std::optional<LLT> Source = operandType(Function, Instruction, 1);
+    return Type && Source && Type->isPointer() && Type->scalarBitWidth() == 64 &&
+                   Source->isInteger() && Source->scalarBitWidth() == 64
+               ? LegalizationAction::Legal
+               : LegalizationAction::Unsupported;
+  }
+  case GenericOpcode::G_BITCAST: {
+    const std::optional<LLT> Source = operandType(Function, Instruction, 1);
+    const bool IntegerFloatPair =
+        Type && Source && Type->scalarBitWidth() == Source->scalarBitWidth() &&
+        ((Type->isFloat() && Source->isInteger()) || (Type->isInteger() && Source->isFloat())) &&
+        (Type->scalarBitWidth() == 32 || Type->scalarBitWidth() == 64);
+    return IntegerFloatPair ? LegalizationAction::Legal : LegalizationAction::Unsupported;
+  }
+  case GenericOpcode::G_ANYEXT:
+  case GenericOpcode::G_SEXT:
+  case GenericOpcode::G_ZEXT: {
+    const std::optional<LLT> Source = operandType(Function, Instruction, 1);
+    if (!Type || !Source)
+      return LegalizationAction::Unsupported;
+    if (!isScalarIntegerExtension(*Type, *Source)) {
+      const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+      const Register *Destination =
+          DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+      MachineInstr *OnlyUse =
+          Destination == nullptr ? nullptr : singleUseInstruction(Function, *Destination);
+      return (Type->isInteger() || Type->isAnyScalar()) &&
+                     (Source->isInteger() || Source->isAnyScalar()) &&
+                     Type->scalarBitWidth() > 64 &&
+                     Source->scalarBitWidth() < Type->scalarBitWidth() && OnlyUse != nullptr &&
+                     OnlyUse->opcode() == opcode(GenericOpcode::G_UNMERGE_VALUES)
+                 ? LegalizationAction::Legal
+                 : LegalizationAction::Unsupported;
+    }
+    const MachineOperand *SourceOperand = Instruction.operandAt(1);
+    const Register *SourceRegister =
+        SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>();
+    const MachineOperand *SourceDefinition =
+        SourceRegister == nullptr || !SourceRegister->isVirtual()
+            ? nullptr
+            : Function.registerInfo().getUniqueVRegDef(*SourceRegister);
+    const MachineInstr *SourceDef =
+        SourceDefinition == nullptr ? nullptr : SourceDefinition->parent();
+    if (SourceDef != nullptr && SourceDef->opcode() == opcode(GenericOpcode::G_CONSTANT))
+      return LegalizationAction::Custom;
+    if (Generic == GenericOpcode::G_ANYEXT &&
+        narrowAnyExtBinaryPlan(Instruction, Function).has_value())
+      return LegalizationAction::Custom;
+    return LegalizationAction::Legal;
+  }
+  case GenericOpcode::G_TRUNC: {
+    const std::optional<LLT> Source = operandType(Function, Instruction, 1);
+    if (Type && Source && (Type->isInteger() || Type->isAnyScalar()) &&
+        (Source->isInteger() || Source->isAnyScalar()) && Type->scalarBitWidth() > 64 &&
+        Source->scalarBitWidth() > Type->scalarBitWidth())
+      return LegalizationAction::Legal;
+    return Type && Source && isSupportedIntegerScalar(*Type) && isSupportedIntegerScalar(*Source) &&
+                   Type->scalarBitWidth() != 0 && Source->scalarBitWidth() >= Type->scalarBitWidth()
+               ? LegalizationAction::Legal
+               : LegalizationAction::Unsupported;
+  }
+  case GenericOpcode::G_MERGE_VALUES:
+  case GenericOpcode::G_UNMERGE_VALUES: {
+    if (Instruction.operandCount() < 3)
+      return LegalizationAction::Unsupported;
+    const std::size_t SmallTypeIndex = Generic == GenericOpcode::G_MERGE_VALUES ? 1 : 0;
+    const std::size_t BigTypeIndex =
+        Generic == GenericOpcode::G_MERGE_VALUES ? 0 : Instruction.operandCount() - 1U;
+    const std::optional<LLT> SmallType = operandType(Function, Instruction, SmallTypeIndex);
+    const std::optional<LLT> BigType = operandType(Function, Instruction, BigTypeIndex);
+    if (SmallType && BigType && Generic == GenericOpcode::G_MERGE_VALUES &&
+        BigType->scalarBitWidth() > 64 && !canFoldWideMergeImmediately(Instruction, Function))
+      return LegalizationAction::Legal;
+    return SmallType && BigType && (SmallType->isInteger() || SmallType->isAnyScalar()) &&
+                   (BigType->isInteger() || BigType->isAnyScalar()) &&
+                   SmallType->scalarBitWidth() != 0 && BigType->scalarBitWidth() != 0
+               ? LegalizationAction::Custom
+               : LegalizationAction::Unsupported;
+  }
+  case GenericOpcode::G_SEXT_INREG: {
+    const std::optional<LLT> Source = operandType(Function, Instruction, 1);
+    const MachineOperand *WidthOperand = Instruction.operandAt(2);
+    const auto Width =
+        WidthOperand == nullptr
+            ? Expected<std::int64_t>(std::unexpected(
+                  Error(ErrorCode::InvalidArgument, "missing in-register extension width")))
+            : signedImmediate(*WidthOperand);
+    return Type && Source && *Type == *Source && isScalarGPRType(*Type) && Width && *Width > 0 &&
+                   static_cast<std::uint64_t>(*Width) <= Type->scalarBitWidth()
+               ? LegalizationAction::Legal
+               : LegalizationAction::Unsupported;
+  }
+  case GenericOpcode::G_ICMP: {
+    const MachineOperand *Predicate = Instruction.operandAt(1);
+    const auto *IntegerPredicate =
+        Predicate == nullptr ? nullptr : Predicate->getIf<PredicateOperand>();
+    const std::optional<LLT> Left = operandType(Function, Instruction, 2);
+    const std::optional<LLT> Right = operandType(Function, Instruction, 3);
+    if (!Type || !isSupportedIntegerScalar(*Type) || IntegerPredicate == nullptr ||
+        IntegerPredicate->integerPredicate() == nullptr || !Left || !Right || *Left != *Right ||
+        !isSupportedIntegerType(*Left)) {
+      return LegalizationAction::Unsupported;
+    }
+    return Type->scalarBitWidth() == 64 && Left->scalarBitWidth() == 64
+               ? LegalizationAction::Legal
+               : LegalizationAction::Custom;
+  }
+  case GenericOpcode::G_FCMP: {
+    const MachineOperand *Predicate = Instruction.operandAt(1);
+    const auto *FloatingPredicate =
+        Predicate == nullptr ? nullptr : Predicate->getIf<PredicateOperand>();
+    const std::optional<LLT> Left = operandType(Function, Instruction, 2);
+    const std::optional<LLT> Right = operandType(Function, Instruction, 3);
+    return Type && isI1Type(*Type) && FloatingPredicate != nullptr &&
+                   FloatingPredicate->floatingPredicate() != nullptr && Left && Right &&
+                   *Left == *Right && isRVFloatType(*Left)
+               ? LegalizationAction::Legal
+               : LegalizationAction::Unsupported;
+  }
+  case GenericOpcode::G_FPEXT:
+  case GenericOpcode::G_FPTRUNC: {
+    const std::optional<LLT> Source = operandType(Function, Instruction, 1);
+    const bool IsExtension = Generic == GenericOpcode::G_FPEXT;
+    return Type && Source && isRVFloatType(*Type) && isRVFloatType(*Source) &&
+                   ((IsExtension && Source->scalarBitWidth() == 32 &&
+                     Type->scalarBitWidth() == 64) ||
+                    (!IsExtension && Source->scalarBitWidth() == 64 &&
+                     Type->scalarBitWidth() == 32))
+               ? LegalizationAction::Legal
+               : LegalizationAction::Unsupported;
+  }
+  case GenericOpcode::G_FPTOSI:
+  case GenericOpcode::G_FPTOUI: {
+    const std::optional<LLT> Source = operandType(Function, Instruction, 1);
+    return Type && Source && isSupportedIntegerType(*Type) && isRVFloatType(*Source)
+               ? LegalizationAction::Legal
+               : LegalizationAction::Unsupported;
+  }
+  case GenericOpcode::G_SITOFP:
+  case GenericOpcode::G_UITOFP: {
+    const std::optional<LLT> Source = operandType(Function, Instruction, 1);
+    return Type && Source && isRVFloatType(*Type) && isSupportedIntegerType(*Source)
+               ? LegalizationAction::Legal
+               : LegalizationAction::Unsupported;
+  }
+  case GenericOpcode::G_FMINNUM:
+  case GenericOpcode::G_FMAXNUM: {
+    const std::optional<LLT> Left = operandType(Function, Instruction, 1);
+    const std::optional<LLT> Right = operandType(Function, Instruction, 2);
+    return Type && Left && Right && isRVFloatType(*Type) && *Left == *Type && *Right == *Type
+               ? LegalizationAction::Legal
+               : LegalizationAction::Unsupported;
+  }
+  case GenericOpcode::G_SELECT: {
+    const std::optional<LLT> Condition = operandType(Function, Instruction, 1);
+    const std::optional<LLT> TrueValue = operandType(Function, Instruction, 2);
+    const std::optional<LLT> FalseValue = operandType(Function, Instruction, 3);
+    return Type && Condition && TrueValue && FalseValue && isScalarGPRType(*Condition) &&
+                   *TrueValue == *Type && *FalseValue == *Type &&
+                   (isSupportedIntegerType(*Type) || isRVFloatType(*Type))
+               ? LegalizationAction::Legal
+               : LegalizationAction::Unsupported;
+  }
+  case GenericOpcode::G_PTRMASK: {
+    const std::optional<LLT> Pointer = operandType(Function, Instruction, 1);
+    const std::optional<LLT> Mask = operandType(Function, Instruction, 2);
+    return Type && Pointer && Mask && Type->isPointer() && Type->addressSpace() == 0 &&
+                   Type->scalarBitWidth() == 64 && *Pointer == *Type &&
+                   isSupportedIntegerScalar(*Mask) && Mask->scalarBitWidth() == 64
+               ? LegalizationAction::Legal
+               : LegalizationAction::Unsupported;
+  }
+  case GenericOpcode::G_ABS: {
+    const std::optional<LLT> Source = operandType(Function, Instruction, 1);
+    return Type && Source && *Source == *Type && isSupportedIntegerScalar(*Type)
+               ? LegalizationAction::Legal
+               : LegalizationAction::Unsupported;
+  }
+  case GenericOpcode::G_SDIVREM:
+  case GenericOpcode::G_UDIVREM: {
+    const std::optional<LLT> Remainder = operandType(Function, Instruction, 1);
+    const std::optional<LLT> Left = operandType(Function, Instruction, 2);
+    const std::optional<LLT> Right = operandType(Function, Instruction, 3);
+    return Type && Remainder && Left && Right && isSupportedIntegerScalar(*Type) &&
+                   *Remainder == *Type && *Left == *Type && *Right == *Type
+               ? LegalizationAction::Legal
+               : LegalizationAction::Unsupported;
+  }
+  case GenericOpcode::G_SMULH:
+  case GenericOpcode::G_UMULH: {
+    const std::optional<LLT> Left = operandType(Function, Instruction, 1);
+    const std::optional<LLT> Right = operandType(Function, Instruction, 2);
+    return Type && Left && Right && isNativeIntegerScalar(*Type) && *Left == *Type &&
+                   *Right == *Type
+               ? LegalizationAction::Legal
+               : LegalizationAction::Unsupported;
+  }
+  case GenericOpcode::G_SMIN:
+  case GenericOpcode::G_SMAX:
+  case GenericOpcode::G_UMIN:
+  case GenericOpcode::G_UMAX: {
+    const std::optional<LLT> Left = operandType(Function, Instruction, 1);
+    const std::optional<LLT> Right = operandType(Function, Instruction, 2);
+    return Type && Left && Right && isSupportedIntegerScalar(*Type) && *Left == *Type &&
+                   *Right == *Type
+               ? LegalizationAction::Legal
+               : LegalizationAction::Unsupported;
+  }
+  case GenericOpcode::G_UADDO:
+  case GenericOpcode::G_USUBO:
+  case GenericOpcode::G_SADDO:
+  case GenericOpcode::G_SSUBO:
+  case GenericOpcode::G_UMULO:
+  case GenericOpcode::G_SMULO: {
+    const std::optional<LLT> Overflow = operandType(Function, Instruction, 1);
+    const std::optional<LLT> Left = operandType(Function, Instruction, 2);
+    const std::optional<LLT> Right = operandType(Function, Instruction, 3);
+    return Type && Overflow && Left && Right && isNativeIntegerScalar(*Type) &&
+                   isI1Type(*Overflow) && *Left == *Type && *Right == *Type
+               ? LegalizationAction::Legal
+               : LegalizationAction::Unsupported;
+  }
+  case GenericOpcode::G_UADDE:
+  case GenericOpcode::G_USUBE:
+  case GenericOpcode::G_SADDE:
+  case GenericOpcode::G_SSUBE: {
+    const std::optional<LLT> CarryOut = operandType(Function, Instruction, 1);
+    const std::optional<LLT> Left = operandType(Function, Instruction, 2);
+    const std::optional<LLT> Right = operandType(Function, Instruction, 3);
+    const std::optional<LLT> CarryIn = operandType(Function, Instruction, 4);
+    return Type && CarryOut && Left && Right && CarryIn && isNativeIntegerScalar(*Type) &&
+                   isI1Type(*CarryOut) && isI1Type(*CarryIn) && *Left == *Type && *Right == *Type
+               ? LegalizationAction::Legal
+               : LegalizationAction::Unsupported;
+  }
+  case GenericOpcode::G_ATOMICRMW_XCHG:
+  case GenericOpcode::G_ATOMICRMW_ADD:
+  case GenericOpcode::G_ATOMICRMW_SUB:
+  case GenericOpcode::G_ATOMICRMW_AND:
+  case GenericOpcode::G_ATOMICRMW_OR:
+  case GenericOpcode::G_ATOMICRMW_XOR:
+  case GenericOpcode::G_ATOMICRMW_MAX:
+  case GenericOpcode::G_ATOMICRMW_MIN:
+  case GenericOpcode::G_ATOMICRMW_UMAX:
+  case GenericOpcode::G_ATOMICRMW_UMIN: {
+    const std::optional<LLT> Address = operandType(Function, Instruction, 1);
+    const std::optional<LLT> Value = operandType(Function, Instruction, 2);
+    const std::uint64_t Bytes = Type ? Type->scalarBitWidth() / 8U : 0;
+    return Type && Address && Value && isNativeIntegerScalar(*Type) && *Value == *Type &&
+                   Address->isPointer() && Address->addressSpace() == 0 &&
+                   Address->scalarBitWidth() == 64 &&
+                   isSupportedAtomicMemoryAccess(Instruction, AtomicOperation::ReadModifyWrite,
+                                                 Bytes)
+               ? LegalizationAction::Legal
+               : LegalizationAction::Unsupported;
+  }
+  case GenericOpcode::G_ATOMIC_CMPXCHG: {
+    const std::optional<LLT> Address = operandType(Function, Instruction, 1);
+    const std::optional<LLT> Compare = operandType(Function, Instruction, 2);
+    const std::optional<LLT> NewValue = operandType(Function, Instruction, 3);
+    if (!Type || !Address || !Compare || !NewValue || *Compare != *Type || *NewValue != *Type ||
+        (!Type->isInteger() && !Type->isAnyScalar()) ||
+        (Type->scalarBitWidth() != 8 && Type->scalarBitWidth() != 16 &&
+         Type->scalarBitWidth() != 32 && Type->scalarBitWidth() != 64) ||
+        !Address->isPointer() || Address->addressSpace() != 0 || Address->scalarBitWidth() != 64 ||
+        !isSupportedAtomicContainerAccess(Instruction, AtomicOperation::CompareExchange,
+                                          (Type->scalarBitWidth() + 7U) / 8U)) {
+      return LegalizationAction::Unsupported;
+    }
+    return Type->scalarBitWidth() == 64 ? LegalizationAction::Legal : LegalizationAction::Custom;
+  }
+  case GenericOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
+    const std::optional<LLT> Success = operandType(Function, Instruction, 1);
+    const std::optional<LLT> Address = operandType(Function, Instruction, 2);
+    const std::optional<LLT> Compare = operandType(Function, Instruction, 3);
+    const std::optional<LLT> NewValue = operandType(Function, Instruction, 4);
+    return Type && Success && Address && Compare && NewValue && isI1Type(*Success) &&
+                   *Compare == *Type && *NewValue == *Type &&
+                   (Type->isInteger() || Type->isAnyScalar()) &&
+                   (Type->scalarBitWidth() == 8 || Type->scalarBitWidth() == 16 ||
+                    Type->scalarBitWidth() == 32 || Type->scalarBitWidth() == 64) &&
+                   Address->isPointer() && Address->addressSpace() == 0 &&
+                   Address->scalarBitWidth() == 64 &&
+                   isSupportedAtomicContainerAccess(Instruction, AtomicOperation::CompareExchange,
+                                                    (Type->scalarBitWidth() + 7U) / 8U)
+               ? LegalizationAction::Custom
+               : LegalizationAction::Unsupported;
+  }
+  case GenericOpcode::G_FENCE: {
+    const MachineOperand *OrderingOperand = Instruction.operandAt(0);
+    const MachineOperand *ScopeOperand = Instruction.operandAt(1);
+    const auto Ordering = OrderingOperand == nullptr
+                              ? Expected<std::int64_t>(std::unexpected(
+                                    Error(ErrorCode::InvalidArgument, "missing fence ordering")))
+                              : signedImmediate(*OrderingOperand);
+    const auto Scope = ScopeOperand == nullptr
+                           ? Expected<std::int64_t>(std::unexpected(
+                                 Error(ErrorCode::InvalidArgument, "missing fence scope")))
+                           : signedImmediate(*ScopeOperand);
+    return Ordering && Scope && *Ordering >= 4 && *Ordering <= 7 && *Scope == 0
+               ? LegalizationAction::Legal
+               : LegalizationAction::Unsupported;
+  }
+  case GenericOpcode::G_CONSTANT_POOL: {
+    const MachineOperand *PoolOperand = Instruction.operandAt(1);
+    const auto *Pool = PoolOperand == nullptr ? nullptr : PoolOperand->getIf<ConstantPoolOperand>();
+    return Type && Type->isPointer() && Type->addressSpace() == 0 && Type->scalarBitWidth() == 64 &&
+                   Pool != nullptr && Function.constantPool().entry(Pool->Index) != nullptr
+               ? LegalizationAction::Legal
+               : LegalizationAction::Unsupported;
+  }
+  case GenericOpcode::G_GLOBAL_VALUE: {
+    const MachineOperand *GlobalOperand = Instruction.operandAt(1);
+    const auto *Global =
+        GlobalOperand == nullptr ? nullptr : GlobalOperand->getIf<GlobalSymbolOperand>();
+    return Type && Type->isPointer() && Type->addressSpace() == 0 && Type->scalarBitWidth() == 64 &&
+                   Global != nullptr && !Global->Name.empty()
+               ? LegalizationAction::Legal
+               : LegalizationAction::Unsupported;
+  }
+  case GenericOpcode::G_BRINDIRECT:
+    return operandType(Function, Instruction, 0)
+                   .transform([](LLT Value) {
+                     return Value.isPointer() && Value.addressSpace() == 0 &&
+                            Value.scalarBitWidth() == 64;
+                   })
+                   .value_or(false)
+               ? LegalizationAction::Legal
+               : LegalizationAction::Unsupported;
+  case GenericOpcode::G_TRAP:
+  case GenericOpcode::G_DEBUGTRAP:
+    return Instruction.operandCount() == 0 ? LegalizationAction::Legal
+                                           : LegalizationAction::Unsupported;
+  case GenericOpcode::G_UBSANTRAP:
+    return Instruction.operandCount() == 1 ? LegalizationAction::Legal
+                                           : LegalizationAction::Unsupported;
+  case GenericOpcode::G_CONSTANT: {
+    const MachineOperand *Immediate = Instruction.operandAt(1);
+    if (!Type || Immediate == nullptr ||
+        (Immediate->kind() != MachineOperand::Kind::Immediate &&
+         Immediate->kind() != MachineOperand::Kind::IntegerImmediate))
+      return LegalizationAction::Unsupported;
+    if (Type->scalarBitWidth() > 64)
+      return wideConstantUnmergePlan(Instruction, Function).has_value()
+                 ? LegalizationAction::Custom
+                 : LegalizationAction::Unsupported;
+    if ((Type->isInteger() || Type->isAnyScalar()) && Type->scalarBitWidth() == 64) {
+      auto Value = signedImmediate(*Immediate);
+      if (Value && shouldUseRV64IntegerConstantPool(*Value))
+        return LegalizationAction::Custom;
+    }
+    return isSupportedIntegerType(*Type) ? LegalizationAction::Legal
+                                         : LegalizationAction::Unsupported;
+  }
+  case GenericOpcode::G_ADD:
+  case GenericOpcode::G_SUB:
+  case GenericOpcode::G_MUL:
+  case GenericOpcode::G_SDIV:
+  case GenericOpcode::G_UDIV:
+  case GenericOpcode::G_SREM:
+  case GenericOpcode::G_UREM:
+  case GenericOpcode::G_AND:
+  case GenericOpcode::G_OR:
+  case GenericOpcode::G_XOR:
+  case GenericOpcode::G_SHL:
+  case GenericOpcode::G_LSHR:
+  case GenericOpcode::G_ASHR: {
+    const std::optional<LLT> Left = operandType(Function, Instruction, 1);
+    const std::optional<LLT> Right = operandType(Function, Instruction, 2);
+    if (!Type || !Left || !Right || *Left != *Type || *Right != *Type ||
+        (!Type->isInteger() && !Type->isAnyScalar()))
+      return LegalizationAction::Unsupported;
+    if (Type->scalarBitWidth() > 64) {
+      return Generic == GenericOpcode::G_ADD || Generic == GenericOpcode::G_SUB ||
+                     Generic == GenericOpcode::G_AND || Generic == GenericOpcode::G_OR ||
+                     Generic == GenericOpcode::G_XOR || Generic == GenericOpcode::G_SHL
+                 ? LegalizationAction::Custom
+                 : LegalizationAction::Unsupported;
+    }
+    if (Generic == GenericOpcode::G_MUL && !Subtarget.features().has(RISCVExtension::M)) {
+      if (Type->scalarBitWidth() == 64)
+        return LegalizationAction::Libcall;
+      const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+      const Register *Destination =
+          DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+      MachineInstr *Extension =
+          Destination == nullptr ? nullptr : singleRegisterUse(Function, *Destination);
+      const std::optional<LLT> ExtensionType =
+          Extension == nullptr ? std::nullopt : resultType(Function, *Extension);
+      if (Extension != nullptr && Extension->opcode() == opcode(GenericOpcode::G_ANYEXT) &&
+          ExtensionType && ExtensionType->scalarBitWidth() == 64) {
+        return LegalizationAction::Legal;
+      }
+      return LegalizationAction::Custom;
+    }
+    return isSupportedIntegerScalar(*Type) ? LegalizationAction::Legal
+                                           : LegalizationAction::Unsupported;
+  }
+  case GenericOpcode::G_PTR_ADD: {
+    const std::optional<LLT> Pointer = operandType(Function, Instruction, 1);
+    const std::optional<LLT> Offset = operandType(Function, Instruction, 2);
+    return Type && Pointer && Offset && Type->isPointer() && Type->addressSpace() == 0 &&
+                   Type->scalarBitWidth() == 64 && *Pointer == *Type &&
+                   isSupportedIntegerScalar(*Offset) && Offset->scalarBitWidth() == 64
+               ? LegalizationAction::Legal
+               : LegalizationAction::Unsupported;
+  }
+  default:
+    break;
+  }
+
+  // Keep the unsupported policy mechanically synchronized with the complete
+  // target-independent opcode inventory. Supported cases return above.
+  switch (Generic) {
+#define RCC_CORE_OPCODE(Name)
+#define RCC_GENERIC_OPCODE(Name) case GenericOpcode::Name:
+#include "rcc/CodeGen/GenericOpcode.def"
+#undef RCC_GENERIC_OPCODE
+#undef RCC_CORE_OPCODE
+    return LegalizationAction::Unsupported;
+  case GenericOpcode::Invalid:
+  case GenericOpcode::CORE_PSEUDO_END:
+  case GenericOpcode::G_GENERIC_END:
+  default:
+    break;
+  }
+  return makeUnexpected(ErrorCode::InternalError,
+                        "RISC-V legalizer received an invalid generic opcode value");
+}
+
+Expected<void> RISCVLegalizerInfo::lower(MachineInstr &Instruction, MachineFunction &Function,
+                                         const TargetDescription &Target,
+                                         LegalizationAction Action) const {
+  if (Action == LegalizationAction::Custom &&
+      Instruction.opcode() == opcode(GenericOpcode::G_INTRINSIC_W_SIDE_EFFECTS)) {
+    return lowerVACopy(Instruction, Function, Target);
+  }
+  if (Action == LegalizationAction::Custom &&
+      Instruction.opcode() == opcode(GenericOpcode::G_VAARG)) {
+    return lowerVAArg(Instruction, Function, Target);
+  }
+  if (Instruction.opcode() == opcode(GenericOpcode::G_MUL) &&
+      !Subtarget.features().has(RISCVExtension::M)) {
+    if (Action == LegalizationAction::Custom)
+      return widenNarrowMultiply(Instruction, Function, Target);
+    if (Action == LegalizationAction::Libcall)
+      return lowerMultiplyLibcall(Instruction, Function, Target,
+                                  Subtarget.features().has(RISCVExtension::D)
+                                      ? CallingConvention::LP64D
+                                      : CallingConvention::LP64,
+                                  *Subtarget.getCallLowering());
+  }
+  if (Action == LegalizationAction::Custom &&
+      resultType(Function, Instruction)
+          .transform([](LLT Type) { return Type.scalarBitWidth() > 64; })
+          .value_or(false)) {
+    const Opcode Value = Instruction.opcode();
+    if (Value == opcode(GenericOpcode::G_ADD) || Value == opcode(GenericOpcode::G_SUB) ||
+        Value == opcode(GenericOpcode::G_AND) || Value == opcode(GenericOpcode::G_OR) ||
+        Value == opcode(GenericOpcode::G_XOR)) {
+      return lowerWideBinary(Instruction, Function, Target,
+                             static_cast<GenericOpcode>(Value.value()));
+    }
+    if (Value == opcode(GenericOpcode::G_SHL))
+      return lowerWideShiftLeft(Instruction, Function, Target);
+    if (Value == opcode(GenericOpcode::G_PHI))
+      return lowerWidePhi(Instruction, Function, Target);
+  }
+  if (Action == LegalizationAction::Custom &&
+      Instruction.opcode() == opcode(GenericOpcode::G_MERGE_VALUES))
+    return lowerMergeValues(Instruction, Function, Target);
+  if (Action == LegalizationAction::Custom &&
+      Instruction.opcode() == opcode(GenericOpcode::G_UNMERGE_VALUES))
+    return lowerUnmergeValues(Instruction, Function, Target);
+  if (Action == LegalizationAction::Custom &&
+      Instruction.opcode() == opcode(GenericOpcode::G_BRCOND))
+    return lowerNarrowBranchCondition(Instruction, Function, Target);
+  if (Action == LegalizationAction::Custom && Instruction.opcode() == opcode(GenericOpcode::G_PHI))
+    return lowerNarrowPhi(Instruction, Function, Target);
+  if (Action == LegalizationAction::Custom && Instruction.opcode() == opcode(GenericOpcode::G_ICMP))
+    return lowerIntegerCompare(Instruction, Function, Target);
+  if (Action == LegalizationAction::Custom &&
+      Instruction.opcode() == opcode(GenericOpcode::G_ATOMIC_CMPXCHG))
+    return lowerAtomicCompareExchange(Instruction, Function, Target);
+  if (Action == LegalizationAction::Custom &&
+      Instruction.opcode() == opcode(GenericOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS))
+    return lowerAtomicCompareExchangeWithSuccess(Instruction, Function, Target);
+  if (Action == LegalizationAction::Custom &&
+      (Instruction.opcode() == opcode(GenericOpcode::G_ANYEXT) ||
+       Instruction.opcode() == opcode(GenericOpcode::G_SEXT) ||
+       Instruction.opcode() == opcode(GenericOpcode::G_ZEXT))) {
+    if (Instruction.opcode() == opcode(GenericOpcode::G_ANYEXT) &&
+        narrowAnyExtBinaryPlan(Instruction, Function).has_value())
+      return foldNarrowAnyExtBinary(Instruction, Function, Target);
+    return foldConstantExtension(Instruction, Function,
+                                 static_cast<GenericOpcode>(Instruction.opcode().value()));
+  }
+  if (Action == LegalizationAction::Custom &&
+      Instruction.opcode() == opcode(GenericOpcode::G_CONSTANT)) {
+    const MachineOperand *Immediate = Instruction.operandAt(1);
+    const std::optional<LLT> Type = resultType(Function, Instruction);
+    if (Immediate != nullptr && Type && (Type->isInteger() || Type->isAnyScalar()) &&
+        Type->scalarBitWidth() == 64) {
+      auto Value = signedImmediate(*Immediate);
+      if (Value && shouldUseRV64IntegerConstantPool(*Value))
+        return lowerConstantToPool(Instruction, Function, Target);
+    }
+    return foldWideConstant(Instruction, Function, Target);
+  }
+  return makeUnexpected(ErrorCode::Unsupported,
+                        "RISC-V legalizer has no lowering rule for this instruction");
+}
+
+} // namespace rcc::riscv
diff --git a/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.h b/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.h
new file mode 100644
index 0000000..f6ba000
--- /dev/null
+++ b/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.h
@@ -0,0 +1,29 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef RCC_LIB_TARGET_RISCV_GISEL_RISCVLEGALIZERINFO_H
+#define RCC_LIB_TARGET_RISCV_GISEL_RISCVLEGALIZERINFO_H
+
+#include "rcc/CodeGen/GlobalISel/LegalizerInfo.h"
+
+namespace rcc::riscv {
+
+class RISCVSubtarget;
+
+class RISCVLegalizerInfo final : public LegalizerInfo {
+public:
+  explicit RISCVLegalizerInfo(const RISCVSubtarget &Subtarget) : Subtarget(Subtarget) {}
+
+  [[nodiscard]] Expected<LegalizationAction>
+  classify(const MachineInstr &Instruction, const MachineFunction &Function,
+           const TargetDescription &Target) const override;
+  [[nodiscard]] Expected<void> lower(MachineInstr &Instruction, MachineFunction &Function,
+                                     const TargetDescription &Target,
+                                     LegalizationAction Action) const override;
+
+private:
+  const RISCVSubtarget &Subtarget;
+};
+
+} // namespace rcc::riscv
+
+#endif // RCC_LIB_TARGET_RISCV_GISEL_RISCVLEGALIZERINFO_H
diff --git a/backend/lib/Target/RISCV/GISel/RISCVRegisterBankInfo.cpp b/backend/lib/Target/RISCV/GISel/RISCVRegisterBankInfo.cpp
new file mode 100644
index 0000000..6e38d52
--- /dev/null
+++ b/backend/lib/Target/RISCV/GISel/RISCVRegisterBankInfo.cpp
@@ -0,0 +1,157 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "RISCVRegisterBankInfo.h"
+
+#include "rcc/CodeGen/MachineFunction.h"
+#include "rcc/CodeGen/MachineInstr.h"
+#include "rcc/CodeGen/TargetDescription.h"
+#include "rcc/Target/RISCV/RISCVRegisters.h"
+
+#include <cstdint>
+#include <ranges>
+#include <utility>
+#include <vector>
+
+namespace rcc::riscv {
+namespace {
+
+[[nodiscard]] bool isRVFloatType(LLT Type) noexcept {
+  return Type.isFloat() && (Type.scalarBitWidth() == 32 || Type.scalarBitWidth() == 64);
+}
+
+[[nodiscard]] bool isScalarGPRType(LLT Type) noexcept {
+  return (Type.isPointer() && Type.addressSpace() == 0 && Type.scalarBitWidth() == 64) ||
+         ((Type.isInteger() || Type.isAnyScalar()) && Type.scalarBitWidth() >= 1 &&
+          Type.scalarBitWidth() <= 64);
+}
+
+[[nodiscard]] Expected<RegisterClassID> requiredRegisterClass(const MachineFunction &Function,
+                                                              Register Reg,
+                                                              const TargetDescription &Target) {
+  if (!Reg.isVirtual())
+    return makeUnexpected(ErrorCode::InvalidArgument,
+                          "RISC-V register-class selection requires a virtual register");
+  auto Info = Function.registerInfo().virtualRegisterInfo(Reg);
+  if (!Info)
+    return std::unexpected(Info.error());
+  if (!Info->get().Type) {
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "RISC-V selection requires a type for every virtual register");
+  }
+  const VirtualRegisterInfo &RegisterInfo = Info->get();
+  const LLT Type = RegisterInfo.Type.value_or(LLT{});
+  RegisterClassID RequiredClass;
+  if (isScalarGPRType(Type)) {
+    RequiredClass = RegisterClassID::fromValue(static_cast<std::uint32_t>(RegisterClass::GPR));
+  } else if (Type.isFloat() && Type.scalarBitWidth() == 32) {
+    RequiredClass = RegisterClassID::fromValue(static_cast<std::uint32_t>(RegisterClass::FPR32));
+  } else if (Type.isFloat() && Type.scalarBitWidth() == 64) {
+    RequiredClass = RegisterClassID::fromValue(static_cast<std::uint32_t>(RegisterClass::FPR64));
+  } else {
+    return makeUnexpected(ErrorCode::Unsupported,
+                          "RISC-V RV64GC has no register class for type '" + Type.toString() + "'");
+  }
+
+  if (Target.registers().registerClass(RequiredClass) == nullptr) {
+    return makeUnexpected(ErrorCode::InternalError,
+                          "RISC-V target description is missing a required register class");
+  }
+  if (RegisterInfo.Bank && !Target.registers().bankCoversClass(*RegisterInfo.Bank, RequiredClass)) {
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "RISC-V virtual register bank cannot cover its required class");
+  }
+  if (RegisterInfo.RegClass &&
+      !Target.registers().isSubclass(*RegisterInfo.RegClass, RequiredClass)) {
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "RISC-V virtual register has a conflicting register class");
+  }
+  return RequiredClass;
+}
+
+} // namespace
+
+Expected<RegisterBankID> RISCVRegisterBankInfo::selectBank(const MachineFunction &, Register,
+                                                           const VirtualRegisterInfo &Info,
+                                                           const MachineInstr *,
+                                                           const TargetDescription &) const {
+  if (!Info.Type) {
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "RISC-V register-bank selection requires a virtual-register type");
+  }
+  if (isScalarGPRType(*Info.Type))
+    return RegisterBankID::fromValue(static_cast<std::uint32_t>(RegisterBank::GPR));
+  if (isRVFloatType(*Info.Type))
+    return RegisterBankID::fromValue(static_cast<std::uint32_t>(RegisterBank::FPR));
+  return makeUnexpected(ErrorCode::Unsupported,
+                        "RISC-V RV64GC has no register bank for virtual register type '" +
+                            Info.Type->toString() + "'");
+}
+
+Expected<void> RISCVRegisterBankInfo::constrainInstruction(MachineInstr &Instruction,
+                                                           MachineFunction &Function,
+                                                           const TargetDescription &Target) const {
+  std::vector<std::pair<Register, RegisterClassID>> PlannedClasses;
+  for (const MachineOperand &Operand : Instruction.operands()) {
+    const Register *Reg = Operand.getIf<Register>();
+    if (Reg == nullptr || !Reg->isVirtual())
+      continue;
+    auto Required = requiredRegisterClass(Function, *Reg, Target);
+    if (!Required)
+      return std::unexpected(Required.error());
+    if (std::ranges::find_if(PlannedClasses, [Reg](const auto &Entry) {
+          return Entry.first == *Reg;
+        }) == PlannedClasses.end())
+      PlannedClasses.emplace_back(*Reg, *Required);
+  }
+  for (const auto &[Reg, Required] : PlannedClasses) {
+    auto Info = Function.registerInfo().virtualRegisterInfo(Reg);
+    if (!Info)
+      return std::unexpected(Info.error());
+    if (Info->get().RegClass)
+      continue;
+    auto SetClass = Function.registerInfo().setRegisterClass(Reg, Required);
+    if (!SetClass)
+      return std::unexpected(SetClass.error());
+  }
+  return {};
+}
+
+Expected<void> RISCVRegisterBankInfo::constrainRegister(MachineFunction &Function, Register Reg,
+                                                        const TargetDescription &Target) const {
+  auto Info = Function.registerInfo().virtualRegisterInfo(Reg);
+  if (!Info)
+    return std::unexpected(Info.error());
+  auto Bank = selectBank(Function, Reg, Info->get(), nullptr, Target);
+  if (!Bank)
+    return std::unexpected(Bank.error());
+  auto RequiredClass = requiredRegisterClass(Function, Reg, Target);
+  if (!RequiredClass)
+    return std::unexpected(RequiredClass.error());
+  if (!Info->get().Bank) {
+    auto SetBank = Function.registerInfo().setRegisterBank(Reg, *Bank);
+    if (!SetBank)
+      return std::unexpected(SetBank.error());
+  }
+  if (!Info->get().RegClass) {
+    auto SetClass = Function.registerInfo().setRegisterClass(Reg, *RequiredClass);
+    if (!SetClass)
+      return std::unexpected(SetClass.error());
+  }
+  return {};
+}
+
+Expected<void> RISCVRegisterBankInfo::validateInstruction(const MachineInstr &Instruction,
+                                                          const MachineFunction &Function,
+                                                          const TargetDescription &Target) const {
+  for (const MachineOperand &Operand : Instruction.operands()) {
+    const Register *Reg = Operand.getIf<Register>();
+    if (Reg == nullptr || !Reg->isVirtual())
+      continue;
+    auto Required = requiredRegisterClass(Function, *Reg, Target);
+    if (!Required)
+      return std::unexpected(Required.error());
+  }
+  return {};
+}
+
+} // namespace rcc::riscv
diff --git a/backend/lib/Target/RISCV/GISel/RISCVRegisterBankInfo.h b/backend/lib/Target/RISCV/GISel/RISCVRegisterBankInfo.h
new file mode 100644
index 0000000..543bc4d
--- /dev/null
+++ b/backend/lib/Target/RISCV/GISel/RISCVRegisterBankInfo.h
@@ -0,0 +1,28 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef RCC_LIB_TARGET_RISCV_GISEL_RISCVREGISTERBANKINFO_H
+#define RCC_LIB_TARGET_RISCV_GISEL_RISCVREGISTERBANKINFO_H
+
+#include "rcc/CodeGen/RegisterBankInfo.h"
+
+namespace rcc::riscv {
+
+class RISCVRegisterBankInfo final : public RegisterBankInfo {
+public:
+  [[nodiscard]] Expected<RegisterBankID> selectBank(const MachineFunction &Function, Register Reg,
+                                                    const VirtualRegisterInfo &Info,
+                                                    const MachineInstr *Instruction,
+                                                    const TargetDescription &Target) const override;
+  [[nodiscard]] Expected<void> constrainInstruction(MachineInstr &Instruction,
+                                                    MachineFunction &Function,
+                                                    const TargetDescription &Target) const override;
+  [[nodiscard]] Expected<void> constrainRegister(MachineFunction &Function, Register Reg,
+                                                 const TargetDescription &Target) const;
+  [[nodiscard]] Expected<void> validateInstruction(const MachineInstr &Instruction,
+                                                   const MachineFunction &Function,
+                                                   const TargetDescription &Target) const override;
+};
+
+} // namespace rcc::riscv
+
+#endif // RCC_LIB_TARGET_RISCV_GISEL_RISCVREGISTERBANKINFO_H