[CodeGen] Complete machine pass and ABI lowering infrastructure

2 files changed, 1477 insertions, 0 deletions

diff --git a/backend/lib/Target/RISCV/CMakeLists.txt b/backend/lib/Target/RISCV/CMakeLists.txt
index a77494a..ff3f334 100644
--- a/backend/lib/Target/RISCV/CMakeLists.txt
+++ b/backend/lib/Target/RISCV/CMakeLists.txt
@@ -47,6 +47,7 @@ set_source_files_properties(
 
 add_library(
   RCCBackendRISCV
+  RISCVCallLowering.cpp
   RISCVTarget.cpp
   "${RCCBackendRISCVGeneratedRegisterHeader}"
   "${RCCBackendRISCVGeneratedInstructionHeader}"
@@ -89,9 +90,12 @@ install(
 )
 
 rcc_backend_register_format_files(
+  "${CMAKE_CURRENT_SOURCE_DIR}/RISCVCallLowering.cpp"
   "${CMAKE_CURRENT_SOURCE_DIR}/RISCVTarget.cpp"
+  "${RCC_BACKEND_SOURCE_DIR}/include/rcc/Target/RISCV/RISCVCallLowering.h"
   "${RCC_BACKEND_SOURCE_DIR}/include/rcc/Target/RISCV/RISCVTarget.h"
 )
 rcc_backend_register_tidy_files(
+  "${CMAKE_CURRENT_SOURCE_DIR}/RISCVCallLowering.cpp"
   "${CMAKE_CURRENT_SOURCE_DIR}/RISCVTarget.cpp"
 )
diff --git a/backend/lib/Target/RISCV/RISCVCallLowering.cpp b/backend/lib/Target/RISCV/RISCVCallLowering.cpp
new file mode 100644
index 0000000..13cd348
--- /dev/null
+++ b/backend/lib/Target/RISCV/RISCVCallLowering.cpp
@@ -0,0 +1,1473 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "rcc/Target/RISCV/RISCVCallLowering.h"
+
+#include "rcc/CodeGen/MachineBasicBlock.h"
+#include "rcc/CodeGen/MachineFrameInfo.h"
+#include "rcc/CodeGen/MachineFunction.h"
+#include "rcc/CodeGen/MachineIRBuilder.h"
+#include "rcc/CodeGen/MachineInstr.h"
+#include "rcc/CodeGen/MachineMemOperand.h"
+#include "rcc/CodeGen/MachineRegisterInfo.h"
+#include "rcc/CodeGen/TargetDescription.h"
+#include "rcc/Target/RISCV/RISCVTarget.h"
+
+#include <algorithm>
+#include <array>
+#include <cstddef>
+#include <cstdint>
+#include <limits>
+#include <optional>
+#include <span>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+#include <utility>
+#include <vector>
+
+namespace rcc::riscv {
+namespace {
+
+constexpr std::string_view PlanTarget = "riscv64-unknown-linux-gnu:lp64d";
+constexpr std::uint64_t XLenBytes = 8;
+constexpr std::uint64_t LP64DStackAlignmentBytes = 16;
+constexpr std::uint32_t AllowedDirectCallFlags =
+    static_cast<std::uint32_t>(CallTargetFlag::DirectCall);
+
+[[nodiscard]] Opcode targetOpcode(InstructionOpcode Value) noexcept {
+  return Opcode::fromValue(static_cast<std::uint32_t>(Value));
+}
+
+[[nodiscard]] std::uint64_t stackObjectFlag(CallStackObjectFlag Flag) noexcept {
+  return static_cast<std::uint64_t>(Flag);
+}
+
+[[nodiscard]] Expected<Alignment> alignment(std::uint64_t Bytes) {
+  return Alignment::create(Bytes);
+}
+
+[[nodiscard]] Expected<std::uint64_t> alignTo(std::uint64_t Value, std::uint64_t AlignmentBytes) {
+  const std::uint64_t Remainder = Value % AlignmentBytes;
+  if (Remainder == 0)
+    return Value;
+  const std::uint64_t Increment = AlignmentBytes - Remainder;
+  if (Value > std::numeric_limits<std::uint64_t>::max() - Increment) {
+    return makeUnexpected(ErrorCode::Unsupported, "LP64D stack-area size overflows uint64");
+  }
+  return Value + Increment;
+}
+
+[[nodiscard]] std::uint64_t valueSizeBytes(LLT Type) noexcept {
+  return (static_cast<std::uint64_t>(Type.scalarBitWidth()) + 7U) / 8U;
+}
+
+[[nodiscard]] bool isSupportedFloat(LLT Type) noexcept {
+  if (!Type.isFloat())
+    return false;
+  const auto Semantics = Type.floatSemantics();
+  return Semantics &&
+         (*Semantics == FloatSemantics::IEEESingle || *Semantics == FloatSemantics::IEEEDouble);
+}
+
+[[nodiscard]] bool requiresIntegerTransport(const ABIValue &Value) noexcept {
+  return !isSupportedFloat(Value.Type) || Value.Flags.has(ABIValueFlag::ByVal) ||
+         Value.Flags.has(ABIValueFlag::StructReturn) || Value.Flags.has(ABIValueFlag::Indirect);
+}
+
+[[nodiscard]] Expected<void> verifyABIValue(const ABIValue &Value, bool RequireRegister) {
+  if (RequireRegister && Value.Value.isNoRegister()) {
+    return makeUnexpected(ErrorCode::InvalidArgument,
+                          "LP64D ABI value requires a source or destination register");
+  }
+  if (!Value.Type.isValid() || !Value.Type.isScalarValue() || Value.Type.isAnyScalar() ||
+      Value.Type.scalarBitWidth() > 64) {
+    return makeUnexpected(
+        ErrorCode::Unsupported,
+        "LP64D accepts explicitly classified integer, pointer, f32, and f64 scalar fragments");
+  }
+  if (Value.Type.isFloat() && !isSupportedFloat(Value.Type)) {
+    return makeUnexpected(ErrorCode::Unsupported,
+                          "RV64GC LP64D only supports f32 and f64 ABI fragments");
+  }
+  if (Value.Type.isPointer() && Value.Type.scalarBitWidth() != 64) {
+    return makeUnexpected(ErrorCode::Unsupported, "RV64GC LP64D requires 64-bit machine pointers");
+  }
+  if (Value.Type.isInteger() && Value.Type.scalarBitWidth() < 64) {
+    if (Value.Extension == ABIExtensionKind::None) {
+      return makeUnexpected(ErrorCode::InvalidArgument,
+                            "narrow LP64D integer fragments require an explicit extension kind");
+    }
+  } else if (Value.Extension != ABIExtensionKind::None) {
+    return makeUnexpected(ErrorCode::InvalidArgument,
+                          "only narrow integer ABI fragments may carry an extension kind");
+  }
+  if (Value.Extension >= ABIExtensionKind::Count || Value.FragmentCount == 0 ||
+      Value.FragmentIndex >= Value.FragmentCount || Value.FragmentCount > 2) {
+    return makeUnexpected(ErrorCode::InvalidArgument,
+                          "LP64D ABI fragment coordinates or extension are invalid");
+  }
+
+  const bool IsByVal = Value.Flags.has(ABIValueFlag::ByVal);
+  const bool IsSRet = Value.Flags.has(ABIValueFlag::StructReturn);
+  const bool IsIndirect = Value.Flags.has(ABIValueFlag::Indirect);
+  const unsigned SpecialFlagCount = static_cast<unsigned>(IsByVal) + static_cast<unsigned>(IsSRet) +
+                                    static_cast<unsigned>(IsIndirect);
+  if (SpecialFlagCount > 1) {
+    return makeUnexpected(ErrorCode::InvalidArgument,
+                          "byval, sret, and indirect ABI flags are mutually exclusive");
+  }
+  if ((IsByVal || IsSRet || IsIndirect) &&
+      (!Value.Type.isPointer() || Value.FragmentCount != 1 || Value.FragmentIndex != 0 ||
+       Value.FragmentOffsetBytes != 0)) {
+    return makeUnexpected(ErrorCode::InvalidArgument,
+                          "byval, sret, and indirect values require one explicit pointer fragment");
+  }
+  if (IsByVal) {
+    if (Value.ByValSizeBytes == 0) {
+      return makeUnexpected(ErrorCode::InvalidArgument,
+                            "byval arguments require a nonzero copy size");
+    }
+  } else if (Value.ByValSizeBytes != 0) {
+    return makeUnexpected(ErrorCode::InvalidArgument,
+                          "non-byval values cannot carry a byval copy size");
+  }
+  return {};
+}
+
+[[nodiscard]] Expected<std::size_t> verifyValueSequence(std::span<const ABIValue> Values,
+                                                        bool RequireRegisters) {
+  std::size_t ExpectedOriginal = 0;
+  std::size_t Index = 0;
+  while (Index < Values.size()) {
+    const ABIValue &First = Values[Index];
+    auto ValueVerification = verifyABIValue(First, RequireRegisters);
+    if (!ValueVerification)
+      return std::unexpected(ValueVerification.error());
+    if (First.OriginalValue != ExpectedOriginal || First.FragmentIndex != 0 ||
+        Index + First.FragmentCount > Values.size()) {
+      return makeUnexpected(
+          ErrorCode::InvalidArgument,
+          "LP64D ABI values must form contiguous, zero-based original-value groups");
+    }
+
+    std::uint64_t PreviousEnd = 0;
+    for (std::size_t Fragment = 0; Fragment < First.FragmentCount; ++Fragment) {
+      const ABIValue &Value = Values[Index + Fragment];
+      auto FragmentVerification = verifyABIValue(Value, RequireRegisters);
+      if (!FragmentVerification)
+        return std::unexpected(FragmentVerification.error());
+      if (Value.OriginalValue != First.OriginalValue || Value.FragmentIndex != Fragment ||
+          Value.FragmentCount != First.FragmentCount ||
+          Value.OriginalAlignment != First.OriginalAlignment || Value.Flags != First.Flags ||
+          Value.ByValSizeBytes != First.ByValSizeBytes ||
+          Value.ByValAlignment != First.ByValAlignment || Value.FragmentOffsetBytes < PreviousEnd) {
+        return makeUnexpected(ErrorCode::InvalidArgument,
+                              "LP64D fragments of one value have inconsistent metadata");
+      }
+      PreviousEnd =
+          static_cast<std::uint64_t>(Value.FragmentOffsetBytes) + valueSizeBytes(Value.Type);
+    }
+    Index += First.FragmentCount;
+    ++ExpectedOriginal;
+  }
+  return ExpectedOriginal;
+}
+
+[[nodiscard]] CCAssignmentFlags assignmentFlags(const ABIValue &Value, bool IsVariadic,
+                                                bool IsBitcast) {
+  CCAssignmentFlags Result;
+  if (IsVariadic)
+    Result.set(CCAssignmentFlag::Variadic);
+  if (Value.Flags.has(ABIValueFlag::ByVal))
+    Result.set(CCAssignmentFlag::ByVal);
+  if (Value.Flags.has(ABIValueFlag::StructReturn))
+    Result.set(CCAssignmentFlag::StructReturn);
+  if (Value.Flags.has(ABIValueFlag::Indirect))
+    Result.set(CCAssignmentFlag::Indirect);
+  if (Value.Flags.has(ABIValueFlag::InRegister))
+    Result.set(CCAssignmentFlag::InRegister);
+  if (IsBitcast)
+    Result.set(CCAssignmentFlag::Bitcast);
+  return Result;
+}
+
+[[nodiscard]] CCLocationKind locationKind(const ABIValue &Value, bool IsRegister) noexcept {
+  if (Value.Flags.has(ABIValueFlag::Indirect))
+    return CCLocationKind::Indirect;
+  return IsRegister ? CCLocationKind::Register : CCLocationKind::Stack;
+}
+
+struct ArgumentAssignmentResult final {
+  std::vector<CCAssignment> Assignments;
+  StackArgumentArea StackArea;
+  std::size_t FirstUnallocatedGPR = 0;
+};
+
+struct AssignmentState final {
+  std::size_t GPR = 0;
+  std::size_t FPR = 0;
+  std::uint64_t StackOffset = 0;
+  bool VariadicStackOnly = false;
+};
+
+[[nodiscard]] Expected<CCAssignment> registerAssignment(const ABIValue &Value,
+                                                        std::size_t ValueIndex,
+                                                        PhysicalRegister Location, LLT LocationType,
+                                                        bool IsVariadic, bool IsBitcast) {
+  auto Align =
+      alignment(std::min<std::uint64_t>(Value.OriginalAlignment.bytes(), LP64DStackAlignmentBytes));
+  if (!Align)
+    return std::unexpected(Align.error());
+  return CCAssignment{
+      .ValueIndex = ValueIndex,
+      .OriginalValue = Value.Value,
+      .OriginalValueIndex = Value.OriginalValue,
+      .FragmentIndex = Value.FragmentIndex,
+      .FragmentCount = Value.FragmentCount,
+      .ValueType = Value.Type,
+      .LocationType = LocationType,
+      .LocationKind = locationKind(Value, true),
+      .LocationRegister = Location,
+      .StackOffsetBytes = std::nullopt,
+      .LocationSizeBytes = valueSizeBytes(LocationType),
+      .StackSlotSizeBytes = XLenBytes,
+      .LocationAlignment = *Align,
+      .Extension = Value.Extension,
+      .CustomFlags = assignmentFlags(Value, IsVariadic, IsBitcast),
+  };
+}
+
+[[nodiscard]] Expected<CCAssignment> stackAssignment(const ABIValue &Value, std::size_t ValueIndex,
+                                                     std::uint64_t StackOffset, LLT LocationType,
+                                                     bool IsVariadic,
+                                                     std::uint64_t StackAlignment) {
+  auto Align = alignment(StackAlignment);
+  if (!Align)
+    return std::unexpected(Align.error());
+  return CCAssignment{
+      .ValueIndex = ValueIndex,
+      .OriginalValue = Value.Value,
+      .OriginalValueIndex = Value.OriginalValue,
+      .FragmentIndex = Value.FragmentIndex,
+      .FragmentCount = Value.FragmentCount,
+      .ValueType = Value.Type,
+      .LocationType = LocationType,
+      .LocationKind = locationKind(Value, false),
+      .LocationRegister = std::nullopt,
+      .StackOffsetBytes = StackOffset,
+      .LocationSizeBytes = valueSizeBytes(LocationType),
+      .StackSlotSizeBytes = XLenBytes,
+      .LocationAlignment = *Align,
+      .Extension = Value.Extension,
+      .CustomFlags = assignmentFlags(Value, IsVariadic, false),
+  };
+}
+
+[[nodiscard]] Expected<ArgumentAssignmentResult>
+assignArguments(std::span<const ABIValue> Values, std::size_t FixedArgumentCount, bool IsVariadic,
+                const CallingConventionInfo &Convention) {
+  auto I64 = LLT::makeInteger(64);
+  auto StackAlign = alignment(Convention.StackArgumentAlignmentBytes);
+  if (!I64 || !StackAlign)
+    return makeUnexpected(ErrorCode::InternalError, "LP64D primitive type facts are invalid");
+
+  ArgumentAssignmentResult Result;
+  AssignmentState State;
+  State.StackOffset = Convention.ShadowSpaceBytes;
+  Result.Assignments.reserve(Values.size());
+
+  std::size_t Index = 0;
+  while (Index < Values.size()) {
+    const ABIValue &First = Values[Index];
+    const std::size_t GroupSize = First.FragmentCount;
+    const bool VariadicArgument = IsVariadic && First.OriginalValue >= FixedArgumentCount;
+    bool HasPreferredFloat = false;
+    std::size_t RequiredGPRs = 0;
+    std::size_t RequiredFPRs = 0;
+    for (std::size_t Fragment = 0; Fragment < GroupSize; ++Fragment) {
+      const ABIValue &Value = Values[Index + Fragment];
+      if (!VariadicArgument && !requiresIntegerTransport(Value)) {
+        HasPreferredFloat = true;
+        ++RequiredFPRs;
+      } else {
+        ++RequiredGPRs;
+      }
+    }
+
+    const bool CanUsePreferredAggregate =
+        HasPreferredFloat &&
+        State.GPR + RequiredGPRs <= Convention.IntegerArgumentRegisters.size() &&
+        State.FPR + RequiredFPRs <= Convention.FloatingArgumentRegisters.size();
+    if (CanUsePreferredAggregate) {
+      for (std::size_t Fragment = 0; Fragment < GroupSize; ++Fragment) {
+        const ABIValue &Value = Values[Index + Fragment];
+        Expected<CCAssignment> Assignment =
+            requiresIntegerTransport(Value)
+                ? registerAssignment(Value, Index + Fragment,
+                                     Convention.IntegerArgumentRegisters[State.GPR++], *I64, false,
+                                     false)
+                : registerAssignment(Value, Index + Fragment,
+                                     Convention.FloatingArgumentRegisters[State.FPR++], Value.Type,
+                                     false, false);
+        if (!Assignment)
+          return std::unexpected(Assignment.error());
+        Result.Assignments.push_back(*Assignment);
+      }
+      Index += GroupSize;
+      continue;
+    }
+
+    const bool RequiresAlignedPair =
+        VariadicArgument && GroupSize == 2 && First.OriginalAlignment.bytes() >= 16;
+    if (RequiresAlignedPair && State.GPR % 2 != 0 &&
+        State.GPR < Convention.IntegerArgumentRegisters.size()) {
+      ++State.GPR;
+    }
+    bool ForceEntireGroupToStack = State.VariadicStackOnly;
+    if (RequiresAlignedPair && State.GPR + GroupSize > Convention.IntegerArgumentRegisters.size()) {
+      ForceEntireGroupToStack = true;
+      State.GPR = Convention.IntegerArgumentRegisters.size();
+    }
+
+    bool FirstStackFragment = true;
+    for (std::size_t Fragment = 0; Fragment < GroupSize; ++Fragment) {
+      const ABIValue &Value = Values[Index + Fragment];
+      const bool HasGPR =
+          !ForceEntireGroupToStack && State.GPR < Convention.IntegerArgumentRegisters.size();
+      if (HasGPR) {
+        const bool Bitcast = isSupportedFloat(Value.Type);
+        auto Assignment = registerAssignment(Value, Index + Fragment,
+                                             Convention.IntegerArgumentRegisters[State.GPR++], *I64,
+                                             VariadicArgument, Bitcast);
+        if (!Assignment)
+          return std::unexpected(Assignment.error());
+        Result.Assignments.push_back(*Assignment);
+        continue;
+      }
+
+      if (Value.Flags.has(ABIValueFlag::InRegister)) {
+        return makeUnexpected(ErrorCode::Unsupported,
+                              "LP64D inreg argument cannot be assigned to the stack");
+      }
+      std::uint64_t RequiredAlignment = XLenBytes;
+      if (FirstStackFragment && ForceEntireGroupToStack) {
+        RequiredAlignment = std::min<std::uint64_t>(
+            std::max<std::uint64_t>(XLenBytes, First.OriginalAlignment.bytes()),
+            LP64DStackAlignmentBytes);
+      }
+      auto AlignedOffset = alignTo(State.StackOffset, RequiredAlignment);
+      if (!AlignedOffset)
+        return std::unexpected(AlignedOffset.error());
+      State.StackOffset = *AlignedOffset;
+      const LLT LocationType = isSupportedFloat(Value.Type) ? Value.Type : *I64;
+      auto Assignment = stackAssignment(Value, Index + Fragment, State.StackOffset, LocationType,
+                                        VariadicArgument, RequiredAlignment);
+      if (!Assignment)
+        return std::unexpected(Assignment.error());
+      Result.Assignments.push_back(*Assignment);
+      if (State.StackOffset > std::numeric_limits<std::uint64_t>::max() - XLenBytes) {
+        return makeUnexpected(ErrorCode::Unsupported, "LP64D argument area overflows uint64");
+      }
+      State.StackOffset += XLenBytes;
+      FirstStackFragment = false;
+      if (VariadicArgument)
+        State.VariadicStackOnly = true;
+    }
+    Index += GroupSize;
+  }
+
+  auto FinalSize = alignTo(State.StackOffset, Convention.StackArgumentAlignmentBytes);
+  if (!FinalSize)
+    return std::unexpected(FinalSize.error());
+  Result.StackArea = StackArgumentArea{*FinalSize, Convention.ShadowSpaceBytes, *StackAlign};
+  Result.FirstUnallocatedGPR = State.GPR;
+  return Result;
+}
+
+[[nodiscard]] Expected<std::vector<CCAssignment>>
+assignReturns(std::span<const ABIValue> Values, bool RequireRegisters,
+              const CallingConventionInfo &Convention) {
+  auto Sequence = verifyValueSequence(Values, RequireRegisters);
+  if (!Sequence)
+    return std::unexpected(Sequence.error());
+  if (Values.size() > 2) {
+    return makeUnexpected(ErrorCode::Unsupported,
+                          "LP64D direct returns are limited to two explicit fragments");
+  }
+  auto I64 = LLT::makeInteger(64);
+  if (!I64)
+    return std::unexpected(I64.error());
+
+  std::vector<CCAssignment> Result;
+  Result.reserve(Values.size());
+  std::size_t GPR = 0;
+  std::size_t FPR = 0;
+  for (std::size_t Index = 0; Index < Values.size(); ++Index) {
+    const ABIValue &Value = Values[Index];
+    if (Value.Flags.rawValue() != 0) {
+      return makeUnexpected(ErrorCode::InvalidArgument,
+                            "return fragments cannot carry argument-only ABI flags");
+    }
+    Expected<CCAssignment> Assignment =
+        isSupportedFloat(Value.Type)
+            ? registerAssignment(Value, Index, Convention.FloatingReturnRegisters[FPR++],
+                                 Value.Type, false, false)
+            : registerAssignment(Value, Index, Convention.IntegerReturnRegisters[GPR++], *I64,
+                                 false, false);
+    if (!Assignment)
+      return std::unexpected(Assignment.error());
+    Result.push_back(*Assignment);
+  }
+  return Result;
+}
+
+[[nodiscard]] bool hasStructReturn(std::span<const ABIValue> Arguments) noexcept {
+  return std::ranges::any_of(
+      Arguments, [](const ABIValue &Value) { return Value.Flags.has(ABIValueFlag::StructReturn); });
+}
+
+[[nodiscard]] bool equivalentReturnLocations(std::span<const CCAssignment> Left,
+                                             std::span<const CCAssignment> Right) noexcept {
+  if (Left.size() != Right.size())
+    return false;
+  for (std::size_t Index = 0; Index < Left.size(); ++Index) {
+    if (Left[Index].ValueType != Right[Index].ValueType ||
+        Left[Index].LocationType != Right[Index].LocationType ||
+        Left[Index].LocationKind != Right[Index].LocationKind ||
+        Left[Index].LocationRegister != Right[Index].LocationRegister ||
+        Left[Index].Extension != Right[Index].Extension ||
+        Left[Index].CustomFlags != Right[Index].CustomFlags) {
+      return false;
+    }
+  }
+  return true;
+}
+
+[[nodiscard]] Expected<std::string>
+tailCallRejection(const CallLoweringRequest &Request, const ArgumentAssignmentResult &Arguments,
+                  std::span<const CCAssignment> Results,
+                  std::span<const ByValCopyAssignment> ByValCopies,
+                  const CallingConventionInfo &Convention) {
+  if (!Convention.SupportsTailCalls)
+    return std::string("the calling convention does not support tail calls");
+  if (!Request.TailContext)
+    return std::string("tail-call context is missing");
+  const TailCallContext &Context = *Request.TailContext;
+  if (Context.CallerConvention != Request.Convention)
+    return std::string("caller and callee calling conventions differ");
+  if (Context.CallerIsVariadic != Request.IsVariadic ||
+      (Request.IsVariadic && Context.CallerFixedArgumentCount != Request.FixedArgumentCount)) {
+    return std::string("caller and callee variadic signatures differ");
+  }
+  if (Context.CallerHasDynamicStackAllocation)
+    return std::string("the caller has a dynamic stack allocation");
+  if (!ByValCopies.empty())
+    return std::string("tail calls cannot create byval copies");
+  if (Context.CallerUsesStructReturn != hasStructReturn(Request.Arguments))
+    return std::string("caller and callee sret conventions differ");
+  if (Arguments.StackArea.SizeBytes > Context.ReusableIncomingArgumentBytes)
+    return std::string("outgoing stack arguments exceed the reusable incoming area");
+  if (std::ranges::any_of(Request.Results,
+                          [](const ABIValue &Value) { return !Value.Value.isNoRegister(); })) {
+    return std::string("tail-call result signatures must not request local result copies");
+  }
+  auto CallerReturns = assignReturns(Context.CallerReturns, false, Convention);
+  if (!CallerReturns)
+    return std::unexpected(CallerReturns.error());
+  if (!equivalentReturnLocations(Results, *CallerReturns))
+    return std::string("caller and callee return locations differ");
+  return std::string{};
+}
+
+[[nodiscard]] Expected<const CallingConventionInfo *>
+lp64dConvention(CallingConventionID Convention) {
+  if (Convention != callingConventionID(CallingConvention::LP64D)) {
+    return makeUnexpected(ErrorCode::Unsupported,
+                          "RISC-V backend only supports the LP64D calling convention");
+  }
+  const CallingConventionInfo *Info = callingConvention(CallingConvention::LP64D);
+  if (Info == nullptr)
+    return makeUnexpected(ErrorCode::InternalError, "LP64D target facts are unavailable");
+  return Info;
+}
+
+class RISCVCallLowering final : public TargetCallLowering {
+public:
+  [[nodiscard]] Expected<CallLoweringPlan>
+  planFormalArguments(const FormalArgumentRequest &Request) const override;
+  [[nodiscard]] Expected<CallLoweringPlan>
+  planCall(const CallLoweringRequest &Request) const override;
+  [[nodiscard]] Expected<CallLoweringPlan>
+  planReturn(const ReturnLoweringRequest &Request) const override;
+  [[nodiscard]] Expected<void> emitPlan(MachineIRBuilder &Builder,
+                                        const CallLoweringPlan &Plan) const override;
+};
+
+Expected<CallLoweringPlan>
+RISCVCallLowering::planFormalArguments(const FormalArgumentRequest &Request) const {
+  auto Convention = lp64dConvention(Request.Convention);
+  if (!Convention)
+    return std::unexpected(Convention.error());
+  auto OriginalCount = verifyValueSequence(Request.Arguments, true);
+  if (!OriginalCount)
+    return std::unexpected(OriginalCount.error());
+  if (Request.FixedArgumentCount != *OriginalCount) {
+    return makeUnexpected(ErrorCode::InvalidArgument,
+                          "formal-argument request must identify every declared argument as fixed");
+  }
+  if (!Request.IsVariadic && Request.FixedArgumentCount != *OriginalCount) {
+    return makeUnexpected(ErrorCode::InvalidArgument,
+                          "non-variadic formal request has an invalid fixed-argument boundary");
+  }
+  if (hasStructReturn(Request.Arguments) &&
+      (Request.Arguments.empty() || Request.Arguments.front().OriginalValue != 0 ||
+       !Request.Arguments.front().Flags.has(ABIValueFlag::StructReturn))) {
+    return makeUnexpected(ErrorCode::InvalidArgument,
+                          "LP64D sret must be the first formal argument");
+  }
+
+  auto Assignments = assignArguments(Request.Arguments, Request.FixedArgumentCount,
+                                     Request.IsVariadic, **Convention);
+  if (!Assignments)
+    return std::unexpected(Assignments.error());
+
+  VarArgLoweringState VarArgs;
+  if (Request.IsVariadic) {
+    VarArgs.IsVariadic = true;
+    VarArgs.FixedArgumentCount = Request.FixedArgumentCount;
+    VarArgs.FirstUnallocatedIntegerRegister = Assignments->FirstUnallocatedGPR;
+    VarArgs.OverflowAreaOffsetBytes = Assignments->StackArea.SizeBytes;
+    const std::size_t First = Assignments->FirstUnallocatedGPR;
+    const std::size_t RegisterCount = (*Convention)->IntegerArgumentRegisters.size();
+    VarArgs.RegisterSaveAreaSizeBytes = (RegisterCount - First) * XLenBytes;
+    VarArgs.RegisterSaveAreaOffsetBytes =
+        VarArgs.RegisterSaveAreaSizeBytes == 0
+            ? static_cast<std::int64_t>(VarArgs.OverflowAreaOffsetBytes)
+            : -static_cast<std::int64_t>(VarArgs.RegisterSaveAreaSizeBytes);
+    if (VarArgs.RegisterSaveAreaSizeBytes != 0 && First % 2 != 0)
+      VarArgs.RegisterSavePaddingBytes = XLenBytes;
+    for (std::size_t Index = First; Index < RegisterCount; ++Index) {
+      VarArgs.RegistersToSave.push_back(
+          {(*Convention)->IntegerArgumentRegisters[Index], (Index - First) * XLenBytes});
+    }
+  }
+
+  CallLoweringPlanDesc Plan;
+  Plan.Kind = CallLoweringPlanKind::FormalArguments;
+  Plan.Target = PlanTarget;
+  Plan.Convention = Request.Convention;
+  Plan.ArgumentAssignments = std::move(Assignments->Assignments);
+  Plan.StackArguments = Assignments->StackArea;
+  Plan.VarArgs = std::move(VarArgs);
+  return CallLoweringPlan::create(std::move(Plan));
+}
+
+Expected<CallLoweringPlan> RISCVCallLowering::planCall(const CallLoweringRequest &Request) const {
+  auto Convention = lp64dConvention(Request.Convention);
+  if (!Convention)
+    return std::unexpected(Convention.error());
+  auto OriginalCount = verifyValueSequence(Request.Arguments, true);
+  if (!OriginalCount)
+    return std::unexpected(OriginalCount.error());
+  if ((!Request.IsVariadic && Request.FixedArgumentCount != *OriginalCount) ||
+      (Request.IsVariadic && Request.FixedArgumentCount > *OriginalCount)) {
+    return makeUnexpected(ErrorCode::InvalidArgument,
+                          "call request has an invalid fixed/variadic argument boundary");
+  }
+  if (Request.CallSiteID == 0 || !Request.Callee.isValid()) {
+    return makeUnexpected(ErrorCode::InvalidArgument,
+                          "call request requires a valid callee and nonzero call-site ID");
+  }
+  if (hasStructReturn(Request.Arguments) &&
+      (Request.Arguments.empty() ||
+       !Request.Arguments.front().Flags.has(ABIValueFlag::StructReturn))) {
+    return makeUnexpected(ErrorCode::InvalidArgument, "LP64D sret must be the first call argument");
+  }
+  if (hasStructReturn(Request.Arguments) && !Request.Results.empty()) {
+    return makeUnexpected(ErrorCode::InvalidArgument,
+                          "LP64D sret calls cannot also request direct return values");
+  }
+
+  MachineCallTarget EffectiveCallee = Request.Callee;
+  switch (Request.Callee.kind()) {
+  case MachineCallTargetKind::GlobalSymbol: {
+    if ((Request.Callee.targetFlags() & ~AllowedDirectCallFlags) != 0) {
+      return makeUnexpected(ErrorCode::InvalidArgument,
+                            "direct RISC-V call uses unknown target flags");
+    }
+    auto Callee = MachineCallTarget::globalSymbol(
+        std::string(Request.Callee.symbolName()),
+        Request.Callee.targetFlags() | static_cast<std::uint32_t>(CallTargetFlag::DirectCall));
+    if (!Callee)
+      return std::unexpected(Callee.error());
+    EffectiveCallee = std::move(*Callee);
+    break;
+  }
+  case MachineCallTargetKind::ExternalSymbol: {
+    if ((Request.Callee.targetFlags() & ~AllowedDirectCallFlags) != 0) {
+      return makeUnexpected(ErrorCode::InvalidArgument,
+                            "direct RISC-V call uses unknown target flags");
+    }
+    auto Callee = MachineCallTarget::externalSymbol(
+        std::string(Request.Callee.symbolName()),
+        Request.Callee.targetFlags() | static_cast<std::uint32_t>(CallTargetFlag::DirectCall));
+    if (!Callee)
+      return std::unexpected(Callee.error());
+    EffectiveCallee = std::move(*Callee);
+    break;
+  }
+  case MachineCallTargetKind::IndirectRegister:
+    if (Request.Callee.targetFlags() != 0) {
+      return makeUnexpected(ErrorCode::InvalidArgument,
+                            "indirect RISC-V calls cannot carry direct-call target flags");
+    }
+    break;
+  case MachineCallTargetKind::Invalid:
+    return makeUnexpected(ErrorCode::InvalidArgument, "call target is invalid");
+  }
+
+  const bool RequireResultRegisters = Request.TailKind == TailCallKind::NotTail;
+  auto ResultSequence = verifyValueSequence(Request.Results, RequireResultRegisters);
+  if (!ResultSequence)
+    return std::unexpected(ResultSequence.error());
+  auto Arguments = assignArguments(Request.Arguments, Request.FixedArgumentCount,
+                                   Request.IsVariadic, **Convention);
+  if (!Arguments)
+    return std::unexpected(Arguments.error());
+  auto Results = assignReturns(Request.Results, RequireResultRegisters, **Convention);
+  if (!Results)
+    return std::unexpected(Results.error());
+
+  auto Align16 = alignment(LP64DStackAlignmentBytes);
+  if (!Align16)
+    return std::unexpected(Align16.error());
+  std::uint64_t CallFrameCursor = Arguments->StackArea.SizeBytes;
+  std::vector<ByValCopyAssignment> ByValCopies;
+  for (std::size_t Index = 0; Index < Arguments->Assignments.size(); ++Index) {
+    const CCAssignment &Assignment = Arguments->Assignments[Index];
+    if (!Assignment.CustomFlags.has(CCAssignmentFlag::ByVal))
+      continue;
+    const ABIValue &Value = Request.Arguments[Assignment.ValueIndex];
+    const std::uint64_t CopyAlignment =
+        std::min<std::uint64_t>(Value.ByValAlignment.bytes(), LP64DStackAlignmentBytes);
+    auto CopyOffset = alignTo(CallFrameCursor, CopyAlignment);
+    if (!CopyOffset)
+      return std::unexpected(CopyOffset.error());
+    if (*CopyOffset > std::numeric_limits<std::uint64_t>::max() - Value.ByValSizeBytes) {
+      return makeUnexpected(ErrorCode::Unsupported, "LP64D byval area overflows uint64");
+    }
+    auto CopyAlign = alignment(CopyAlignment);
+    if (!CopyAlign)
+      return std::unexpected(CopyAlign.error());
+    ByValCopies.push_back(
+        {Index, Value.Value, Value.Type, *CopyOffset, Value.ByValSizeBytes, *CopyAlign});
+    CallFrameCursor = *CopyOffset + Value.ByValSizeBytes;
+  }
+  auto CallFrameSize = alignTo(CallFrameCursor, LP64DStackAlignmentBytes);
+  if (!CallFrameSize)
+    return std::unexpected(CallFrameSize.error());
+
+  TailCallKind Decision = TailCallKind::NotTail;
+  std::string RejectionReason;
+  if (Request.TailKind != TailCallKind::NotTail) {
+    auto Rejection = tailCallRejection(Request, *Arguments, *Results, ByValCopies, **Convention);
+    if (!Rejection)
+      return std::unexpected(Rejection.error());
+    RejectionReason = std::move(*Rejection);
+    if (RejectionReason.empty()) {
+      Decision = Request.TailKind;
+    } else if (Request.TailKind == TailCallKind::MustTail) {
+      return makeUnexpected(ErrorCode::Unsupported,
+                            "musttail call is not ABI-compatible: " + RejectionReason);
+    } else {
+      auto NormalResults = verifyValueSequence(Request.Results, true);
+      if (!NormalResults)
+        return std::unexpected(NormalResults.error());
+    }
+  }
+
+  CallLoweringPlanDesc Plan;
+  Plan.Kind = CallLoweringPlanKind::Call;
+  Plan.Target = PlanTarget;
+  Plan.Convention = Request.Convention;
+  Plan.ArgumentAssignments = std::move(Arguments->Assignments);
+  Plan.ReturnAssignments = std::move(*Results);
+  Plan.StackArguments = Arguments->StackArea;
+  Plan.CallFrame = CallFrameRequirements{
+      Decision == TailCallKind::NotTail ? *CallFrameSize : Arguments->StackArea.SizeBytes, *Align16,
+      Decision == TailCallKind::NotTail, Decision == TailCallKind::NotTail,
+      Decision != TailCallKind::NotTail};
+  if (Request.IsVariadic) {
+    Plan.VarArgs.IsVariadic = true;
+    Plan.VarArgs.FixedArgumentCount = Request.FixedArgumentCount;
+  }
+  Plan.ByValCopies = std::move(ByValCopies);
+  Plan.PreservedMask = createCallPreservedMask();
+  Plan.Callee = std::move(EffectiveCallee);
+  Plan.CallSiteID = Request.CallSiteID;
+  Plan.TailDecision = Decision;
+  Plan.TailRejectionReason = std::move(RejectionReason);
+  return CallLoweringPlan::create(std::move(Plan));
+}
+
+Expected<CallLoweringPlan>
+RISCVCallLowering::planReturn(const ReturnLoweringRequest &Request) const {
+  auto Convention = lp64dConvention(Request.Convention);
+  if (!Convention)
+    return std::unexpected(Convention.error());
+  auto Values = assignReturns(Request.Values, true, **Convention);
+  if (!Values)
+    return std::unexpected(Values.error());
+
+  CallLoweringPlanDesc Plan;
+  Plan.Kind = CallLoweringPlanKind::Return;
+  Plan.Target = PlanTarget;
+  Plan.Convention = Request.Convention;
+  Plan.ReturnAssignments = std::move(*Values);
+  return CallLoweringPlan::create(std::move(Plan));
+}
+
+[[nodiscard]] CallValueTransform outgoingTransform(const CCAssignment &Assignment) noexcept {
+  if (Assignment.CustomFlags.has(CCAssignmentFlag::Bitcast))
+    return CallValueTransform::Bitcast;
+  switch (Assignment.Extension) {
+  case ABIExtensionKind::None:
+    return CallValueTransform::Full;
+  case ABIExtensionKind::AnyExtend:
+    return CallValueTransform::AnyExtend;
+  case ABIExtensionKind::SignExtend:
+    return CallValueTransform::SignExtend;
+  case ABIExtensionKind::ZeroExtend:
+    return CallValueTransform::ZeroExtend;
+  case ABIExtensionKind::Count:
+    break;
+  }
+  return CallValueTransform::Full;
+}
+
+[[nodiscard]] CallValueTransform incomingTransform(const CCAssignment &Assignment) noexcept {
+  if (Assignment.CustomFlags.has(CCAssignmentFlag::Bitcast))
+    return CallValueTransform::Bitcast;
+  if (Assignment.ValueType.scalarBitWidth() < Assignment.LocationType.scalarBitWidth())
+    return CallValueTransform::Truncate;
+  return CallValueTransform::Full;
+}
+
+[[nodiscard]] Expected<void> addUnsignedImmediate(MachineInstrBuilder &Builder,
+                                                  std::uint64_t Value) {
+  auto Immediate = IntegerImmediate::fromUnsigned(64, Value);
+  if (!Immediate)
+    return std::unexpected(Immediate.error());
+  return Builder.addIntegerImmediate(std::move(*Immediate));
+}
+
+[[nodiscard]] Expected<const MachineMemOperand *>
+createMemoryOperand(MachineFunction &Function, MemoryAccessFlags Accesses, std::uint64_t SizeBytes,
+                    Alignment Align, MemoryBase Base, std::int64_t Offset = 0) {
+  auto Size = MemorySize::fixed(SizeBytes);
+  if (!Size)
+    return std::unexpected(Size.error());
+  MachineMemOperandDesc Description;
+  Description.Accesses = Accesses;
+  Description.Size = *Size;
+  Description.Align = Align;
+  Description.Base = std::move(Base);
+  Description.Offset = Offset;
+  return Function.createMemOperand(std::move(Description));
+}
+
+[[nodiscard]] Expected<void> emitCCCopy(MachineIRBuilder &Builder, Register Destination,
+                                        Register Source, CallValueTransform Transform) {
+  auto Instruction = Builder.buildInstr(targetOpcode(InstructionOpcode::PseudoCC_COPY));
+  if (!Instruction)
+    return std::unexpected(Instruction.error());
+  auto Def = Instruction->addDef(Destination);
+  if (!Def)
+    return std::unexpected(Def.error());
+  auto Use = Instruction->addUse(Source);
+  if (!Use)
+    return std::unexpected(Use.error());
+  auto Action = Instruction->addImmediate(static_cast<std::int64_t>(Transform));
+  if (!Action)
+    return std::unexpected(Action.error());
+  auto Built = Instruction->build();
+  if (!Built)
+    return std::unexpected(Built.error());
+  return {};
+}
+
+[[nodiscard]] Expected<void> emitStackLoad(MachineIRBuilder &Builder,
+                                           const CCAssignment &Assignment, FrameIndex Object,
+                                           std::uint64_t ObjectOffset,
+                                           const MachineMemOperand &Memory) {
+  auto Instruction = Builder.buildInstr(targetOpcode(InstructionOpcode::PseudoLOAD_STACK_ARG));
+  if (!Instruction)
+    return std::unexpected(Instruction.error());
+  auto Def = Instruction->addDef(Assignment.OriginalValue);
+  if (!Def)
+    return std::unexpected(Def.error());
+  auto Frame = Instruction->addFrameIndex(Object);
+  if (!Frame)
+    return std::unexpected(Frame.error());
+  auto Offset = addUnsignedImmediate(*Instruction, ObjectOffset);
+  if (!Offset)
+    return std::unexpected(Offset.error());
+  auto Transform =
+      Instruction->addImmediate(static_cast<std::int64_t>(incomingTransform(Assignment)));
+  if (!Transform)
+    return std::unexpected(Transform.error());
+  auto Mem = Instruction->addMemOperand(Memory);
+  if (!Mem)
+    return std::unexpected(Mem.error());
+  auto Built = Instruction->build();
+  if (!Built)
+    return std::unexpected(Built.error());
+  return {};
+}
+
+[[nodiscard]] Expected<void> emitStackStore(MachineIRBuilder &Builder, Register Source,
+                                            FrameIndex Object, std::uint64_t ObjectOffset,
+                                            const MachineMemOperand &Memory,
+                                            CallValueTransform Transform) {
+  auto Instruction = Builder.buildInstr(targetOpcode(InstructionOpcode::PseudoSTORE_STACK_ARG));
+  if (!Instruction)
+    return std::unexpected(Instruction.error());
+  auto Use = Instruction->addUse(Source);
+  if (!Use)
+    return std::unexpected(Use.error());
+  auto Frame = Instruction->addFrameIndex(Object);
+  if (!Frame)
+    return std::unexpected(Frame.error());
+  auto Offset = addUnsignedImmediate(*Instruction, ObjectOffset);
+  if (!Offset)
+    return std::unexpected(Offset.error());
+  auto Action = Instruction->addImmediate(static_cast<std::int64_t>(Transform));
+  if (!Action)
+    return std::unexpected(Action.error());
+  auto Mem = Instruction->addMemOperand(Memory);
+  if (!Mem)
+    return std::unexpected(Mem.error());
+  auto Built = Instruction->build();
+  if (!Built)
+    return std::unexpected(Built.error());
+  return {};
+}
+
+[[nodiscard]] Expected<void> ensureFunctionLiveIn(MachineRegisterInfo &Registers,
+                                                  PhysicalRegister Reg,
+                                                  std::optional<Register> InitialValue) {
+  const auto Existing =
+      std::ranges::find(Registers.liveIns(), Reg, &PhysicalRegisterLiveIn::PhysReg);
+  if (Existing == Registers.liveIns().end())
+    return Registers.addLiveIn(Reg, InitialValue);
+  if (InitialValue && Existing->InitialVirtualRegister &&
+      Existing->InitialVirtualRegister != InitialValue) {
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "physical formal register already maps to another virtual live-in");
+  }
+  return {};
+}
+
+[[nodiscard]] Expected<void> ensureBlockLiveIn(MachineBasicBlock &Block, PhysicalRegister Reg) {
+  return Block.addLiveIn(Reg, LaneMask::all());
+}
+
+[[nodiscard]] Expected<void> verifyPlanRegisters(const CallLoweringPlan &Plan,
+                                                 const MachineFunction &Function,
+                                                 const TargetDescription &Target) {
+  const MachineRegisterInfo &Registers = Function.registerInfo();
+  const auto Verify = [&Registers, &Target](const CCAssignment &Assignment) -> Expected<void> {
+    if (Assignment.LocationRegister &&
+        !Target.registers().isKnownPhysicalRegister(*Assignment.LocationRegister)) {
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "call-lowering plan names an unknown ABI location register");
+    }
+    const Register Reg = Assignment.OriginalValue;
+    if (Reg.isNoRegister())
+      return {};
+    if (Reg.isPhysical()) {
+      if (!Target.registers().isKnownPhysicalRegister(*Reg.physicalRegister())) {
+        return makeUnexpected(ErrorCode::VerificationError,
+                              "call-lowering plan names an unknown physical value register");
+      }
+      return {};
+    }
+    auto Info = Registers.virtualRegisterInfo(Reg);
+    if (!Info)
+      return std::unexpected(Info.error());
+    if (!Info->get().Type || *Info->get().Type != Assignment.ValueType) {
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "call-lowering plan register LLT differs from MachineRegisterInfo");
+    }
+    return {};
+  };
+  for (const CCAssignment &Assignment : Plan.argumentAssignments()) {
+    auto Result = Verify(Assignment);
+    if (!Result)
+      return std::unexpected(Result.error());
+  }
+  for (const CCAssignment &Assignment : Plan.returnAssignments()) {
+    auto Result = Verify(Assignment);
+    if (!Result)
+      return std::unexpected(Result.error());
+  }
+  if (const MachineCallTarget *Callee = Plan.callee();
+      Callee != nullptr && Callee->kind() == MachineCallTargetKind::IndirectRegister) {
+    const Register CalleeReg = Callee->indirectRegister();
+    if (CalleeReg.isPhysical()) {
+      const std::optional<PhysicalRegister> Physical = CalleeReg.physicalRegister();
+      if (!Physical || !Target.registers().isKnownPhysicalRegister(*Physical)) {
+        return makeUnexpected(ErrorCode::VerificationError,
+                              "indirect LP64D callee names an unknown physical register");
+      }
+    } else {
+      auto Info = Registers.virtualRegisterInfo(CalleeReg);
+      if (!Info)
+        return std::unexpected(Info.error());
+      const std::optional<LLT> Type = Info->get().Type;
+      if (!Type || ((!Type->isPointer() && !Type->isInteger()) || Type->scalarBitWidth() != 64)) {
+        return makeUnexpected(ErrorCode::VerificationError,
+                              "indirect LP64D callee requires a 64-bit pointer or integer LLT");
+      }
+    }
+  }
+  return {};
+}
+
+[[nodiscard]] Expected<void> verifyRequiredOpcodes(const MachineIRBuilder &Builder) {
+  static constexpr std::array Required{
+      InstructionOpcode::PseudoCC_COPY,
+      InstructionOpcode::PseudoLOAD_STACK_ARG,
+      InstructionOpcode::PseudoSTORE_STACK_ARG,
+      InstructionOpcode::PseudoBYVAL_COPY,
+      InstructionOpcode::ADJCALLSTACKDOWN,
+      InstructionOpcode::ADJCALLSTACKUP,
+      InstructionOpcode::PseudoCALL,
+      InstructionOpcode::PseudoTAIL,
+      InstructionOpcode::PseudoRET,
+  };
+  for (const InstructionOpcode OpcodeValue : Required) {
+    if (Builder.target().instructionDescriptor(targetOpcode(OpcodeValue)) == nullptr) {
+      return makeUnexpected(ErrorCode::InternalError,
+                            "RISC-V target description lacks a call-lowering pseudo");
+    }
+  }
+  return {};
+}
+
+[[nodiscard]] Expected<void> verifyFormalLiveIns(const CallLoweringPlan &Plan,
+                                                 const MachineFunction &Function) {
+  if (Plan.kind() != CallLoweringPlanKind::FormalArguments)
+    return {};
+
+  const auto LiveIns = Function.registerInfo().liveIns();
+  for (const CCAssignment &Assignment : Plan.argumentAssignments()) {
+    if (!Assignment.LocationRegister || !Assignment.OriginalValue.isVirtual())
+      continue;
+    const auto Existing =
+        std::ranges::find(LiveIns, *Assignment.LocationRegister, &PhysicalRegisterLiveIn::PhysReg);
+    if (Existing != LiveIns.end() && Existing->InitialVirtualRegister &&
+        *Existing->InitialVirtualRegister != Assignment.OriginalValue) {
+      return makeUnexpected(ErrorCode::VerificationError,
+                            "physical formal register already maps to another virtual live-in");
+    }
+  }
+  return {};
+}
+
+[[nodiscard]] Expected<void> preflightPlan(MachineIRBuilder &Builder,
+                                           const CallLoweringPlan &Plan) {
+  if (Plan.target() != PlanTarget ||
+      Plan.convention() != callingConventionID(CallingConvention::LP64D)) {
+    return makeUnexpected(ErrorCode::InvalidArgument,
+                          "call-lowering plan belongs to another target or ABI");
+  }
+  auto InsertPoint = Builder.verifyInsertPoint();
+  if (!InsertPoint)
+    return std::unexpected(InsertPoint.error());
+  auto Opcodes = verifyRequiredOpcodes(Builder);
+  if (!Opcodes)
+    return std::unexpected(Opcodes.error());
+  auto Registers = verifyPlanRegisters(Plan, Builder.function(), Builder.target());
+  if (!Registers)
+    return std::unexpected(Registers.error());
+  auto LiveIns = verifyFormalLiveIns(Plan, Builder.function());
+  if (!LiveIns)
+    return std::unexpected(LiveIns.error());
+  for (const CCAssignment &Assignment : Plan.argumentAssignments()) {
+    if (Assignment.StackOffsetBytes &&
+        *Assignment.StackOffsetBytes >
+            static_cast<std::uint64_t>(std::numeric_limits<std::int64_t>::max())) {
+      return makeUnexpected(ErrorCode::Unsupported,
+                            "LP64D stack assignment offset exceeds the frame-index range");
+    }
+  }
+  for (const ByValCopyAssignment &Copy : Plan.byValCopies()) {
+    if (Copy.StackOffsetBytes >
+        static_cast<std::uint64_t>(std::numeric_limits<std::int64_t>::max())) {
+      return makeUnexpected(ErrorCode::Unsupported,
+                            "LP64D byval offset exceeds the frame-index range");
+    }
+  }
+  if (Plan.kind() != CallLoweringPlanKind::Return && Builder.function().frameInfo().isFrozen()) {
+    return makeUnexpected(ErrorCode::VerificationError,
+                          "call lowering cannot modify a frozen machine frame");
+  }
+  if (Plan.kind() == CallLoweringPlanKind::FormalArguments &&
+      (Builder.function().firstBlock() == nullptr ||
+       Builder.insertionBlock() != Builder.function().firstBlock())) {
+    return makeUnexpected(ErrorCode::InvalidArgument,
+                          "formal arguments must be lowered in the entry machine block");
+  }
+  return {};
+}
+
+[[nodiscard]] Expected<void> updateMaximumFrameAlignment(MachineFrameState &State,
+                                                         Alignment Required) {
+  if (!State.MaxAlignment || State.MaxAlignment->bytes() < Required.bytes())
+    State.MaxAlignment = Required;
+  return {};
+}
+
+[[nodiscard]] Expected<void> emitFormalArguments(MachineIRBuilder &Builder,
+                                                 const CallLoweringPlan &Plan) {
+  MachineFunction &Function = Builder.function();
+  MachineBasicBlock &Entry = *Function.firstBlock();
+  MachineRegisterInfo &Registers = Function.registerInfo();
+  MachineFrameInfo &Frame = Function.frameInfo();
+  Alignment RequiredFrameAlignment;
+
+  for (const CCAssignment &Assignment : Plan.argumentAssignments()) {
+    if (Assignment.LocationRegister) {
+      const PhysicalRegister Location = *Assignment.LocationRegister;
+      const std::optional<Register> Initial =
+          Assignment.OriginalValue.isVirtual() ? std::optional<Register>(Assignment.OriginalValue)
+                                               : std::nullopt;
+      auto FunctionLiveIn = ensureFunctionLiveIn(Registers, Location, Initial);
+      if (!FunctionLiveIn)
+        return std::unexpected(FunctionLiveIn.error());
+      auto BlockLiveIn = ensureBlockLiveIn(Entry, Location);
+      if (!BlockLiveIn)
+        return std::unexpected(BlockLiveIn.error());
+      auto Copy = emitCCCopy(Builder, Assignment.OriginalValue, Register::fromPhysical(Location),
+                             incomingTransform(Assignment));
+      if (!Copy)
+        return std::unexpected(Copy.error());
+      continue;
+    }
+
+    if (!Assignment.StackOffsetBytes) {
+      return makeUnexpected(ErrorCode::InternalError,
+                            "verified formal stack assignment has no stack offset");
+    }
+    StackObjectDesc Object;
+    Object.Kind = StackObjectKind::FixedABI;
+    Object.Size = Assignment.StackSlotSizeBytes;
+    Object.Align = Assignment.LocationAlignment;
+    Object.Offset = static_cast<std::int64_t>(*Assignment.StackOffsetBytes);
+    Object.IsImmutable = true;
+    Object.TargetFlags = stackObjectFlag(CallStackObjectFlag::IncomingArgument);
+    auto FrameObject = Frame.createObject(std::move(Object));
+    if (!FrameObject)
+      return std::unexpected(FrameObject.error());
+    auto Memory =
+        createMemoryOperand(Function, {MemoryAccessFlag::Load}, Assignment.LocationSizeBytes,
+                            Assignment.LocationAlignment, MemoryBase::frameIndex(*FrameObject));
+    if (!Memory)
+      return std::unexpected(Memory.error());
+    auto Load = emitStackLoad(Builder, Assignment, *FrameObject, 0, **Memory);
+    if (!Load)
+      return std::unexpected(Load.error());
+    if (RequiredFrameAlignment.bytes() < Assignment.LocationAlignment.bytes())
+      RequiredFrameAlignment = Assignment.LocationAlignment;
+  }
+
+  const VarArgLoweringState &VarArgs = Plan.varArgs();
+  if (!VarArgs.IsVariadic) {
+    if (RequiredFrameAlignment.bytes() == 1)
+      return {};
+    MachineFrameState State = Frame.state();
+    auto AlignmentUpdate = updateMaximumFrameAlignment(State, RequiredFrameAlignment);
+    if (!AlignmentUpdate)
+      return std::unexpected(AlignmentUpdate.error());
+    return Frame.setState(std::move(State));
+  }
+
+  auto Align8 = alignment(XLenBytes);
+  auto Align16 = alignment(LP64DStackAlignmentBytes);
+  if (!Align8 || !Align16)
+    return makeUnexpected(ErrorCode::InternalError, "LP64D primitive type facts are invalid");
+
+  FrameIndex SaveObject;
+  if (VarArgs.RegisterSaveAreaSizeBytes == 0) {
+    StackObjectDesc Object;
+    Object.Kind = StackObjectKind::FixedABI;
+    Object.Size = XLenBytes;
+    Object.Align = *Align8;
+    Object.Offset = VarArgs.RegisterSaveAreaOffsetBytes;
+    Object.IsImmutable = true;
+    Object.TargetFlags = stackObjectFlag(CallStackObjectFlag::VarArgSaveArea);
+    auto Created = Frame.createObject(std::move(Object));
+    if (!Created)
+      return std::unexpected(Created.error());
+    SaveObject = *Created;
+  } else {
+    StackObjectDesc Object;
+    Object.Kind = StackObjectKind::FixedABI;
+    Object.Size = VarArgs.RegisterSaveAreaSizeBytes;
+    Object.Align = VarArgs.FirstUnallocatedIntegerRegister % 2 == 0 ? *Align16 : *Align8;
+    Object.Offset = VarArgs.RegisterSaveAreaOffsetBytes;
+    Object.TargetFlags = stackObjectFlag(CallStackObjectFlag::VarArgSaveArea);
+    auto Created = Frame.createObject(std::move(Object));
+    if (!Created)
+      return std::unexpected(Created.error());
+    SaveObject = *Created;
+
+    if (VarArgs.RegisterSavePaddingBytes != 0) {
+      StackObjectDesc Padding;
+      Padding.Kind = StackObjectKind::FixedABI;
+      Padding.Size = VarArgs.RegisterSavePaddingBytes;
+      Padding.Align = *Align8;
+      Padding.Offset = VarArgs.RegisterSaveAreaOffsetBytes -
+                       static_cast<std::int64_t>(VarArgs.RegisterSavePaddingBytes);
+      Padding.IsImmutable = true;
+      Padding.TargetFlags = stackObjectFlag(CallStackObjectFlag::VarArgAlignmentPadding);
+      auto CreatedPadding = Frame.createObject(std::move(Padding));
+      if (!CreatedPadding)
+        return std::unexpected(CreatedPadding.error());
+    }
+  }
+
+  for (const VarArgRegisterSave Save : VarArgs.RegistersToSave) {
+    auto FunctionLiveIn = ensureFunctionLiveIn(Registers, Save.Reg, std::nullopt);
+    if (!FunctionLiveIn)
+      return std::unexpected(FunctionLiveIn.error());
+    auto BlockLiveIn = ensureBlockLiveIn(Entry, Save.Reg);
+    if (!BlockLiveIn)
+      return std::unexpected(BlockLiveIn.error());
+    auto Memory = createMemoryOperand(Function, {MemoryAccessFlag::Store}, XLenBytes, *Align8,
+                                      MemoryBase::frameIndex(SaveObject),
+                                      static_cast<std::int64_t>(Save.OffsetBytes));
+    if (!Memory)
+      return std::unexpected(Memory.error());
+    auto Store = emitStackStore(Builder, Register::fromPhysical(Save.Reg), SaveObject,
+                                Save.OffsetBytes, **Memory, CallValueTransform::Full);
+    if (!Store)
+      return std::unexpected(Store.error());
+  }
+
+  MachineFrameState State = Frame.state();
+  const Alignment VarArgAlignment =
+      RequiredFrameAlignment.bytes() > Align16->bytes() ? RequiredFrameAlignment : *Align16;
+  auto AlignmentUpdate = updateMaximumFrameAlignment(State, VarArgAlignment);
+  if (!AlignmentUpdate)
+    return std::unexpected(AlignmentUpdate.error());
+  return Frame.setState(std::move(State));
+}
+
+[[nodiscard]] Expected<void> emitCallFrameAdjustment(MachineIRBuilder &Builder,
+                                                     InstructionOpcode OpcodeValue,
+                                                     const CallFrameRequirements &Frame,
+                                                     MachineInstrFlag Flag) {
+  auto Instruction = Builder.buildInstr(targetOpcode(OpcodeValue));
+  if (!Instruction)
+    return std::unexpected(Instruction.error());
+  auto Size = addUnsignedImmediate(*Instruction, Frame.SizeBytes);
+  if (!Size)
+    return std::unexpected(Size.error());
+  auto Align = addUnsignedImmediate(*Instruction, Frame.Align.bytes());
+  if (!Align)
+    return std::unexpected(Align.error());
+  auto Flags = Instruction->setFlags({Flag});
+  if (!Flags)
+    return std::unexpected(Flags.error());
+  auto Built = Instruction->build();
+  if (!Built)
+    return std::unexpected(Built.error());
+  return {};
+}
+
+[[nodiscard]] Expected<std::unordered_map<std::size_t, Register>>
+emitByValCopies(MachineIRBuilder &Builder, const CallLoweringPlan &Plan) {
+  MachineFunction &Function = Builder.function();
+  MachineFrameInfo &Frame = Function.frameInfo();
+  std::unordered_map<std::size_t, Register> Replacements;
+  for (const ByValCopyAssignment Copy : Plan.byValCopies()) {
+    StackObjectDesc Object;
+    Object.Kind = StackObjectKind::Target;
+    Object.Size = Copy.SizeBytes;
+    Object.Align = Copy.Align;
+    Object.LocalOffset = static_cast<std::int64_t>(Copy.StackOffsetBytes);
+    Object.TargetFlags = stackObjectFlag(CallStackObjectFlag::ByValCopy);
+    auto FrameObject = Frame.createObject(std::move(Object));
+    if (!FrameObject)
+      return std::unexpected(FrameObject.error());
+    auto Temporary = Function.registerInfo().createVirtualRegister(Copy.PointerType);
+    if (!Temporary)
+      return std::unexpected(Temporary.error());
+
+    auto SourceMemory = createMemoryOperand(Function, {MemoryAccessFlag::Load}, Copy.SizeBytes,
+                                            Copy.Align, MemoryBase{});
+    if (!SourceMemory)
+      return std::unexpected(SourceMemory.error());
+    auto DestinationMemory =
+        createMemoryOperand(Function, {MemoryAccessFlag::Store}, Copy.SizeBytes, Copy.Align,
+                            MemoryBase::frameIndex(*FrameObject));
+    if (!DestinationMemory)
+      return std::unexpected(DestinationMemory.error());
+
+    auto Instruction = Builder.buildInstr(targetOpcode(InstructionOpcode::PseudoBYVAL_COPY));
+    if (!Instruction)
+      return std::unexpected(Instruction.error());
+    auto Def = Instruction->addDef(*Temporary);
+    if (!Def)
+      return std::unexpected(Def.error());
+    auto Use = Instruction->addUse(Copy.Source);
+    if (!Use)
+      return std::unexpected(Use.error());
+    auto FrameOperand = Instruction->addFrameIndex(*FrameObject);
+    if (!FrameOperand)
+      return std::unexpected(FrameOperand.error());
+    auto Size = addUnsignedImmediate(*Instruction, Copy.SizeBytes);
+    if (!Size)
+      return std::unexpected(Size.error());
+    auto Align = addUnsignedImmediate(*Instruction, Copy.Align.bytes());
+    if (!Align)
+      return std::unexpected(Align.error());
+    auto SourceMem = Instruction->addMemOperand(**SourceMemory);
+    if (!SourceMem)
+      return std::unexpected(SourceMem.error());
+    auto DestinationMem = Instruction->addMemOperand(**DestinationMemory);
+    if (!DestinationMem)
+      return std::unexpected(DestinationMem.error());
+    auto Built = Instruction->build();
+    if (!Built)
+      return std::unexpected(Built.error());
+    Replacements.emplace(Copy.ArgumentAssignmentIndex, *Temporary);
+  }
+  return Replacements;
+}
+
+[[nodiscard]] Expected<void>
+emitOutgoingArguments(MachineIRBuilder &Builder, const CallLoweringPlan &Plan,
+                      const std::unordered_map<std::size_t, Register> &Replacements) {
+  MachineFunction &Function = Builder.function();
+  MachineFrameInfo &Frame = Function.frameInfo();
+  const auto Assignments = Plan.argumentAssignments();
+  for (std::size_t Index = 0; Index < Assignments.size(); ++Index) {
+    const CCAssignment &Assignment = Assignments[Index];
+    const auto Replacement = Replacements.find(Index);
+    const Register Source =
+        Replacement == Replacements.end() ? Assignment.OriginalValue : Replacement->second;
+    if (Assignment.LocationRegister) {
+      auto Copy = emitCCCopy(Builder, Register::fromPhysical(*Assignment.LocationRegister), Source,
+                             outgoingTransform(Assignment));
+      if (!Copy)
+        return std::unexpected(Copy.error());
+      continue;
+    }
+
+    if (!Assignment.StackOffsetBytes) {
+      return makeUnexpected(ErrorCode::InternalError,
+                            "verified outgoing stack assignment has no stack offset");
+    }
+    StackObjectDesc Object;
+    Object.Size = Assignment.StackSlotSizeBytes;
+    Object.Align = Assignment.LocationAlignment;
+    Object.TargetFlags = stackObjectFlag(CallStackObjectFlag::OutgoingArgument);
+    if (Plan.tailDecision() == TailCallKind::NotTail) {
+      Object.Kind = StackObjectKind::Target;
+      Object.LocalOffset = static_cast<std::int64_t>(*Assignment.StackOffsetBytes);
+    } else {
+      Object.Kind = StackObjectKind::FixedABI;
+      Object.Offset = static_cast<std::int64_t>(*Assignment.StackOffsetBytes);
+    }
+    auto FrameObject = Frame.createObject(std::move(Object));
+    if (!FrameObject)
+      return std::unexpected(FrameObject.error());
+    auto Memory =
+        createMemoryOperand(Function, {MemoryAccessFlag::Store}, Assignment.LocationSizeBytes,
+                            Assignment.LocationAlignment, MemoryBase::frameIndex(*FrameObject));
+    if (!Memory)
+      return std::unexpected(Memory.error());
+    auto Store =
+        emitStackStore(Builder, Source, *FrameObject, 0, **Memory, outgoingTransform(Assignment));
+    if (!Store)
+      return std::unexpected(Store.error());
+  }
+  return {};
+}
+
+[[nodiscard]] Expected<void> addCallTarget(MachineInstrBuilder &Instruction,
+                                           const MachineCallTarget &Callee) {
+  switch (Callee.kind()) {
+  case MachineCallTargetKind::GlobalSymbol:
+    return Instruction.addGlobalSymbol(std::string(Callee.symbolName()), 0, Callee.targetFlags());
+  case MachineCallTargetKind::ExternalSymbol:
+    return Instruction.addExternalSymbol(std::string(Callee.symbolName()), 0, Callee.targetFlags());
+  case MachineCallTargetKind::IndirectRegister:
+    return Instruction.addUse(Callee.indirectRegister());
+  case MachineCallTargetKind::Invalid:
+    break;
+  }
+  return makeUnexpected(ErrorCode::InternalError,
+                        "verified call-lowering plan has an invalid callee");
+}
+
+[[nodiscard]] Expected<PhysicalRegister> requiredLocationRegister(const CCAssignment &Assignment) {
+  if (!Assignment.LocationRegister) {
+    return makeUnexpected(ErrorCode::InternalError,
+                          "verified return assignment has no location register");
+  }
+  return *Assignment.LocationRegister;
+}
+
+[[nodiscard]] Expected<void> emitCallInstruction(MachineIRBuilder &Builder,
+                                                 const CallLoweringPlan &Plan) {
+  const bool IsTail = Plan.tailDecision() != TailCallKind::NotTail;
+  auto Instruction = Builder.buildInstr(
+      targetOpcode(IsTail ? InstructionOpcode::PseudoTAIL : InstructionOpcode::PseudoCALL));
+  if (!Instruction)
+    return std::unexpected(Instruction.error());
+  auto Callee = addCallTarget(*Instruction, *Plan.callee());
+  if (!Callee)
+    return std::unexpected(Callee.error());
+  auto CallSite = addUnsignedImmediate(*Instruction, Plan.callSiteID());
+  if (!CallSite)
+    return std::unexpected(CallSite.error());
+  auto Mask = Instruction->addRegisterMask(Plan.preservedMask());
+  if (!Mask)
+    return std::unexpected(Mask.error());
+
+  std::unordered_set<PhysicalRegister> DynamicDefs;
+  if (!IsTail) {
+    for (const CCAssignment &Assignment : Plan.returnAssignments()) {
+      if (Assignment.LocationRegister && DynamicDefs.insert(*Assignment.LocationRegister).second) {
+        auto Def =
+            Instruction->addImplicitDef(Register::fromPhysical(*Assignment.LocationRegister));
+        if (!Def)
+          return std::unexpected(Def.error());
+      }
+    }
+  }
+  std::unordered_set<PhysicalRegister> DynamicUses;
+  for (const CCAssignment &Assignment : Plan.argumentAssignments()) {
+    if (Assignment.LocationRegister && DynamicUses.insert(*Assignment.LocationRegister).second) {
+      auto Use = Instruction->addImplicitUse(Register::fromPhysical(*Assignment.LocationRegister));
+      if (!Use)
+        return std::unexpected(Use.error());
+    }
+  }
+  auto Built = Instruction->build();
+  if (!Built)
+    return std::unexpected(Built.error());
+  return {};
+}
+
+[[nodiscard]] Expected<void> emitCall(MachineIRBuilder &Builder, const CallLoweringPlan &Plan) {
+  if (Plan.callFrame().EmitSetup) {
+    auto Down = emitCallFrameAdjustment(Builder, InstructionOpcode::ADJCALLSTACKDOWN,
+                                        Plan.callFrame(), MachineInstrFlag::FrameSetup);
+    if (!Down)
+      return std::unexpected(Down.error());
+  }
+
+  auto Replacements = emitByValCopies(Builder, Plan);
+  if (!Replacements)
+    return std::unexpected(Replacements.error());
+  auto Arguments = emitOutgoingArguments(Builder, Plan, *Replacements);
+  if (!Arguments)
+    return std::unexpected(Arguments.error());
+  auto Call = emitCallInstruction(Builder, Plan);
+  if (!Call)
+    return std::unexpected(Call.error());
+
+  if (Plan.callFrame().EmitDestroy) {
+    auto Up = emitCallFrameAdjustment(Builder, InstructionOpcode::ADJCALLSTACKUP, Plan.callFrame(),
+                                      MachineInstrFlag::FrameDestroy);
+    if (!Up)
+      return std::unexpected(Up.error());
+  }
+  if (Plan.tailDecision() == TailCallKind::NotTail) {
+    for (const CCAssignment &Assignment : Plan.returnAssignments()) {
+      auto Location = requiredLocationRegister(Assignment);
+      if (!Location)
+        return std::unexpected(Location.error());
+      auto Copy = emitCCCopy(Builder, Assignment.OriginalValue, Register::fromPhysical(*Location),
+                             incomingTransform(Assignment));
+      if (!Copy)
+        return std::unexpected(Copy.error());
+    }
+  }
+
+  MachineFrameInfo &Frame = Builder.function().frameInfo();
+  MachineFrameState State = Frame.state();
+  State.HasCalls = true;
+  State.HasTailCall = State.HasTailCall || Plan.tailDecision() != TailCallKind::NotTail;
+  State.AdjustsStack =
+      State.AdjustsStack || (Plan.callFrame().EmitSetup && Plan.callFrame().SizeBytes != 0);
+  State.HasMustTailInVarArgFunc =
+      State.HasMustTailInVarArgFunc ||
+      (Plan.tailDecision() == TailCallKind::MustTail && Plan.varArgs().IsVariadic);
+  if (Plan.tailDecision() == TailCallKind::NotTail) {
+    State.MaxCallFrameSize =
+        std::max(State.MaxCallFrameSize.value_or(0), Plan.callFrame().SizeBytes);
+  }
+  auto AlignmentUpdate = updateMaximumFrameAlignment(State, Plan.callFrame().Align);
+  if (!AlignmentUpdate)
+    return std::unexpected(AlignmentUpdate.error());
+  return Frame.setState(std::move(State));
+}
+
+[[nodiscard]] Expected<void> emitReturn(MachineIRBuilder &Builder, const CallLoweringPlan &Plan) {
+  for (const CCAssignment &Assignment : Plan.returnAssignments()) {
+    auto Location = requiredLocationRegister(Assignment);
+    if (!Location)
+      return std::unexpected(Location.error());
+    auto Copy = emitCCCopy(Builder, Register::fromPhysical(*Location), Assignment.OriginalValue,
+                           outgoingTransform(Assignment));
+    if (!Copy)
+      return std::unexpected(Copy.error());
+  }
+
+  auto Return = Builder.buildInstr(targetOpcode(InstructionOpcode::PseudoRET));
+  if (!Return)
+    return std::unexpected(Return.error());
+  for (const CCAssignment &Assignment : Plan.returnAssignments()) {
+    auto Location = requiredLocationRegister(Assignment);
+    if (!Location)
+      return std::unexpected(Location.error());
+    auto Use = Return->addImplicitUse(Register::fromPhysical(*Location));
+    if (!Use)
+      return std::unexpected(Use.error());
+  }
+  auto Built = Return->build();
+  if (!Built)
+    return std::unexpected(Built.error());
+  return {};
+}
+
+Expected<void> RISCVCallLowering::emitPlan(MachineIRBuilder &Builder,
+                                           const CallLoweringPlan &Plan) const {
+  auto Preflight = preflightPlan(Builder, Plan);
+  if (!Preflight)
+    return std::unexpected(Preflight.error());
+  switch (Plan.kind()) {
+  case CallLoweringPlanKind::FormalArguments:
+    return emitFormalArguments(Builder, Plan);
+  case CallLoweringPlanKind::Call:
+    return emitCall(Builder, Plan);
+  case CallLoweringPlanKind::Return:
+    return emitReturn(Builder, Plan);
+  }
+  return makeUnexpected(ErrorCode::InternalError, "call-lowering plan kind is invalid");
+}
+
+} // namespace
+
+const TargetCallLowering &callLowering() noexcept {
+  static const RISCVCallLowering Lowering;
+  return Lowering;
+}
+
+} // namespace rcc::riscv