aboutsummaryrefslogtreecommitdiff
path: root/backend/lib/Target/RISCV
diff options
context:
space:
mode:
Diffstat (limited to 'backend/lib/Target/RISCV')
-rw-r--r--backend/lib/Target/RISCV/CMakeLists.txt3
-rw-r--r--backend/lib/Target/RISCV/RISCVFrameLowering.cpp2019
-rw-r--r--backend/lib/Target/RISCV/RISCVRegisterAllocation.cpp84
3 files changed, 2106 insertions, 0 deletions
diff --git a/backend/lib/Target/RISCV/CMakeLists.txt b/backend/lib/Target/RISCV/CMakeLists.txt
index 1d9a27a..442e75e 100644
--- a/backend/lib/Target/RISCV/CMakeLists.txt
+++ b/backend/lib/Target/RISCV/CMakeLists.txt
@@ -49,6 +49,7 @@ add_library(
RCCBackendRISCV
RISCVCodeGen.cpp
RISCVCallLowering.cpp
+ RISCVFrameLowering.cpp
RISCVRegisterAllocation.cpp
RISCVTarget.cpp
"${RCCBackendRISCVGeneratedRegisterHeader}"
@@ -94,6 +95,7 @@ install(
rcc_backend_register_format_files(
"${CMAKE_CURRENT_SOURCE_DIR}/RISCVCodeGen.cpp"
"${CMAKE_CURRENT_SOURCE_DIR}/RISCVCallLowering.cpp"
+ "${CMAKE_CURRENT_SOURCE_DIR}/RISCVFrameLowering.cpp"
"${CMAKE_CURRENT_SOURCE_DIR}/RISCVRegisterAllocation.cpp"
"${CMAKE_CURRENT_SOURCE_DIR}/RISCVTarget.cpp"
"${RCC_BACKEND_SOURCE_DIR}/include/rcc/Target/RISCV/RISCVCodeGen.h"
@@ -103,6 +105,7 @@ rcc_backend_register_format_files(
rcc_backend_register_tidy_files(
"${CMAKE_CURRENT_SOURCE_DIR}/RISCVCodeGen.cpp"
"${CMAKE_CURRENT_SOURCE_DIR}/RISCVCallLowering.cpp"
+ "${CMAKE_CURRENT_SOURCE_DIR}/RISCVFrameLowering.cpp"
"${CMAKE_CURRENT_SOURCE_DIR}/RISCVRegisterAllocation.cpp"
"${CMAKE_CURRENT_SOURCE_DIR}/RISCVTarget.cpp"
)
diff --git a/backend/lib/Target/RISCV/RISCVFrameLowering.cpp b/backend/lib/Target/RISCV/RISCVFrameLowering.cpp
new file mode 100644
index 0000000..0e6fec9
--- /dev/null
+++ b/backend/lib/Target/RISCV/RISCVFrameLowering.cpp
@@ -0,0 +1,2019 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "rcc/CodeGen/GenericOpcodes.h"
+#include "rcc/CodeGen/InstructionDescriptor.h"
+#include "rcc/CodeGen/MachineBasicBlock.h"
+#include "rcc/CodeGen/MachineCFIInfo.h"
+#include "rcc/CodeGen/MachineFrameInfo.h"
+#include "rcc/CodeGen/MachineFunction.h"
+#include "rcc/CodeGen/MachineIRBuilder.h"
+#include "rcc/CodeGen/MachineInstr.h"
+#include "rcc/CodeGen/MachineMemOperand.h"
+#include "rcc/CodeGen/MachineOperand.h"
+#include "rcc/CodeGen/RegisterMask.h"
+#include "rcc/CodeGen/RegisterScavenger.h"
+#include "rcc/CodeGen/TargetDescription.h"
+#include "rcc/Target/RISCV/RISCVCallLowering.h"
+#include "rcc/Target/RISCV/RISCVCodeGen.h"
+#include "rcc/Target/RISCV/RISCVInstructions.h"
+#include "rcc/Target/RISCV/RISCVRegisters.h"
+#include "rcc/Target/RISCV/RISCVTarget.h"
+
+#include <algorithm>
+#include <array>
+#include <bit>
+#include <cstddef>
+#include <cstdint>
+#include <iterator>
+#include <limits>
+#include <optional>
+#include <ranges>
+#include <span>
+#include <unordered_set>
+#include <utility>
+#include <vector>
+
+namespace rcc::riscv {
+namespace {
+
+constexpr std::uint64_t StackAlignmentBytes = 16;
+constexpr std::int64_t MinSigned12 = -2048;
+constexpr std::int64_t MaxSigned12 = 2047;
+constexpr std::uint8_t DwarfCFAExpression = 0x10U;
+constexpr std::uint8_t DwarfOpBaseRegister0 = 0x70U;
+constexpr std::uint8_t DwarfOpBaseRegisterExtended = 0x92U;
+
+[[nodiscard]] constexpr Opcode targetOpcode(InstructionOpcode Value) noexcept {
+ return Opcode::fromValue(static_cast<std::uint32_t>(Value));
+}
+
+[[nodiscard]] Expected<PhysicalRegister> physical(PhysicalRegisterID ID) {
+ return physicalRegister(ID);
+}
+
+[[nodiscard]] constexpr bool fitsSigned12(std::int64_t Value) noexcept {
+ return Value >= MinSigned12 && Value <= MaxSigned12;
+}
+
+[[nodiscard]] constexpr bool hasStackObjectFlag(const StackObjectDesc &Object,
+ CallStackObjectFlag Flag) noexcept {
+ return (Object.TargetFlags & static_cast<std::uint64_t>(Flag)) != 0;
+}
+
+[[nodiscard]] constexpr bool isOutgoingCallObject(const StackObjectDesc &Object) noexcept {
+ return Object.Kind == StackObjectKind::Target &&
+ (hasStackObjectFlag(Object, CallStackObjectFlag::OutgoingArgument) ||
+ hasStackObjectFlag(Object, CallStackObjectFlag::ByValCopy));
+}
+
+[[nodiscard]] Expected<std::int64_t> checkedAdd(std::int64_t Left, std::int64_t Right) {
+ std::int64_t Result = 0;
+ if (__builtin_add_overflow(Left, Right, &Result))
+ return makeUnexpected(ErrorCode::FrameLayoutFailed, "RISC-V frame offset overflows int64");
+ return Result;
+}
+
+[[nodiscard]] Expected<std::uint64_t> alignTo(std::uint64_t Value, std::uint64_t Align) {
+ if (Align == 0 || !std::has_single_bit(Align))
+ return makeUnexpected(ErrorCode::InvalidArgument, "RISC-V frame alignment is invalid");
+ const std::uint64_t Mask = Align - 1U;
+ if (Value > std::numeric_limits<std::uint64_t>::max() - Mask)
+ return makeUnexpected(ErrorCode::FrameLayoutFailed, "RISC-V frame size overflows uint64");
+ return (Value + Mask) & ~Mask;
+}
+
+[[nodiscard]] Expected<std::int64_t> immediateValue(const MachineOperand &Operand) {
+ if (const auto *Immediate = Operand.getIf<std::int64_t>())
+ return *Immediate;
+ const auto *Integer = Operand.getIf<IntegerImmediateOperand>();
+ if (Integer == nullptr || Integer->Value.bitWidth() == 0) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V frame operand requires an integer immediate");
+ }
+ if (Integer->Value.bitWidth() > 64) {
+ return makeUnexpected(ErrorCode::Unsupported, "RISC-V frame immediate must fit in 64 bits");
+ }
+ std::uint64_t Raw = Integer->Value.words().empty() ? 0 : Integer->Value.words().front();
+ const std::uint32_t Width = Integer->Value.bitWidth();
+ if (Width < 64 && (Raw & (std::uint64_t{1} << (Width - 1U))) != 0)
+ Raw |= ~std::uint64_t{0} << Width;
+ return std::bit_cast<std::int64_t>(Raw);
+}
+
+struct CallFrameRequirement final {
+ std::int64_t Amount = 0;
+ std::int64_t Alignment = 0;
+ std::int64_t AlignedAmount = 0;
+};
+
+[[nodiscard]] Expected<CallFrameRequirement> callFrameRequirement(const MachineInstr &Instruction) {
+ if (Instruction.operandCount() != 2) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V call-frame pseudo requires exactly amount and alignment");
+ }
+ auto Amount = immediateValue(*Instruction.operandAt(0));
+ auto RequestedAlignment = immediateValue(*Instruction.operandAt(1));
+ if (!Amount || !RequestedAlignment || *Amount < 0 || *RequestedAlignment <= 0 ||
+ !std::has_single_bit(static_cast<std::uint64_t>(*RequestedAlignment))) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V call-frame amount or alignment is invalid");
+ }
+ const std::uint64_t Alignment =
+ std::max(StackAlignmentBytes, static_cast<std::uint64_t>(*RequestedAlignment));
+ auto Aligned = alignTo(static_cast<std::uint64_t>(*Amount), Alignment);
+ if (!Aligned || *Aligned > static_cast<std::uint64_t>(std::numeric_limits<std::int64_t>::max())) {
+ return makeUnexpected(ErrorCode::FrameLayoutFailed, "RISC-V call-frame amount is too large");
+ }
+ return CallFrameRequirement{*Amount, *RequestedAlignment, static_cast<std::int64_t>(*Aligned)};
+}
+
+[[nodiscard]] Expected<std::int64_t> callFrameAmount(const MachineInstr &Instruction) {
+ auto Requirement = callFrameRequirement(Instruction);
+ if (!Requirement)
+ return std::unexpected(Requirement.error());
+ return Requirement->AlignedAmount;
+}
+
+[[nodiscard]] bool hasVariableObject(const MachineFunction &Function) noexcept {
+ return std::ranges::any_of(Function.frameInfo().objects(), [](const StackObject &Object) {
+ return !Object.isTombstone() && !Object.description()->IsDead && Object.isVariableSized();
+ });
+}
+
+[[nodiscard]] std::uint64_t requiredStackAlignment(const MachineFunction &Function) noexcept {
+ std::uint64_t Result = StackAlignmentBytes;
+ Result =
+ std::max(Result, Function.frameInfo().state().MaxAlignment.value_or(Alignment{}).bytes());
+ for (const StackObject &Object : Function.frameInfo().objects()) {
+ if (!Object.isTombstone() && !Object.description()->IsDead)
+ Result = std::max(Result, Object.description()->Align.bytes());
+ }
+ for (const MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr;
+ Block = Block->next()) {
+ for (const MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr;
+ Instruction = Instruction->next()) {
+ if (Instruction->opcode() != targetOpcode(InstructionOpcode::ADJCALLSTACKDOWN) &&
+ Instruction->opcode() != targetOpcode(InstructionOpcode::ADJCALLSTACKUP)) {
+ continue;
+ }
+ if (Instruction->operandCount() < 2)
+ continue;
+ auto Requested = immediateValue(*Instruction->operandAt(1));
+ if (Requested && *Requested > 0 &&
+ std::has_single_bit(static_cast<std::uint64_t>(*Requested))) {
+ Result = std::max(Result, static_cast<std::uint64_t>(*Requested));
+ }
+ }
+ }
+ return Result;
+}
+
+[[nodiscard]] bool requiresStackRealignment(const MachineFunction &Function) noexcept {
+ return requiredStackAlignment(Function) > StackAlignmentBytes;
+}
+
+[[nodiscard]] bool requiresFramePointer(const MachineFunction &Function) noexcept {
+ const MachineFrameState &State = Function.frameInfo().state();
+ if (State.IsFrameAddressTaken || State.HasOpaqueSPAdjustment || hasVariableObject(Function) ||
+ requiresStackRealignment(Function))
+ return true;
+ switch (State.FramePointerPolicy) {
+ case MachineFramePointerPolicy::All:
+ return true;
+ case MachineFramePointerPolicy::NonLeaf:
+ case MachineFramePointerPolicy::NonLeafNoReserve:
+ return State.HasCalls;
+ case MachineFramePointerPolicy::None:
+ case MachineFramePointerPolicy::Reserved:
+ case MachineFramePointerPolicy::Count:
+ return false;
+ }
+ return false;
+}
+
+[[nodiscard]] bool requiresBasePointer(const MachineFunction &Function) noexcept {
+ return requiresStackRealignment(Function);
+}
+
+[[nodiscard]] Expected<void> validateFrameRegisterUses(const MachineFunction &Function,
+ const TargetDescription &Target) {
+ std::vector<PhysicalRegister> FrameRegisters;
+ if (requiresFramePointer(Function)) {
+ auto FP = physical(PhysicalRegisterID::X8);
+ if (!FP)
+ return std::unexpected(FP.error());
+ FrameRegisters.push_back(*FP);
+ }
+ if (requiresBasePointer(Function)) {
+ auto BP = physical(PhysicalRegisterID::X9);
+ if (!BP)
+ return std::unexpected(BP.error());
+ FrameRegisters.push_back(*BP);
+ }
+ for (const MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr;
+ Block = Block->next()) {
+ for (const MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr;
+ Instruction = Instruction->next()) {
+ for (const MachineOperand &Operand : Instruction->operands()) {
+ const Register *Reg = Operand.getIf<Register>();
+ if (Reg == nullptr || !Reg->isPhysical())
+ continue;
+ const auto Physical = Reg->physicalRegister();
+ if (Physical && std::ranges::any_of(FrameRegisters, [&](PhysicalRegister FrameReg) {
+ return Target.registers().regsOverlap(*Physical, FrameReg);
+ })) {
+ return makeUnexpected(
+ ErrorCode::VerificationError,
+ "RISC-V post-allocation code uses a register reserved for frame lowering");
+ }
+ }
+ }
+ }
+ if (!requiresStackRealignment(Function))
+ return {};
+ auto OldFPScratch = physical(PhysicalRegisterID::X5);
+ if (!OldFPScratch)
+ return std::unexpected(OldFPScratch.error());
+ const auto OverlapsScratch = [&](PhysicalRegister Reg) {
+ return Target.registers().regsOverlap(Reg, *OldFPScratch);
+ };
+ if (std::ranges::any_of(Function.registerInfo().liveIns(), [&](const PhysicalRegisterLiveIn &In) {
+ return OverlapsScratch(In.PhysReg);
+ })) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V realignment scratch register is a function live-in");
+ }
+ const MachineBasicBlock *Entry = Function.firstBlock();
+ if (Entry == nullptr)
+ return {};
+ if (std::ranges::any_of(Entry->liveIns(), [&](const MachineBasicBlockLiveIn &In) {
+ return OverlapsScratch(In.PhysReg);
+ })) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V realignment scratch register is an entry live-in");
+ }
+ bool Defined = false;
+ for (const MachineInstr *Instruction = Entry->firstInstruction(); Instruction != nullptr;
+ Instruction = Instruction->next()) {
+ for (const MachineOperand &Operand : Instruction->operands()) {
+ const Register *Reg = Operand.getIf<Register>();
+ const auto Physical =
+ Reg != nullptr && Reg->isPhysical() ? Reg->physicalRegister() : std::nullopt;
+ if (Physical && OverlapsScratch(*Physical) && Operand.isUse() && !Operand.isDebugUse() &&
+ !Operand.registerFlags().has(RegisterOperandFlag::Undef) && !Defined) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V realignment scratch register is used before its first def");
+ }
+ }
+ for (const MachineOperand &Operand : Instruction->operands()) {
+ const Register *Reg = Operand.getIf<Register>();
+ const auto Physical =
+ Reg != nullptr && Reg->isPhysical() ? Reg->physicalRegister() : std::nullopt;
+ Defined |= Physical && OverlapsScratch(*Physical) && Operand.isDef();
+ if (const auto *Mask = Operand.getIf<RegisterMaskOperand>();
+ Mask != nullptr && Mask->Mask && Mask->Mask->clobbers(*OldFPScratch)) {
+ Defined = true;
+ }
+ }
+ }
+ return {};
+}
+
+[[nodiscard]] bool hasReservedCallFrame(const MachineFunction &Function) noexcept {
+ const MachineFrameState &State = Function.frameInfo().state();
+ return !hasVariableObject(Function) && !State.HasOpaqueSPAdjustment;
+}
+
+[[nodiscard]] Expected<void> computeMaxCallFrameSize(MachineFunction &Function) {
+ std::uint64_t Maximum = Function.frameInfo().maxCallFrameSize().value_or(0);
+ bool SawPseudo = false;
+ for (const MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr;
+ Block = Block->next()) {
+ std::optional<CallFrameRequirement> Active;
+ for (const MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr;
+ Instruction = Instruction->next()) {
+ const bool IsDown =
+ Instruction->opcode() == targetOpcode(InstructionOpcode::ADJCALLSTACKDOWN);
+ const bool IsUp = Instruction->opcode() == targetOpcode(InstructionOpcode::ADJCALLSTACKUP);
+ if (!IsDown && !IsUp)
+ continue;
+ SawPseudo = true;
+ auto Requirement = callFrameRequirement(*Instruction);
+ if (!Requirement)
+ return std::unexpected(Requirement.error());
+ if (IsDown) {
+ if (Active) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V call-frame sequences must not be nested");
+ }
+ Active = *Requirement;
+ Maximum = std::max(Maximum, static_cast<std::uint64_t>(Requirement->AlignedAmount));
+ continue;
+ }
+ if (!Active) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V call-frame destroy has no matching setup");
+ }
+ if (Active->Amount != Requirement->Amount || Active->Alignment != Requirement->Alignment) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V call-frame setup and destroy requirements differ");
+ }
+ Active.reset();
+ }
+ if (Active) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V call-frame setup has no matching destroy");
+ }
+ }
+ if (!SawPseudo && !Function.frameInfo().maxCallFrameSize())
+ return {};
+ return Function.frameInfo().setMaxCallFrameSize(Maximum);
+}
+
+[[nodiscard]] std::array<PhysicalRegisterID, 25> saveOrder() noexcept {
+ return {
+ PhysicalRegisterID::X1, PhysicalRegisterID::X8, PhysicalRegisterID::X9,
+ PhysicalRegisterID::X18, PhysicalRegisterID::X19, PhysicalRegisterID::X20,
+ PhysicalRegisterID::X21, PhysicalRegisterID::X22, PhysicalRegisterID::X23,
+ PhysicalRegisterID::X24, PhysicalRegisterID::X25, PhysicalRegisterID::X26,
+ PhysicalRegisterID::X27, PhysicalRegisterID::F8, PhysicalRegisterID::F9,
+ PhysicalRegisterID::F18, PhysicalRegisterID::F19, PhysicalRegisterID::F20,
+ PhysicalRegisterID::F21, PhysicalRegisterID::F22, PhysicalRegisterID::F23,
+ PhysicalRegisterID::F24, PhysicalRegisterID::F25, PhysicalRegisterID::F26,
+ PhysicalRegisterID::F27,
+ };
+}
+
+[[nodiscard]] Expected<std::vector<PhysicalRegister>>
+modifiedCalleeSavedRegisters(const MachineFunction &Function, const TargetDescription &Target) {
+ std::unordered_set<PhysicalRegister> Modified;
+ for (const StackObject &Object : Function.frameInfo().objects()) {
+ if (!Object.isTombstone() && !Object.description()->IsDead &&
+ Object.description()->CalleeSavedRegister) {
+ Modified.insert(*Object.description()->CalleeSavedRegister);
+ }
+ }
+ for (const MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr;
+ Block = Block->next()) {
+ for (const MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr;
+ Instruction = Instruction->next()) {
+ for (const MachineOperand &Operand : Instruction->operands()) {
+ if (const auto *Mask = Operand.getIf<RegisterMaskOperand>(); Mask != nullptr) {
+ if (!Mask->Mask) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V callee-save analysis found a null register mask");
+ }
+ for (const PhysicalRegisterID ID : saveOrder()) {
+ auto Reg = physical(ID);
+ if (!Reg)
+ return std::unexpected(Reg.error());
+ if (Mask->Mask->clobbers(*Reg))
+ Modified.insert(*Reg);
+ }
+ }
+ const Register *Reg = Operand.getIf<Register>();
+ if (Reg == nullptr || !Reg->isPhysical() || !Operand.isDef())
+ continue;
+ const auto Physical = Reg->physicalRegister();
+ if (Physical)
+ Modified.insert(*Physical);
+ }
+ }
+ }
+
+ std::vector<PhysicalRegister> Result;
+ for (const PhysicalRegisterID ID : saveOrder()) {
+ auto Reg = physical(ID);
+ if (!Reg)
+ return std::unexpected(Reg.error());
+ if (Modified.contains(*Reg))
+ Result.push_back(*Reg);
+ }
+ if (requiresFramePointer(Function)) {
+ auto FP = physical(PhysicalRegisterID::X8);
+ if (!FP)
+ return std::unexpected(FP.error());
+ if (std::ranges::find(Result, *FP) == Result.end()) {
+ const auto Position = static_cast<std::vector<PhysicalRegister>::difference_type>(
+ std::min<std::size_t>(1, Result.size()));
+ Result.insert(Result.begin() + Position, *FP);
+ }
+ }
+ if (requiresBasePointer(Function)) {
+ auto BP = physical(PhysicalRegisterID::X9);
+ if (!BP)
+ return std::unexpected(BP.error());
+ if (std::ranges::find(Result, *BP) == Result.end()) {
+ const auto Position = static_cast<std::vector<PhysicalRegister>::difference_type>(
+ std::min<std::size_t>(2, Result.size()));
+ Result.insert(Result.begin() + Position, *BP);
+ }
+ }
+ for (const PhysicalRegister Reg : Result) {
+ if (Target.registers().physicalRegister(Reg) == nullptr) {
+ return makeUnexpected(ErrorCode::InternalError,
+ "RISC-V callee-save register is absent from target facts");
+ }
+ }
+ return Result;
+}
+
+[[nodiscard]] Expected<const MachineMemOperand *>
+createStackMemory(MachineFunction &Function, FrameIndex Index, MemoryAccessFlag Access) {
+ auto Size = MemorySize::fixed(8);
+ auto Align = Alignment::create(8);
+ if (!Size || !Align)
+ return makeUnexpected(ErrorCode::InternalError, "RISC-V spill facts are invalid");
+ MachineMemOperandDesc Description;
+ Description.Accesses = {Access};
+ Description.Size = *Size;
+ Description.Align = *Align;
+ Description.Base = MemoryBase::frameIndex(Index);
+ return Function.createMemOperand(std::move(Description));
+}
+
+[[nodiscard]] Expected<const MachineMemOperand *>
+createResolvedStackMemory(MachineFunction &Function, MemoryAccessFlag Access, std::int64_t Offset) {
+ auto Size = MemorySize::fixed(8);
+ auto Align = Alignment::create(8);
+ if (!Size || !Align)
+ return makeUnexpected(ErrorCode::InternalError, "RISC-V spill facts are invalid");
+ MachineMemOperandDesc Description;
+ Description.Accesses = {Access};
+ Description.Size = *Size;
+ Description.Align = *Align;
+ Description.Base = MemoryBase::stack();
+ Description.Offset = Offset;
+ return Function.createMemOperand(std::move(Description));
+}
+
+[[nodiscard]] Expected<void> replaceImmediate(MachineInstr &Instruction, std::size_t Index,
+ std::int64_t Value) {
+ const MachineOperand *Old = Instruction.operandAt(Index);
+ if (Old == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V frame instruction lacks its offset operand");
+ const std::uint32_t TargetFlags = Old->targetFlags();
+ auto Erased = Instruction.eraseOperand(Index);
+ if (!Erased)
+ return std::unexpected(Erased.error());
+ return Instruction.insertOperand(Index, MachineOperand::immediate(Value, TargetFlags));
+}
+
+[[nodiscard]] Expected<void> replaceWithRegister(MachineInstr &Instruction, std::size_t Index,
+ PhysicalRegister Reg) {
+ const MachineOperand *Old = Instruction.operandAt(Index);
+ if (Old == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V frame instruction lacks its frame operand");
+ const std::uint32_t TargetFlags = Old->targetFlags();
+ auto Replacement = MachineOperand::registerOperand(Register::fromPhysical(Reg),
+ {RegisterOperandFlag::Use}, {}, TargetFlags);
+ if (!Replacement)
+ return std::unexpected(Replacement.error());
+ auto Erased = Instruction.eraseOperand(Index);
+ if (!Erased)
+ return std::unexpected(Erased.error());
+ return Instruction.insertOperand(Index, std::move(*Replacement));
+}
+
+[[nodiscard]] Expected<void> setFrameFlag(MachineInstrBuilder &Builder,
+ std::optional<MachineInstrFlag> Flag) {
+ if (!Flag)
+ return {};
+ return Builder.setFlags({*Flag});
+}
+
+[[nodiscard]] Expected<DwarfRegisterNumber> dwarfRegister(const TargetDescription &Target,
+ PhysicalRegister Reg) {
+ const auto Number = Target.mirText().dwarfRegister(Reg);
+ if (!Number) {
+ return makeUnexpected(ErrorCode::InternalError, "RISC-V frame register has no DWARF mapping");
+ }
+ return DwarfRegisterNumber::fromValue(*Number);
+}
+
+[[nodiscard]] Expected<void> emitCFI(MachineIRBuilder &Builder, MachineCFIInstruction Instruction) {
+ auto Index = Builder.function().cfiInfo().addInstruction(std::move(Instruction));
+ if (!Index)
+ return std::unexpected(Index.error());
+ auto Pending = Builder.buildInstr(rcc::opcode(GenericOpcode::CFI_INSTRUCTION));
+ if (!Pending)
+ return std::unexpected(Pending.error());
+ if (auto Result = Pending->addCFIInstruction(*Index); !Result)
+ return std::unexpected(Result.error());
+ auto Built = Pending->build();
+ if (!Built)
+ return std::unexpected(Built.error());
+ return {};
+}
+
+void appendULEB128(std::vector<std::uint8_t> &Bytes, std::uint64_t Value) {
+ do {
+ std::uint8_t Byte = static_cast<std::uint8_t>(Value & 0x7FU);
+ Value >>= 7U;
+ if (Value != 0)
+ Byte |= 0x80U;
+ Bytes.push_back(Byte);
+ } while (Value != 0);
+}
+
+void appendSLEB128(std::vector<std::uint8_t> &Bytes, std::int64_t Value) {
+ bool More = true;
+ while (More) {
+ std::uint8_t Byte = static_cast<std::uint8_t>(Value & 0x7F);
+ const bool Sign = (Byte & 0x40U) != 0;
+ Value >>= 7;
+ More = (Value != 0 || Sign) && (Value != -1 || !Sign);
+ if (More)
+ Byte |= 0x80U;
+ Bytes.push_back(Byte);
+ }
+}
+
+[[nodiscard]] Expected<void>
+emitSavedRegisterCFI(MachineIRBuilder &Builder, const TargetDescription &Target,
+ PhysicalRegister Reg, std::int64_t Offset,
+ std::optional<PhysicalRegister> ExpressionBase = std::nullopt) {
+ auto DwarfReg = dwarfRegister(Target, Reg);
+ if (!DwarfReg)
+ return std::unexpected(DwarfReg.error());
+ if (!ExpressionBase)
+ return emitCFI(Builder, MachineCFIInstruction::offset(*DwarfReg, Offset));
+
+ auto DwarfBase = dwarfRegister(Target, *ExpressionBase);
+ if (!DwarfBase)
+ return std::unexpected(DwarfBase.error());
+ std::vector<std::uint8_t> Expression;
+ if (DwarfBase->value() <= 31U) {
+ Expression.push_back(static_cast<std::uint8_t>(DwarfOpBaseRegister0 + DwarfBase->value()));
+ } else {
+ Expression.push_back(DwarfOpBaseRegisterExtended);
+ appendULEB128(Expression, DwarfBase->value());
+ }
+ appendSLEB128(Expression, Offset);
+
+ std::vector<std::uint8_t> Escape{DwarfCFAExpression};
+ appendULEB128(Escape, DwarfReg->value());
+ appendULEB128(Escape, Expression.size());
+ Escape.insert(Escape.end(), Expression.begin(), Expression.end());
+ auto Rule = MachineCFIInstruction::escape(std::move(Escape));
+ if (!Rule)
+ return std::unexpected(Rule.error());
+ return emitCFI(Builder, std::move(*Rule));
+}
+
+[[nodiscard]] Expected<void> emitRestoreRegisterCFI(MachineIRBuilder &Builder,
+ const TargetDescription &Target,
+ PhysicalRegister Reg) {
+ auto DwarfReg = dwarfRegister(Target, Reg);
+ if (!DwarfReg)
+ return std::unexpected(DwarfReg.error());
+ return emitCFI(Builder, MachineCFIInstruction::restore(*DwarfReg));
+}
+
+[[nodiscard]] Expected<void> emitDefCFI(MachineIRBuilder &Builder, const TargetDescription &Target,
+ PhysicalRegister Reg, std::int64_t Offset) {
+ auto DwarfReg = dwarfRegister(Target, Reg);
+ if (!DwarfReg)
+ return std::unexpected(DwarfReg.error());
+ return emitCFI(Builder, MachineCFIInstruction::defCFA(*DwarfReg, Offset));
+}
+
+[[nodiscard]] Expected<void> emitConstant(MachineIRBuilder &Builder, PhysicalRegister Destination,
+ PhysicalRegister Zero, std::int64_t Value,
+ std::optional<MachineInstrFlag> Flag) {
+ std::uint64_t Remaining = static_cast<std::uint64_t>(Value);
+ std::vector<std::int64_t> Digits;
+ while (!fitsSigned12(std::bit_cast<std::int64_t>(Remaining))) {
+ std::int64_t Digit = static_cast<std::int64_t>(Remaining & 0xFFFU);
+ if (Digit >= 0x800)
+ Digit -= 0x1000;
+ Digits.push_back(Digit);
+ const std::uint64_t Adjusted = Remaining - static_cast<std::uint64_t>(Digit);
+ Remaining = Adjusted >> 12U;
+ if ((Adjusted & (std::uint64_t{1} << 63U)) != 0)
+ Remaining |= ~std::uint64_t{0} << 52U;
+ }
+
+ auto Base = Builder.buildInstr(targetOpcode(InstructionOpcode::ADDI));
+ if (!Base)
+ return std::unexpected(Base.error());
+ if (auto Result = Base->addDef(Register::fromPhysical(Destination)); !Result)
+ return std::unexpected(Result.error());
+ if (auto Result = Base->addUse(Register::fromPhysical(Zero)); !Result)
+ return std::unexpected(Result.error());
+ if (auto Result = Base->addImmediate(std::bit_cast<std::int64_t>(Remaining)); !Result)
+ return std::unexpected(Result.error());
+ if (auto Result = setFrameFlag(*Base, Flag); !Result)
+ return std::unexpected(Result.error());
+ if (auto Result = Base->build(); !Result)
+ return std::unexpected(Result.error());
+
+ for (std::size_t Reverse = Digits.size(); Reverse != 0; --Reverse) {
+ auto Shift = Builder.buildInstr(targetOpcode(InstructionOpcode::SLLI));
+ if (!Shift)
+ return std::unexpected(Shift.error());
+ if (auto Result = Shift->addDef(Register::fromPhysical(Destination)); !Result)
+ return std::unexpected(Result.error());
+ if (auto Result = Shift->addUse(Register::fromPhysical(Destination)); !Result)
+ return std::unexpected(Result.error());
+ if (auto Result = Shift->addImmediate(12); !Result)
+ return std::unexpected(Result.error());
+ if (auto Result = setFrameFlag(*Shift, Flag); !Result)
+ return std::unexpected(Result.error());
+ if (auto Result = Shift->build(); !Result)
+ return std::unexpected(Result.error());
+
+ auto Add = Builder.buildInstr(targetOpcode(InstructionOpcode::ADDI));
+ if (!Add)
+ return std::unexpected(Add.error());
+ if (auto Result = Add->addDef(Register::fromPhysical(Destination)); !Result)
+ return std::unexpected(Result.error());
+ if (auto Result = Add->addUse(Register::fromPhysical(Destination)); !Result)
+ return std::unexpected(Result.error());
+ if (auto Result = Add->addImmediate(Digits[Reverse - 1U]); !Result)
+ return std::unexpected(Result.error());
+ if (auto Result = setFrameFlag(*Add, Flag); !Result)
+ return std::unexpected(Result.error());
+ if (auto Result = Add->build(); !Result)
+ return std::unexpected(Result.error());
+ }
+ return {};
+}
+
+[[nodiscard]] Expected<void> emitRegisterAdd(MachineIRBuilder &Builder,
+ PhysicalRegister Destination, PhysicalRegister Left,
+ PhysicalRegister Right,
+ std::optional<MachineInstrFlag> Flag) {
+ auto Add = Builder.buildInstr(targetOpcode(InstructionOpcode::ADD));
+ if (!Add)
+ return std::unexpected(Add.error());
+ if (auto Result = Add->addDef(Register::fromPhysical(Destination)); !Result)
+ return std::unexpected(Result.error());
+ if (auto Result = Add->addUse(Register::fromPhysical(Left)); !Result)
+ return std::unexpected(Result.error());
+ if (auto Result = Add->addUse(Register::fromPhysical(Right)); !Result)
+ return std::unexpected(Result.error());
+ if (auto Result = setFrameFlag(*Add, Flag); !Result)
+ return std::unexpected(Result.error());
+ auto Built = Add->build();
+ if (!Built)
+ return std::unexpected(Built.error());
+ return {};
+}
+
+[[nodiscard]] Expected<void> emitAddImmediate(MachineIRBuilder &Builder,
+ PhysicalRegister Destination, PhysicalRegister Source,
+ std::int64_t Amount,
+ std::optional<MachineInstrFlag> Flag) {
+ auto Add = Builder.buildInstr(targetOpcode(InstructionOpcode::ADDI));
+ if (!Add)
+ return std::unexpected(Add.error());
+ if (auto Result = Add->addDef(Register::fromPhysical(Destination)); !Result)
+ return std::unexpected(Result.error());
+ if (auto Result = Add->addUse(Register::fromPhysical(Source)); !Result)
+ return std::unexpected(Result.error());
+ if (auto Result = Add->addImmediate(Amount); !Result)
+ return std::unexpected(Result.error());
+ if (auto Result = setFrameFlag(*Add, Flag); !Result)
+ return std::unexpected(Result.error());
+ auto Built = Add->build();
+ if (!Built)
+ return std::unexpected(Built.error());
+ return {};
+}
+
+[[nodiscard]] Expected<void> emitAndImmediate(MachineIRBuilder &Builder,
+ PhysicalRegister Destination, PhysicalRegister Source,
+ std::int64_t Amount,
+ std::optional<MachineInstrFlag> Flag) {
+ auto And = Builder.buildInstr(targetOpcode(InstructionOpcode::ANDI));
+ if (!And)
+ return std::unexpected(And.error());
+ if (auto Result = And->addDef(Register::fromPhysical(Destination)); !Result)
+ return std::unexpected(Result.error());
+ if (auto Result = And->addUse(Register::fromPhysical(Source)); !Result)
+ return std::unexpected(Result.error());
+ if (auto Result = And->addImmediate(Amount); !Result)
+ return std::unexpected(Result.error());
+ if (auto Result = setFrameFlag(*And, Flag); !Result)
+ return std::unexpected(Result.error());
+ auto Built = And->build();
+ if (!Built)
+ return std::unexpected(Built.error());
+ return {};
+}
+
+[[nodiscard]] Expected<void> emitRegisterAnd(MachineIRBuilder &Builder,
+ PhysicalRegister Destination, PhysicalRegister Left,
+ PhysicalRegister Right,
+ std::optional<MachineInstrFlag> Flag) {
+ auto And = Builder.buildInstr(targetOpcode(InstructionOpcode::AND));
+ if (!And)
+ return std::unexpected(And.error());
+ if (auto Result = And->addDef(Register::fromPhysical(Destination)); !Result)
+ return std::unexpected(Result.error());
+ if (auto Result = And->addUse(Register::fromPhysical(Left)); !Result)
+ return std::unexpected(Result.error());
+ if (auto Result = And->addUse(Register::fromPhysical(Right)); !Result)
+ return std::unexpected(Result.error());
+ if (auto Result = setFrameFlag(*And, Flag); !Result)
+ return std::unexpected(Result.error());
+ auto Built = And->build();
+ if (!Built)
+ return std::unexpected(Built.error());
+ return {};
+}
+
+[[nodiscard]] std::array<PhysicalRegisterID, 7> scratchOrder() noexcept {
+ return {PhysicalRegisterID::X5, PhysicalRegisterID::X6, PhysicalRegisterID::X7,
+ PhysicalRegisterID::X28, PhysicalRegisterID::X29, PhysicalRegisterID::X30,
+ PhysicalRegisterID::X31};
+}
+
+[[nodiscard]] Expected<std::vector<PhysicalRegister>> scratchRegisters() {
+ std::vector<PhysicalRegister> Result;
+ for (const PhysicalRegisterID ID : scratchOrder()) {
+ auto Reg = physical(ID);
+ if (!Reg)
+ return std::unexpected(Reg.error());
+ Result.push_back(*Reg);
+ }
+ return Result;
+}
+
+[[nodiscard]] const StackObject *emergencyObject(const MachineFunction &Function) noexcept {
+ for (const StackObject &Object : Function.frameInfo().objects()) {
+ if (!Object.isTombstone() && !Object.description()->IsDead &&
+ Object.description()->Kind == StackObjectKind::EmergencySpill)
+ return &Object;
+ }
+ return nullptr;
+}
+
+[[nodiscard]] Expected<FrameIndex> createEmergencyObject(MachineFunction &Function) {
+ auto Align = Alignment::create(8);
+ if (!Align)
+ return std::unexpected(Align.error());
+ StackObjectDesc Description;
+ Description.Kind = StackObjectKind::EmergencySpill;
+ Description.Size = 8;
+ Description.Align = *Align;
+ return Function.frameInfo().createObject(std::move(Description));
+}
+
+[[nodiscard]] Expected<std::uint64_t> layoutFrame(MachineFunction &Function, bool HasFP,
+ bool ReserveCallFrame) {
+ MachineFrameInfo &Frame = Function.frameInfo();
+ const std::uint64_t CallFrame = ReserveCallFrame ? Frame.maxCallFrameSize().value_or(0) : 0;
+ auto Initial = alignTo(CallFrame, StackAlignmentBytes);
+ if (!Initial)
+ return std::unexpected(Initial.error());
+ std::uint64_t Cursor = *Initial;
+
+ std::vector<const StackObject *> Specials;
+ std::vector<const StackObject *> Locals;
+ std::vector<const StackObject *> PreallocatedLocals;
+ std::uint64_t LocalBlobSize = Frame.state().LocalFrameSize;
+ std::uint64_t LocalBlobAlignment = 1;
+ std::uint64_t MaximumAlignment = requiredStackAlignment(Function);
+ for (const StackObject &Object : Frame.objects()) {
+ if (Object.isTombstone() || Object.description()->IsDead)
+ continue;
+ const StackObjectDesc &Description = *Object.description();
+ if (Description.Stack != StackID::defaultStack()) {
+ return makeUnexpected(ErrorCode::Unsupported,
+ "RV64GC does not support non-default stack IDs");
+ }
+ MaximumAlignment = std::max(MaximumAlignment, Description.Align.bytes());
+ if (Object.isVariableSized())
+ continue;
+ if (Description.LocalOffset) {
+ if (*Description.LocalOffset < 0 ||
+ static_cast<std::uint64_t>(*Description.LocalOffset) % Description.Align.bytes() != 0) {
+ return makeUnexpected(ErrorCode::FrameLayoutFailed,
+ "RISC-V local stack object has an invalid offset");
+ }
+ const std::uint64_t Offset = static_cast<std::uint64_t>(*Description.LocalOffset);
+ if (Description.Size.value_or(0) > std::numeric_limits<std::uint64_t>::max() - Offset) {
+ return makeUnexpected(ErrorCode::FrameLayoutFailed,
+ "RISC-V local stack object range overflows");
+ }
+ const std::uint64_t End = Offset + Description.Size.value_or(0);
+ if (isOutgoingCallObject(Description)) {
+ if (!Frame.maxCallFrameSize() || End > *Frame.maxCallFrameSize()) {
+ return makeUnexpected(ErrorCode::FrameLayoutFailed,
+ "RISC-V outgoing stack object exceeds the call frame");
+ }
+ } else {
+ LocalBlobSize = std::max(LocalBlobSize, End);
+ LocalBlobAlignment = std::max(LocalBlobAlignment, Description.Align.bytes());
+ PreallocatedLocals.push_back(&Object);
+ }
+ continue;
+ }
+ if (Object.isFixed())
+ continue;
+ if (Description.Kind == StackObjectKind::CalleeSaved ||
+ Description.Kind == StackObjectKind::EmergencySpill) {
+ Specials.push_back(&Object);
+ } else {
+ Locals.push_back(&Object);
+ }
+ }
+ const auto Order = [](const StackObject *Left, const StackObject *Right) {
+ const StackObjectDesc &L = *Left->description();
+ const StackObjectDesc &R = *Right->description();
+ if (L.Align.bytes() != R.Align.bytes())
+ return L.Align.bytes() > R.Align.bytes();
+ if (L.Size != R.Size)
+ return L.Size.value_or(0) > R.Size.value_or(0);
+ return Left->index().value() < Right->index().value();
+ };
+ std::ranges::sort(Locals, Order);
+ std::ranges::sort(Specials, Order);
+ std::ranges::sort(PreallocatedLocals, [](const StackObject *Left, const StackObject *Right) {
+ return Left->index().value() < Right->index().value();
+ });
+
+ const auto Place = [&](std::span<const StackObject *const> Objects) -> Expected<void> {
+ for (const StackObject *Object : Objects) {
+ const StackObjectDesc &Description = *Object->description();
+ auto Aligned = alignTo(Cursor, Description.Align.bytes());
+ if (!Aligned)
+ return std::unexpected(Aligned.error());
+ Cursor = *Aligned;
+ if (Cursor > static_cast<std::uint64_t>(std::numeric_limits<std::int64_t>::max())) {
+ return makeUnexpected(ErrorCode::FrameLayoutFailed,
+ "RISC-V frame offset exceeds signed 64-bit range");
+ }
+ auto Set = Frame.setObjectOffset(Object->index(), static_cast<std::int64_t>(Cursor));
+ if (!Set)
+ return std::unexpected(Set.error());
+ if (Description.Size.value_or(0) > std::numeric_limits<std::uint64_t>::max() - Cursor) {
+ return makeUnexpected(ErrorCode::FrameLayoutFailed, "RISC-V frame size overflows uint64");
+ }
+ Cursor += Description.Size.value_or(0);
+ }
+ return {};
+ };
+
+ const auto PlacePreallocated = [&]() -> Expected<void> {
+ if (LocalBlobSize == 0)
+ return {};
+ auto Aligned = alignTo(Cursor, LocalBlobAlignment);
+ if (!Aligned)
+ return std::unexpected(Aligned.error());
+ const std::uint64_t BlobStart = *Aligned;
+ for (const StackObject *Object : PreallocatedLocals) {
+ const std::uint64_t LocalOffset =
+ static_cast<std::uint64_t>(*Object->description()->LocalOffset);
+ if (BlobStart > static_cast<std::uint64_t>(std::numeric_limits<std::int64_t>::max()) ||
+ LocalOffset >
+ static_cast<std::uint64_t>(std::numeric_limits<std::int64_t>::max()) - BlobStart) {
+ return makeUnexpected(ErrorCode::FrameLayoutFailed,
+ "RISC-V local frame offset exceeds int64");
+ }
+ auto Set = Frame.setObjectOffset(Object->index(),
+ static_cast<std::int64_t>(BlobStart + LocalOffset));
+ if (!Set)
+ return std::unexpected(Set.error());
+ }
+ if (LocalBlobSize > std::numeric_limits<std::uint64_t>::max() - BlobStart)
+ return makeUnexpected(ErrorCode::FrameLayoutFailed,
+ "RISC-V local frame size overflows uint64");
+ Cursor = BlobStart + LocalBlobSize;
+ return {};
+ };
+
+ if (!HasFP || requiresStackRealignment(Function)) {
+ auto Placed = Place(Specials);
+ if (!Placed)
+ return std::unexpected(Placed.error());
+ }
+ auto PlacedLocals = Place(Locals);
+ if (!PlacedLocals)
+ return std::unexpected(PlacedLocals.error());
+ auto PlacedPreallocated = PlacePreallocated();
+ if (!PlacedPreallocated)
+ return std::unexpected(PlacedPreallocated.error());
+ if (HasFP && !requiresStackRealignment(Function)) {
+ auto Placed = Place(Specials);
+ if (!Placed)
+ return std::unexpected(Placed.error());
+ }
+
+ for (const StackObject &Object : Frame.objects()) {
+ if (Object.isTombstone() || Object.description()->IsDead || Object.isVariableSized())
+ continue;
+ const StackObjectDesc &Description = *Object.description();
+ if (Description.LocalOffset && isOutgoingCallObject(Description)) {
+ auto Set = Frame.setObjectOffset(Object.index(), *Description.LocalOffset);
+ if (!Set)
+ return std::unexpected(Set.error());
+ }
+ }
+ if (LocalBlobSize > std::numeric_limits<std::uint32_t>::max()) {
+ return makeUnexpected(ErrorCode::FrameLayoutFailed,
+ "RISC-V local frame blob exceeds MIR limits");
+ }
+ MachineFrameState State = Frame.state();
+ State.LocalFrameSize = static_cast<std::uint32_t>(LocalBlobSize);
+ auto UpdatedState = Frame.setState(std::move(State));
+ if (!UpdatedState)
+ return std::unexpected(UpdatedState.error());
+ auto FinalSize = alignTo(Cursor, MaximumAlignment);
+ if (!FinalSize)
+ return std::unexpected(FinalSize.error());
+ if (*FinalSize > static_cast<std::uint64_t>(std::numeric_limits<std::int64_t>::max())) {
+ return makeUnexpected(ErrorCode::FrameLayoutFailed,
+ "RISC-V frame size exceeds signed 64-bit range");
+ }
+ return *FinalSize;
+}
+
+[[nodiscard]] Expected<std::int64_t> objectReferenceOffset(const MachineFunction &Function,
+ const MachineInstr &Instruction,
+ const StackObjectDesc &Object,
+ std::int64_t OperandOffset) {
+ if (!Object.Offset)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V FrameIndex references an object without a final offset");
+ const bool HasFP = requiresFramePointer(Function);
+ std::int64_t BaseOffset = *Object.Offset;
+ if (Object.LocalOffset && isOutgoingCallObject(Object))
+ BaseOffset = *Object.LocalOffset;
+ else if (Object.Kind == StackObjectKind::FixedABI ||
+ Object.Kind == StackObjectKind::FixedSpillSlot) {
+ if (!HasFP) {
+ if (Function.frameInfo().state().StackSize >
+ static_cast<std::uint64_t>(std::numeric_limits<std::int64_t>::max())) {
+ return makeUnexpected(ErrorCode::FrameLayoutFailed,
+ "RISC-V stack size exceeds signed 64-bit range");
+ }
+ auto Added =
+ checkedAdd(BaseOffset, static_cast<std::int64_t>(Function.frameInfo().state().StackSize));
+ if (!Added)
+ return std::unexpected(Added.error());
+ BaseOffset = *Added;
+ }
+ } else if (!requiresStackRealignment(Function) && HasFP &&
+ !Instruction.flags().has(MachineInstrFlag::FrameSetup)) {
+ if (Function.frameInfo().state().StackSize >
+ static_cast<std::uint64_t>(std::numeric_limits<std::int64_t>::max())) {
+ return makeUnexpected(ErrorCode::FrameLayoutFailed,
+ "RISC-V stack size exceeds signed 64-bit range");
+ }
+ auto Added =
+ checkedAdd(BaseOffset, -static_cast<std::int64_t>(Function.frameInfo().state().StackSize));
+ if (!Added)
+ return std::unexpected(Added.error());
+ BaseOffset = *Added;
+ }
+ return checkedAdd(BaseOffset, OperandOffset);
+}
+
+[[nodiscard]] Expected<FrameIndexReference> emergencyFrameReference(const MachineFunction &Function,
+ const StackObjectDesc &Object) {
+ if (!Object.Offset) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V emergency spill object has no final offset");
+ }
+ PhysicalRegisterID BaseID = PhysicalRegisterID::X2;
+ std::int64_t Offset = *Object.Offset;
+ if (requiresBasePointer(Function)) {
+ BaseID = PhysicalRegisterID::X9;
+ } else if (requiresFramePointer(Function)) {
+ BaseID = PhysicalRegisterID::X8;
+ if (Function.frameInfo().state().StackSize >
+ static_cast<std::uint64_t>(std::numeric_limits<std::int64_t>::max())) {
+ return makeUnexpected(ErrorCode::FrameLayoutFailed,
+ "RISC-V stack size exceeds signed 64-bit range");
+ }
+ auto Adjusted =
+ checkedAdd(Offset, -static_cast<std::int64_t>(Function.frameInfo().state().StackSize));
+ if (!Adjusted)
+ return std::unexpected(Adjusted.error());
+ Offset = *Adjusted;
+ }
+ auto Base = physical(BaseID);
+ if (!Base)
+ return std::unexpected(Base.error());
+ return FrameIndexReference{*Base, Offset};
+}
+
+[[nodiscard]] Expected<MachineInstr *>
+emitCalleeSave(MachineIRBuilder &Builder, PhysicalRegister Reg, FrameIndex Index, bool Restore) {
+ const bool IsFPR = Reg.number() >= static_cast<std::uint32_t>(PhysicalRegisterID::F0);
+ const InstructionOpcode OpcodeValue =
+ Restore ? (IsFPR ? InstructionOpcode::FLD : InstructionOpcode::LD)
+ : (IsFPR ? InstructionOpcode::FSD : InstructionOpcode::SD);
+ auto Memory = createStackMemory(Builder.function(), Index,
+ Restore ? MemoryAccessFlag::Load : MemoryAccessFlag::Store);
+ if (!Memory)
+ return std::unexpected(Memory.error());
+ auto Pending = Builder.buildInstr(targetOpcode(OpcodeValue));
+ if (!Pending)
+ return std::unexpected(Pending.error());
+ const Register Value = Register::fromPhysical(Reg);
+ auto RegisterResult = Restore ? Pending->addDef(Value) : Pending->addUse(Value);
+ if (!RegisterResult)
+ return std::unexpected(RegisterResult.error());
+ if (auto Result = Pending->addFrameIndex(Index); !Result)
+ return std::unexpected(Result.error());
+ if (auto Result = Pending->addImmediate(0); !Result)
+ return std::unexpected(Result.error());
+ if (auto Result = Pending->addMemOperand(**Memory); !Result)
+ return std::unexpected(Result.error());
+ if (auto Result = setFrameFlag(*Pending, Restore ? MachineInstrFlag::FrameDestroy
+ : MachineInstrFlag::FrameSetup);
+ !Result) {
+ return std::unexpected(Result.error());
+ }
+ auto Built = Pending->build();
+ if (!Built)
+ return std::unexpected(Built.error());
+ return *Built;
+}
+
+[[nodiscard]] std::optional<FrameIndex> calleeSaveSlot(const MachineFunction &Function,
+ PhysicalRegister Reg) noexcept {
+ for (const StackObject &Object : Function.frameInfo().objects()) {
+ if (Object.isTombstone() || Object.description()->IsDead)
+ continue;
+ if (Object.description()->CalleeSavedRegister == Reg)
+ return Object.index();
+ }
+ return std::nullopt;
+}
+
+[[nodiscard]] bool isABIStackSaveRegister(PhysicalRegister Reg) {
+ return std::ranges::any_of(saveOrder(), [Reg](PhysicalRegisterID ID) {
+ auto Candidate = physical(ID);
+ return Candidate && *Candidate == Reg;
+ });
+}
+
+[[nodiscard]] Expected<void>
+validateCalleeSaveObjects(const MachineFunction &Function,
+ std::span<const PhysicalRegister> SavedRegisters) {
+ std::unordered_set<PhysicalRegister> Slots;
+ for (const StackObject &Object : Function.frameInfo().objects()) {
+ if (Object.isTombstone() || Object.description()->IsDead ||
+ !Object.description()->CalleeSavedRegister) {
+ continue;
+ }
+ const StackObjectDesc &Description = *Object.description();
+ const PhysicalRegister Reg = *Description.CalleeSavedRegister;
+ if (!isABIStackSaveRegister(Reg)) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V callee-save object names a caller-saved register");
+ }
+ if (Description.Kind != StackObjectKind::CalleeSaved ||
+ Description.Size != std::optional<std::uint64_t>{8} || Description.Align.bytes() < 8 ||
+ Description.Stack != StackID::defaultStack()) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V callee-save object has an invalid kind, size, or alignment");
+ }
+ if (!Slots.insert(Reg).second) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V callee-save register has duplicate stack objects");
+ }
+ if (std::ranges::find(SavedRegisters, Reg) == SavedRegisters.end()) {
+ return makeUnexpected(ErrorCode::InternalError,
+ "RISC-V callee-save object was omitted from save analysis");
+ }
+ }
+ return {};
+}
+
+[[nodiscard]] bool calleeSaveIsRestored(const MachineFunction &Function,
+ PhysicalRegister Reg) noexcept {
+ const auto Slot = calleeSaveSlot(Function, Reg);
+ const StackObject *Object = Slot ? Function.frameInfo().object(*Slot) : nullptr;
+ return Object != nullptr && Object->description()->CalleeSavedRestored;
+}
+
+struct CalleeSaveOperation final {
+ MachineInstr *Instruction = nullptr;
+ PhysicalRegister Reg;
+};
+
+[[nodiscard]] std::vector<CalleeSaveOperation>
+calleeSaveOperations(MachineBasicBlock &Block, const MachineFunction &Function, bool Restore) {
+ std::vector<CalleeSaveOperation> Result;
+ for (MachineInstr *Instruction = Block.firstInstruction(); Instruction != nullptr;
+ Instruction = Instruction->next()) {
+ if (!Instruction->flags().has(Restore ? MachineInstrFlag::FrameDestroy
+ : MachineInstrFlag::FrameSetup)) {
+ continue;
+ }
+ for (const MachineOperand &Operand : Instruction->operands()) {
+ const auto *Frame = Operand.getIf<FrameIndexOperand>();
+ if (Frame == nullptr)
+ continue;
+ const StackObject *Object = Function.frameInfo().object(Frame->Index);
+ if (Object != nullptr && Object->description()->CalleeSavedRegister) {
+ Result.push_back(
+ CalleeSaveOperation{Instruction, *Object->description()->CalleeSavedRegister});
+ break;
+ }
+ }
+ }
+ return Result;
+}
+
+class RISCVFrameLowering final : public TargetFrameLowering {
+public:
+ [[nodiscard]] Expected<void> prepareFrame(MachineFunction &Function,
+ const TargetDescription &Target) const override {
+ MachineFrameInfo &Frame = Function.frameInfo();
+ if (Frame.isFrozen())
+ return makeUnexpected(ErrorCode::InvalidArgument, "RISC-V frame layout is already frozen");
+
+ MachineFrameState State = Frame.state();
+ for (const MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr;
+ Block = Block->next()) {
+ for (const MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr;
+ Instruction = Instruction->next()) {
+ const InstructionDescriptor *Descriptor =
+ Target.instructionDescriptor(Instruction->opcode());
+ if (Descriptor != nullptr && Descriptor->Flags.has(InstructionFlag::Call))
+ State.HasCalls = true;
+ }
+ }
+ auto UpdatedState = Frame.setState(State);
+ if (!UpdatedState)
+ return std::unexpected(UpdatedState.error());
+
+ auto CallFrame = computeMaxCallFrameSize(Function);
+ if (!CallFrame)
+ return std::unexpected(CallFrame.error());
+ auto FrameRegisters = validateFrameRegisterUses(Function, Target);
+ if (!FrameRegisters)
+ return std::unexpected(FrameRegisters.error());
+
+ auto SavedRegisters = modifiedCalleeSavedRegisters(Function, Target);
+ if (!SavedRegisters)
+ return std::unexpected(SavedRegisters.error());
+ auto ExistingSlots = validateCalleeSaveObjects(Function, *SavedRegisters);
+ if (!ExistingSlots)
+ return std::unexpected(ExistingSlots.error());
+ auto Align8 = Alignment::create(8);
+ if (!Align8)
+ return std::unexpected(Align8.error());
+ for (const PhysicalRegister Reg : *SavedRegisters) {
+ if (calleeSaveSlot(Function, Reg))
+ continue;
+ StackObjectDesc Description;
+ Description.Kind = StackObjectKind::CalleeSaved;
+ Description.Size = 8;
+ Description.Align = *Align8;
+ Description.CalleeSavedRegister = Reg;
+ auto Created = Frame.createObject(std::move(Description));
+ if (!Created)
+ return std::unexpected(Created.error());
+ }
+ auto CompleteSlots = validateCalleeSaveObjects(Function, *SavedRegisters);
+ if (!CompleteSlots)
+ return std::unexpected(CompleteSlots.error());
+
+ const bool HasFP = requiresFramePointer(Function);
+ const bool ReserveCallFrame = hasReservedCallFrame(Function);
+ auto StackSize = layoutFrame(Function, HasFP, ReserveCallFrame);
+ if (!StackSize)
+ return std::unexpected(StackSize.error());
+
+ State = Frame.state();
+ State.StackSize = *StackSize;
+ auto RequiredAlignment = Alignment::create(requiredStackAlignment(Function));
+ if (!RequiredAlignment)
+ return std::unexpected(RequiredAlignment.error());
+ State.MaxAlignment = *RequiredAlignment;
+ auto StateForReferences = Frame.setState(State);
+ if (!StateForReferences)
+ return std::unexpected(StateForReferences.error());
+
+ bool NeedsEmergency = false;
+ for (const MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr;
+ Block = Block->next()) {
+ for (const MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr;
+ Instruction = Instruction->next()) {
+ if (isCallFramePseudo(*Instruction)) {
+ auto Amount = callFrameAmount(*Instruction);
+ if (!Amount)
+ return std::unexpected(Amount.error());
+ NeedsEmergency |= !ReserveCallFrame && !fitsSigned12(*Amount);
+ }
+ for (std::size_t Index = 0; Index < Instruction->operandCount(); ++Index) {
+ const auto *FrameOperand = Instruction->operandAt(Index)->getIf<FrameIndexOperand>();
+ if (FrameOperand == nullptr)
+ continue;
+ std::int64_t Extra = FrameOperand->Offset;
+ if (Index + 1U >= Instruction->operandCount()) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V frame operand lacks an adjacent immediate");
+ }
+ auto Immediate = immediateValue(*Instruction->operandAt(Index + 1U));
+ if (!Immediate)
+ return std::unexpected(Immediate.error());
+ auto Added = checkedAdd(Extra, *Immediate);
+ if (!Added)
+ return std::unexpected(Added.error());
+ Extra = *Added;
+ const StackObject *Object = Frame.object(FrameOperand->Index);
+ if (Object == nullptr || Object->description()->IsDead)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V frame instruction references an invalid object");
+ auto Offset =
+ objectReferenceOffset(Function, *Instruction, *Object->description(), Extra);
+ if (!Offset)
+ return std::unexpected(Offset.error());
+ NeedsEmergency |= !fitsSigned12(*Offset);
+ }
+ }
+ }
+ if (NeedsEmergency && emergencyObject(Function) == nullptr) {
+ auto Emergency = createEmergencyObject(Function);
+ if (!Emergency)
+ return std::unexpected(Emergency.error());
+ StackSize = layoutFrame(Function, HasFP, ReserveCallFrame);
+ if (!StackSize)
+ return std::unexpected(StackSize.error());
+ }
+
+ State = Frame.state();
+ State.StackSize = *StackSize;
+ State.AdjustsStack = State.AdjustsStack || *StackSize != 0;
+ State.IsCalleeSavedInfoValid = true;
+ State.SavePoints.clear();
+ State.RestorePoints.clear();
+ if (!SavedRegisters->empty() && Function.firstBlock() != nullptr) {
+ State.SavePoints.push_back(
+ MachineFrameSaveRestorePoint{Function.firstBlock()->stableID(), *SavedRegisters});
+ }
+ for (const MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr;
+ Block = Block->next()) {
+ bool HasEpilogue = false;
+ for (const MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr;
+ Instruction = Instruction->next()) {
+ HasEpilogue |= isEpilogueTerminator(*Instruction, Target);
+ }
+ if (HasEpilogue && !SavedRegisters->empty()) {
+ std::vector<PhysicalRegister> Restored;
+ std::ranges::copy_if(
+ *SavedRegisters, std::back_inserter(Restored),
+ [&Function](PhysicalRegister Reg) { return calleeSaveIsRestored(Function, Reg); });
+ if (!Restored.empty()) {
+ State.RestorePoints.push_back(
+ MachineFrameSaveRestorePoint{Block->stableID(), std::move(Restored)});
+ }
+ }
+ }
+ auto SetState = Frame.setState(std::move(State));
+ if (!SetState)
+ return std::unexpected(SetState.error());
+ auto Records = Function.records().setCalleeSavedRegisters(*SavedRegisters);
+ if (!Records)
+ return std::unexpected(Records.error());
+ return {};
+ }
+
+ [[nodiscard]] Expected<void>
+ insertCalleeSavedSpillsAndRestores(MachineFunction &Function,
+ const TargetDescription &Target) const override {
+ if (Function.frameInfo().isFrozen()) {
+ return makeUnexpected(ErrorCode::InternalError,
+ "RISC-V callee saves must be inserted before frame freeze");
+ }
+ const std::span<const PhysicalRegister> Saved = Function.records().calleeSavedRegisters();
+ if (Saved.empty())
+ return {};
+ MachineBasicBlock *Entry = Function.firstBlock();
+ if (Entry == nullptr) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V callee-save frame has no entry block");
+ }
+ auto FP = physical(PhysicalRegisterID::X8);
+ auto BP = physical(PhysicalRegisterID::X9);
+ auto OldFP = physical(PhysicalRegisterID::X5);
+ if (!FP || !BP || !OldFP)
+ return makeUnexpected(ErrorCode::InternalError, "RISC-V frame registers are unavailable");
+ const bool Realign = requiresStackRealignment(Function);
+
+ MachineIRBuilder SaveBuilder(Function, Target);
+ auto SavePoint = Entry->firstInstruction() != nullptr
+ ? SaveBuilder.setInsertPointBefore(*Entry->firstInstruction())
+ : SaveBuilder.setInsertPoint(*Entry);
+ if (!SavePoint)
+ return std::unexpected(SavePoint.error());
+ for (const PhysicalRegister Reg : Saved) {
+ const auto Slot = calleeSaveSlot(Function, Reg);
+ if (!Slot) {
+ return makeUnexpected(ErrorCode::InternalError,
+ "RISC-V callee-save register has no stack slot");
+ }
+ const PhysicalRegister Value = Realign && Reg == *FP ? *OldFP : Reg;
+ auto Stored = emitCalleeSave(SaveBuilder, Value, *Slot, false);
+ if (!Stored)
+ return std::unexpected(Stored.error());
+ }
+
+ for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr;
+ Block = Block->next()) {
+ std::vector<MachineInstr *> Terminators;
+ for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr;
+ Instruction = Instruction->next()) {
+ if (isEpilogueTerminator(*Instruction, Target))
+ Terminators.push_back(Instruction);
+ }
+ for (MachineInstr *Terminator : Terminators) {
+ MachineIRBuilder RestoreBuilder(Function, Target);
+ auto Point = RestoreBuilder.setInsertPointBefore(*Terminator);
+ if (!Point)
+ return std::unexpected(Point.error());
+ const auto EmitRestore = [&](PhysicalRegister Reg) -> Expected<void> {
+ if (!calleeSaveIsRestored(Function, Reg))
+ return {};
+ const auto Slot = calleeSaveSlot(Function, Reg);
+ if (!Slot) {
+ return makeUnexpected(ErrorCode::InternalError,
+ "RISC-V callee-save register has no restore slot");
+ }
+ auto Loaded = emitCalleeSave(RestoreBuilder, Reg, *Slot, true);
+ if (!Loaded)
+ return std::unexpected(Loaded.error());
+ return {};
+ };
+ for (const PhysicalRegister Reg : Saved) {
+ if ((requiresFramePointer(Function) && Reg == *FP) || (Realign && Reg == *BP))
+ continue;
+ auto Loaded = EmitRestore(Reg);
+ if (!Loaded)
+ return std::unexpected(Loaded.error());
+ }
+ if (requiresFramePointer(Function)) {
+ auto Loaded = EmitRestore(*FP);
+ if (!Loaded)
+ return std::unexpected(Loaded.error());
+ }
+ if (Realign) {
+ auto Loaded = EmitRestore(*BP);
+ if (!Loaded)
+ return std::unexpected(Loaded.error());
+ }
+ }
+ }
+ return {};
+ }
+
+ [[nodiscard]] Expected<void> finalizeFrame(MachineFunction &Function,
+ const TargetDescription &) const override {
+ MachineFrameInfo &Frame = Function.frameInfo();
+ if (Frame.isFrozen())
+ return makeUnexpected(ErrorCode::InternalError,
+ "RISC-V frame was frozen before final layout");
+ std::vector<std::optional<std::int64_t>> PreliminaryOffsets;
+ PreliminaryOffsets.reserve(Frame.objects().size());
+ for (const StackObject &Object : Frame.objects()) {
+ PreliminaryOffsets.push_back(Object.isTombstone() ? std::nullopt
+ : Object.description()->Offset);
+ }
+ const std::uint64_t PreliminarySize = Frame.state().StackSize;
+ auto StackSize =
+ layoutFrame(Function, requiresFramePointer(Function), hasReservedCallFrame(Function));
+ if (!StackSize)
+ return std::unexpected(StackSize.error());
+ if (*StackSize != PreliminarySize || Frame.objects().size() != PreliminaryOffsets.size()) {
+ return makeUnexpected(ErrorCode::InternalError,
+ "RISC-V frame resources changed after preliminary layout");
+ }
+ for (const StackObject &Object : Frame.objects()) {
+ if (!Object.isTombstone() &&
+ Object.description()->Offset != PreliminaryOffsets[Object.index().value()]) {
+ return makeUnexpected(ErrorCode::InternalError,
+ "RISC-V frame offsets changed after callee-save insertion");
+ }
+ }
+ auto StackAlignment = Alignment::create(requiredStackAlignment(Function));
+ if (!StackAlignment)
+ return std::unexpected(StackAlignment.error());
+ return Frame.freeze(*StackSize, *StackAlignment);
+ }
+
+ [[nodiscard]] bool isCallFramePseudo(const MachineInstr &Instruction) const noexcept override {
+ return Instruction.opcode() == targetOpcode(InstructionOpcode::ADJCALLSTACKDOWN) ||
+ Instruction.opcode() == targetOpcode(InstructionOpcode::ADJCALLSTACKUP);
+ }
+
+ [[nodiscard]] Expected<void>
+ eliminateCallFramePseudo(MachineInstr &Instruction, MachineFunction &Function,
+ const TargetDescription &Target,
+ RegisterScavenger &Scavenger) const override {
+ MachineBasicBlock *Block = Instruction.parent();
+ if (Block == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError, "RISC-V call-frame pseudo is detached");
+ if (!hasReservedCallFrame(Function)) {
+ auto Amount = callFrameAmount(Instruction);
+ if (!Amount)
+ return std::unexpected(Amount.error());
+ std::int64_t Adjustment = *Amount;
+ if (Instruction.opcode() == targetOpcode(InstructionOpcode::ADJCALLSTACKDOWN))
+ Adjustment = -Adjustment;
+ auto Adjusted = emitStackAdjustment(Instruction, Function, Target, Scavenger, Adjustment,
+ Adjustment < 0 ? MachineInstrFlag::FrameSetup
+ : MachineInstrFlag::FrameDestroy);
+ if (!Adjusted)
+ return std::unexpected(Adjusted.error());
+ }
+ return Block->removeInstruction(Instruction);
+ }
+
+ [[nodiscard]] Expected<void> emitPrologue(MachineFunction &Function,
+ const TargetDescription &Target,
+ RegisterScavenger &Scavenger) const override {
+ MachineBasicBlock *Entry = Function.firstBlock();
+ if (Entry == nullptr)
+ return Function.frameInfo().state().StackSize == 0
+ ? Expected<void>{}
+ : makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V nonempty frame has no entry block");
+ const std::vector<CalleeSaveOperation> Saves = calleeSaveOperations(*Entry, Function, false);
+ if (Saves.size() != Function.records().calleeSavedRegisters().size()) {
+ return makeUnexpected(ErrorCode::InternalError,
+ "RISC-V prologue has an incomplete callee-save sequence");
+ }
+ MachineInstr *Anchor = Entry->firstInstruction();
+ MachineIRBuilder Builder(Function, Target);
+ auto Point =
+ Anchor != nullptr ? Builder.setInsertPointBefore(*Anchor) : Builder.setInsertPoint(*Entry);
+ if (!Point)
+ return std::unexpected(Point.error());
+
+ const std::uint64_t StackSize = Function.frameInfo().state().StackSize;
+ const std::uint64_t StackAlignment = requiredStackAlignment(Function);
+ const bool Realign = requiresStackRealignment(Function);
+ auto SP = physical(PhysicalRegisterID::X2);
+ auto FP = physical(PhysicalRegisterID::X8);
+ auto BP = physical(PhysicalRegisterID::X9);
+ auto Zero = physical(PhysicalRegisterID::X0);
+ auto OldFP = physical(PhysicalRegisterID::X5);
+ auto DefaultTemporary = physical(PhysicalRegisterID::X6);
+ if (!SP || !FP || !BP || !Zero || !OldFP || !DefaultTemporary) {
+ return makeUnexpected(ErrorCode::InternalError, "RISC-V frame registers are unavailable");
+ }
+ PhysicalRegister Temporary = *DefaultTemporary;
+ const std::int64_t AlignmentMask = std::bit_cast<std::int64_t>(~(StackAlignment - 1U));
+ const bool NeedsTemporary = StackSize > 2048U ||
+ (!Realign && requiresFramePointer(Function) &&
+ StackSize > static_cast<std::uint64_t>(MaxSigned12)) ||
+ (Realign && !fitsSigned12(AlignmentMask));
+ if (NeedsTemporary) {
+ auto Candidates = scratchRegisters();
+ if (!Candidates)
+ return std::unexpected(Candidates.error());
+ if (Anchor != nullptr) {
+ auto Scratch = Scavenger.scavenge(*Anchor, *Candidates);
+ if (!Scratch)
+ return std::unexpected(Scratch.error());
+ if (Scratch->RequiresSpill) {
+ return makeUnexpected(ErrorCode::RegisterAllocationFailed,
+ "RISC-V prologue has no dead scratch register");
+ }
+ Temporary = Scratch->Reg;
+ } else {
+ Temporary = Candidates->front();
+ }
+ }
+
+ if (Realign) {
+ auto PreserveFP = emitAddImmediate(Builder, *OldFP, *FP, 0, MachineInstrFlag::FrameSetup);
+ if (!PreserveFP)
+ return std::unexpected(PreserveFP.error());
+ auto CaptureIncomingSP = emitAddImmediate(Builder, *FP, *SP, 0, MachineInstrFlag::FrameSetup);
+ if (!CaptureIncomingSP)
+ return std::unexpected(CaptureIncomingSP.error());
+ auto CFA = emitDefCFI(Builder, Target, *FP, 0);
+ if (!CFA)
+ return std::unexpected(CFA.error());
+ }
+
+ const std::uint64_t AllocationSize = StackSize;
+ if (AllocationSize > static_cast<std::uint64_t>(std::numeric_limits<std::int64_t>::max())) {
+ return makeUnexpected(ErrorCode::FrameLayoutFailed, "RISC-V stack adjustment exceeds int64");
+ }
+ if (AllocationSize != 0) {
+ const std::int64_t Amount = -static_cast<std::int64_t>(AllocationSize);
+ if (fitsSigned12(Amount)) {
+ auto Adjust = emitAddImmediate(Builder, *SP, *SP, Amount, MachineInstrFlag::FrameSetup);
+ if (!Adjust)
+ return std::unexpected(Adjust.error());
+ } else {
+ auto Constant =
+ emitConstant(Builder, Temporary, *Zero, Amount, MachineInstrFlag::FrameSetup);
+ if (!Constant)
+ return std::unexpected(Constant.error());
+ auto Adjust = emitRegisterAdd(Builder, *SP, *SP, Temporary, MachineInstrFlag::FrameSetup);
+ if (!Adjust)
+ return std::unexpected(Adjust.error());
+ }
+ if (!Realign) {
+ auto CFA = emitDefCFI(Builder, Target, *SP, static_cast<std::int64_t>(StackSize));
+ if (!CFA)
+ return std::unexpected(CFA.error());
+ }
+ }
+ if (Realign) {
+ const std::int64_t Mask = AlignmentMask;
+ Expected<void> Aligned =
+ fitsSigned12(Mask)
+ ? emitAndImmediate(Builder, *SP, *SP, Mask, MachineInstrFlag::FrameSetup)
+ : emitConstant(Builder, Temporary, *Zero, Mask, MachineInstrFlag::FrameSetup);
+ if (!Aligned)
+ return std::unexpected(Aligned.error());
+ if (!fitsSigned12(Mask)) {
+ Aligned = emitRegisterAnd(Builder, *SP, *SP, Temporary, MachineInstrFlag::FrameSetup);
+ if (!Aligned)
+ return std::unexpected(Aligned.error());
+ }
+ }
+
+ if (Realign) {
+ if (Saves.empty()) {
+ return makeUnexpected(ErrorCode::InternalError,
+ "RISC-V realigned frame has no callee-save sequence");
+ }
+ MachineIRBuilder AfterSaves(Function, Target);
+ auto PointAfterSaves = AfterSaves.setInsertPointAfter(*Saves.back().Instruction);
+ if (!PointAfterSaves)
+ return std::unexpected(PointAfterSaves.error());
+ auto Set = emitAddImmediate(AfterSaves, *BP, *SP, 0, MachineInstrFlag::FrameSetup);
+ if (!Set)
+ return std::unexpected(Set.error());
+ for (const PhysicalRegister Reg : Function.records().calleeSavedRegisters()) {
+ const auto Slot = calleeSaveSlot(Function, Reg);
+ const StackObject *Object = Slot ? Function.frameInfo().object(*Slot) : nullptr;
+ if (Object == nullptr || !Object->description()->Offset) {
+ return makeUnexpected(ErrorCode::InternalError,
+ "RISC-V realigned CFI slot is incomplete");
+ }
+ auto CFI =
+ emitSavedRegisterCFI(AfterSaves, Target, Reg, *Object->description()->Offset, *BP);
+ if (!CFI)
+ return std::unexpected(CFI.error());
+ }
+ return {};
+ }
+
+ for (std::size_t Index = 0; Index < Saves.size(); ++Index) {
+ const CalleeSaveOperation &Save = Saves[Index];
+ const auto Slot = calleeSaveSlot(Function, Save.Reg);
+ const StackObject *Object = Slot ? Function.frameInfo().object(*Slot) : nullptr;
+ if (Object == nullptr || !Object->description()->Offset) {
+ return makeUnexpected(ErrorCode::InternalError,
+ "RISC-V callee-save CFI slot is incomplete");
+ }
+ auto Offset =
+ checkedAdd(*Object->description()->Offset, -static_cast<std::int64_t>(StackSize));
+ if (!Offset)
+ return std::unexpected(Offset.error());
+ MachineIRBuilder AfterSave(Function, Target);
+ auto PointAfterSave = AfterSave.setInsertPointAfter(*Save.Instruction);
+ if (!PointAfterSave)
+ return std::unexpected(PointAfterSave.error());
+ auto CFI = emitSavedRegisterCFI(AfterSave, Target, Save.Reg, *Offset);
+ if (!CFI)
+ return std::unexpected(CFI.error());
+ if (Index + 1U != Saves.size() || !requiresFramePointer(Function))
+ continue;
+ Expected<void> Set =
+ StackSize <= static_cast<std::uint64_t>(MaxSigned12)
+ ? emitAddImmediate(AfterSave, *FP, *SP, static_cast<std::int64_t>(StackSize),
+ MachineInstrFlag::FrameSetup)
+ : emitConstant(AfterSave, Temporary, *Zero, static_cast<std::int64_t>(StackSize),
+ MachineInstrFlag::FrameSetup);
+ if (!Set)
+ return std::unexpected(Set.error());
+ if (StackSize > static_cast<std::uint64_t>(MaxSigned12)) {
+ Set = emitRegisterAdd(AfterSave, *FP, *SP, Temporary, MachineInstrFlag::FrameSetup);
+ if (!Set)
+ return std::unexpected(Set.error());
+ }
+ auto CFA = emitDefCFI(AfterSave, Target, *FP, 0);
+ if (!CFA)
+ return std::unexpected(CFA.error());
+ }
+ if (Saves.empty() && requiresFramePointer(Function)) {
+ Expected<void> Set =
+ StackSize <= static_cast<std::uint64_t>(MaxSigned12)
+ ? emitAddImmediate(Builder, *FP, *SP, static_cast<std::int64_t>(StackSize),
+ MachineInstrFlag::FrameSetup)
+ : emitConstant(Builder, Temporary, *Zero, static_cast<std::int64_t>(StackSize),
+ MachineInstrFlag::FrameSetup);
+ if (!Set)
+ return std::unexpected(Set.error());
+ if (StackSize > static_cast<std::uint64_t>(MaxSigned12)) {
+ Set = emitRegisterAdd(Builder, *FP, *SP, Temporary, MachineInstrFlag::FrameSetup);
+ if (!Set)
+ return std::unexpected(Set.error());
+ }
+ auto CFA = emitDefCFI(Builder, Target, *FP, 0);
+ if (!CFA)
+ return std::unexpected(CFA.error());
+ }
+ return {};
+ }
+
+ [[nodiscard]] bool isEpilogueTerminator(const MachineInstr &Instruction,
+ const TargetDescription &Target) const noexcept override {
+ if (Instruction.opcode() == targetOpcode(InstructionOpcode::PseudoTAIL))
+ return true;
+ const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction.opcode());
+ return Descriptor != nullptr && Descriptor->Flags.has(InstructionFlag::Return);
+ }
+
+ [[nodiscard]] Expected<void> emitEpilogue(MachineBasicBlock &Block, MachineInstr &Terminator,
+ MachineFunction &Function,
+ const TargetDescription &Target,
+ RegisterScavenger &Scavenger) const override {
+ auto FP = physical(PhysicalRegisterID::X8);
+ auto SP = physical(PhysicalRegisterID::X2);
+ auto BP = physical(PhysicalRegisterID::X9);
+ if (!FP || !SP || !BP) {
+ return makeUnexpected(ErrorCode::InternalError, "RISC-V frame registers are unavailable");
+ }
+ const bool HasFP = requiresFramePointer(Function);
+ const bool Realign = requiresStackRealignment(Function);
+ const std::uint64_t StackSize = Function.frameInfo().state().StackSize;
+ const bool HasFrameCFI = StackSize != 0 || !Function.records().calleeSavedRegisters().empty();
+
+ const std::vector<CalleeSaveOperation> BlockRestores =
+ calleeSaveOperations(Block, Function, true);
+ std::vector<CalleeSaveOperation> Restores;
+ for (MachineInstr *Cursor = Terminator.previous(); Cursor != nullptr;
+ Cursor = Cursor->previous()) {
+ const auto Position =
+ std::ranges::find(BlockRestores, Cursor, &CalleeSaveOperation::Instruction);
+ if (Position == BlockRestores.end())
+ break;
+ Restores.push_back(*Position);
+ }
+ std::ranges::reverse(Restores);
+ const std::size_t ExpectedRestores = static_cast<std::size_t>(std::ranges::count_if(
+ Function.records().calleeSavedRegisters(),
+ [&Function](PhysicalRegister Reg) { return calleeSaveIsRestored(Function, Reg); }));
+ if (Restores.size() != ExpectedRestores) {
+ return makeUnexpected(ErrorCode::InternalError,
+ "RISC-V epilogue has an incomplete callee-restore sequence");
+ }
+
+ if (HasFrameCFI) {
+ MachineIRBuilder RememberBuilder(Function, Target);
+ auto Point = Restores.empty()
+ ? RememberBuilder.setInsertPointBefore(Terminator)
+ : RememberBuilder.setInsertPointBefore(*Restores.front().Instruction);
+ if (!Point)
+ return std::unexpected(Point.error());
+ auto Remember = emitCFI(RememberBuilder, MachineCFIInstruction::rememberState());
+ if (!Remember)
+ return std::unexpected(Remember.error());
+ }
+
+ for (const CalleeSaveOperation &Restore : Restores) {
+ if ((HasFP && Restore.Reg == *FP) || (Realign && Restore.Reg == *BP))
+ continue;
+ MachineIRBuilder AfterRestore(Function, Target);
+ auto Point = AfterRestore.setInsertPointAfter(*Restore.Instruction);
+ if (!Point)
+ return std::unexpected(Point.error());
+ auto CFI = emitRestoreRegisterCFI(AfterRestore, Target, Restore.Reg);
+ if (!CFI)
+ return std::unexpected(CFI.error());
+ }
+
+ if (Realign) {
+ const auto FPRestore = std::ranges::find(Restores, *FP, &CalleeSaveOperation::Reg);
+ const auto BPRestore = std::ranges::find(Restores, *BP, &CalleeSaveOperation::Reg);
+ MachineIRBuilder Teardown(Function, Target);
+ MachineInstr *SpecialRestore =
+ FPRestore != Restores.end()
+ ? FPRestore->Instruction
+ : (BPRestore != Restores.end() ? BPRestore->Instruction : &Terminator);
+ auto Point = Teardown.setInsertPointBefore(*SpecialRestore);
+ if (!Point)
+ return std::unexpected(Point.error());
+ auto RestoreSP = emitAddImmediate(Teardown, *SP, *FP, 0, MachineInstrFlag::FrameDestroy);
+ if (!RestoreSP)
+ return std::unexpected(RestoreSP.error());
+ auto CFA = emitDefCFI(Teardown, Target, *SP, 0);
+ if (!CFA)
+ return std::unexpected(CFA.error());
+ if (FPRestore != Restores.end()) {
+ MachineIRBuilder AfterFP(Function, Target);
+ auto After = AfterFP.setInsertPointAfter(*FPRestore->Instruction);
+ if (!After)
+ return std::unexpected(After.error());
+ auto FPCFI = emitRestoreRegisterCFI(AfterFP, Target, *FP);
+ if (!FPCFI)
+ return std::unexpected(FPCFI.error());
+ }
+ if (BPRestore != Restores.end()) {
+ MachineIRBuilder AfterBP(Function, Target);
+ auto After = AfterBP.setInsertPointAfter(*BPRestore->Instruction);
+ if (!After)
+ return std::unexpected(After.error());
+ auto BPCFI = emitRestoreRegisterCFI(AfterBP, Target, *BP);
+ if (!BPCFI)
+ return std::unexpected(BPCFI.error());
+ }
+ } else if (HasFP) {
+ const auto FPRestore = std::ranges::find(Restores, *FP, &CalleeSaveOperation::Reg);
+ MachineIRBuilder Teardown(Function, Target);
+ auto Point = FPRestore == Restores.end()
+ ? Teardown.setInsertPointBefore(Terminator)
+ : Teardown.setInsertPointBefore(*FPRestore->Instruction);
+ if (!Point)
+ return std::unexpected(Point.error());
+ auto RestoreSP = emitAddImmediate(Teardown, *SP, *FP, 0, MachineInstrFlag::FrameDestroy);
+ if (!RestoreSP)
+ return std::unexpected(RestoreSP.error());
+ auto CFA = emitDefCFI(Teardown, Target, *SP, 0);
+ if (!CFA)
+ return std::unexpected(CFA.error());
+ if (FPRestore != Restores.end()) {
+ MachineIRBuilder AfterFP(Function, Target);
+ auto After = AfterFP.setInsertPointAfter(*FPRestore->Instruction);
+ if (!After)
+ return std::unexpected(After.error());
+ auto FPCFI = emitRestoreRegisterCFI(AfterFP, Target, *FP);
+ if (!FPCFI)
+ return std::unexpected(FPCFI.error());
+ }
+ } else if (StackSize != 0) {
+ if (StackSize > static_cast<std::uint64_t>(std::numeric_limits<std::int64_t>::max()))
+ return makeUnexpected(ErrorCode::FrameLayoutFailed,
+ "RISC-V stack adjustment exceeds int64");
+ MachineIRBuilder Teardown(Function, Target);
+ auto Point = Teardown.setInsertPointBefore(Terminator);
+ if (!Point)
+ return std::unexpected(Point.error());
+ const std::int64_t Amount = static_cast<std::int64_t>(StackSize);
+ if (fitsSigned12(Amount)) {
+ auto Adjust = emitAddImmediate(Teardown, *SP, *SP, Amount, MachineInstrFlag::FrameDestroy);
+ if (!Adjust)
+ return std::unexpected(Adjust.error());
+ } else {
+ auto Candidates = scratchRegisters();
+ auto Zero = physical(PhysicalRegisterID::X0);
+ if (!Candidates || !Zero)
+ return std::unexpected(!Candidates ? Candidates.error() : Zero.error());
+ auto Scratch = Scavenger.scavenge(Terminator, *Candidates);
+ if (!Scratch)
+ return std::unexpected(Scratch.error());
+ if (Scratch->RequiresSpill) {
+ return makeUnexpected(ErrorCode::RegisterAllocationFailed,
+ "RISC-V epilogue has no dead scratch register");
+ }
+ auto Constant =
+ emitConstant(Teardown, Scratch->Reg, *Zero, Amount, MachineInstrFlag::FrameDestroy);
+ if (!Constant)
+ return std::unexpected(Constant.error());
+ auto Adjust =
+ emitRegisterAdd(Teardown, *SP, *SP, Scratch->Reg, MachineInstrFlag::FrameDestroy);
+ if (!Adjust)
+ return std::unexpected(Adjust.error());
+ }
+ auto CFA = emitDefCFI(Teardown, Target, *SP, 0);
+ if (!CFA)
+ return std::unexpected(CFA.error());
+ }
+ if (HasFrameCFI) {
+ MachineIRBuilder RestoreBuilder(Function, Target);
+ auto After = RestoreBuilder.setInsertPointAfter(Terminator);
+ if (!After)
+ return std::unexpected(After.error());
+ auto Restore = emitCFI(RestoreBuilder, MachineCFIInstruction::restoreState());
+ if (!Restore)
+ return std::unexpected(Restore.error());
+ }
+ return {};
+ }
+
+ [[nodiscard]] Expected<FrameIndexReference>
+ getFrameIndexReference(const MachineFunction &Function, const MachineInstr &Instruction,
+ FrameIndex Index, std::int64_t OperandOffset) const override {
+ const StackObject *Object = Function.frameInfo().object(Index);
+ if (Object == nullptr || Object->description()->IsDead)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V FrameIndex references an invalid stack object");
+ if (Object->isVariableSized())
+ return makeUnexpected(ErrorCode::Unsupported,
+ "RISC-V variable-sized object cannot be addressed by FrameIndex");
+ auto Offset =
+ objectReferenceOffset(Function, Instruction, *Object->description(), OperandOffset);
+ if (!Offset)
+ return std::unexpected(Offset.error());
+ const StackObjectDesc &Description = *Object->description();
+ PhysicalRegisterID BaseID = PhysicalRegisterID::X2;
+ if (!isOutgoingCallObject(Description) &&
+ !Instruction.flags().has(MachineInstrFlag::FrameSetup)) {
+ const bool FixedObject = Description.Kind == StackObjectKind::FixedABI ||
+ Description.Kind == StackObjectKind::FixedSpillSlot;
+ const bool HasFP = requiresFramePointer(Function);
+ const bool HasBP = requiresBasePointer(Function);
+ if (HasFP && (FixedObject || !HasBP)) {
+ BaseID = PhysicalRegisterID::X8;
+ } else if (HasBP) {
+ BaseID = PhysicalRegisterID::X9;
+ }
+ }
+ auto Base = physical(BaseID);
+ if (!Base)
+ return std::unexpected(Base.error());
+ return FrameIndexReference{*Base, *Offset};
+ }
+
+ [[nodiscard]] Expected<void> eliminateFrameIndex(MachineInstr &Instruction,
+ std::size_t OperandIndex,
+ MachineFunction &Function,
+ const TargetDescription &Target,
+ RegisterScavenger &Scavenger) const override {
+ const auto *Frame = Instruction.operandAt(OperandIndex)->getIf<FrameIndexOperand>();
+ if (Frame == nullptr)
+ return makeUnexpected(ErrorCode::InvalidArgument,
+ "RISC-V frame elimination requires a FrameIndex operand");
+ if (OperandIndex + 1U >= Instruction.operandCount())
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V frame operand lacks an adjacent immediate");
+ auto Immediate = immediateValue(*Instruction.operandAt(OperandIndex + 1U));
+ if (!Immediate)
+ return std::unexpected(Immediate.error());
+ auto Extra = checkedAdd(Frame->Offset, *Immediate);
+ if (!Extra)
+ return std::unexpected(Extra.error());
+ auto Reference = getFrameIndexReference(Function, Instruction, Frame->Index, *Extra);
+ if (!Reference)
+ return std::unexpected(Reference.error());
+ if (fitsSigned12(Reference->Offset)) {
+ auto Base = replaceWithRegister(Instruction, OperandIndex, Reference->BaseRegister);
+ if (!Base)
+ return std::unexpected(Base.error());
+ return replaceImmediate(Instruction, OperandIndex + 1U, Reference->Offset);
+ }
+
+ auto Candidates = scratchRegisters();
+ if (!Candidates)
+ return std::unexpected(Candidates.error());
+ auto Scratch = Scavenger.scavenge(Instruction, *Candidates);
+ if (!Scratch)
+ return std::unexpected(Scratch.error());
+ std::optional<FrameIndexReference> Emergency;
+ if (Scratch->RequiresSpill) {
+ const StackObject *Object = emergencyObject(Function);
+ if (Object == nullptr)
+ return makeUnexpected(ErrorCode::RegisterAllocationFailed,
+ "RISC-V scavenging requires an emergency spill slot");
+ auto Ref = emergencyFrameReference(Function, *Object->description());
+ if (!Ref)
+ return std::unexpected(Ref.error());
+ if (!fitsSigned12(Ref->Offset)) {
+ return makeUnexpected(ErrorCode::RegisterAllocationFailed,
+ "RISC-V emergency spill slot is out of range");
+ }
+ Emergency = *Ref;
+ auto Saved =
+ emitResolvedScratchAccess(Instruction, Function, Target, Scratch->Reg, *Emergency, false);
+ if (!Saved)
+ return std::unexpected(Saved.error());
+ }
+
+ MachineIRBuilder Builder(Function, Target);
+ auto Point = Builder.setInsertPointBefore(Instruction);
+ if (!Point)
+ return std::unexpected(Point.error());
+ auto Zero = physical(PhysicalRegisterID::X0);
+ if (!Zero)
+ return std::unexpected(Zero.error());
+ auto Constant = emitConstant(Builder, Scratch->Reg, *Zero, Reference->Offset, std::nullopt);
+ if (!Constant)
+ return std::unexpected(Constant.error());
+ auto Address =
+ emitRegisterAdd(Builder, Scratch->Reg, Reference->BaseRegister, Scratch->Reg, std::nullopt);
+ if (!Address)
+ return std::unexpected(Address.error());
+ auto Base = replaceWithRegister(Instruction, OperandIndex, Scratch->Reg);
+ if (!Base)
+ return std::unexpected(Base.error());
+ auto Offset = replaceImmediate(Instruction, OperandIndex + 1U, 0);
+ if (!Offset)
+ return std::unexpected(Offset.error());
+ if (Emergency) {
+ auto Restored =
+ emitResolvedScratchAccess(Instruction, Function, Target, Scratch->Reg, *Emergency, true);
+ if (!Restored)
+ return std::unexpected(Restored.error());
+ }
+ return {};
+ }
+
+private:
+ [[nodiscard]] Expected<void> emitStackAdjustment(MachineInstr &Anchor, MachineFunction &Function,
+ const TargetDescription &Target,
+ RegisterScavenger &Scavenger,
+ std::int64_t Amount,
+ MachineInstrFlag Flag) const {
+ if (Amount == 0)
+ return {};
+ auto SP = physical(PhysicalRegisterID::X2);
+ if (!SP)
+ return std::unexpected(SP.error());
+ MachineIRBuilder Builder(Function, Target);
+ auto Point = Builder.setInsertPointBefore(Anchor);
+ if (!Point)
+ return std::unexpected(Point.error());
+ if (fitsSigned12(Amount))
+ return emitAddImmediate(Builder, *SP, *SP, Amount, Flag);
+
+ auto Candidates = scratchRegisters();
+ if (!Candidates)
+ return std::unexpected(Candidates.error());
+ auto Scratch = Scavenger.scavenge(Anchor, *Candidates);
+ if (!Scratch)
+ return std::unexpected(Scratch.error());
+ std::optional<FrameIndexReference> Emergency;
+ if (Scratch->RequiresSpill) {
+ const StackObject *Object = emergencyObject(Function);
+ if (Object == nullptr)
+ return makeUnexpected(ErrorCode::RegisterAllocationFailed,
+ "RISC-V stack adjustment requires an emergency spill slot");
+ auto Ref = emergencyFrameReference(Function, *Object->description());
+ if (!Ref || !fitsSigned12(Ref->Offset))
+ return makeUnexpected(ErrorCode::RegisterAllocationFailed,
+ "RISC-V emergency spill slot is unavailable");
+ Emergency = *Ref;
+ auto Saved =
+ emitResolvedScratchAccess(Anchor, Function, Target, Scratch->Reg, *Emergency, false);
+ if (!Saved)
+ return std::unexpected(Saved.error());
+ }
+ auto Zero = physical(PhysicalRegisterID::X0);
+ if (!Zero)
+ return std::unexpected(Zero.error());
+ auto Constant = emitConstant(Builder, Scratch->Reg, *Zero, Amount, Flag);
+ if (!Constant)
+ return std::unexpected(Constant.error());
+ auto Adjust = emitRegisterAdd(Builder, *SP, *SP, Scratch->Reg, Flag);
+ if (!Adjust)
+ return std::unexpected(Adjust.error());
+ if (Emergency) {
+ auto Restored =
+ emitResolvedScratchAccess(Anchor, Function, Target, Scratch->Reg, *Emergency, true);
+ if (!Restored)
+ return std::unexpected(Restored.error());
+ }
+ return {};
+ }
+
+ [[nodiscard]] Expected<void>
+ emitResolvedScratchAccess(MachineInstr &Anchor, MachineFunction &Function,
+ const TargetDescription &Target, PhysicalRegister Scratch,
+ FrameIndexReference Reference, bool Restore) const {
+ MachineIRBuilder Builder(Function, Target);
+ auto Point =
+ Restore ? Builder.setInsertPointAfter(Anchor) : Builder.setInsertPointBefore(Anchor);
+ if (!Point)
+ return std::unexpected(Point.error());
+ auto Memory = createResolvedStackMemory(
+ Function, Restore ? MemoryAccessFlag::Load : MemoryAccessFlag::Store, Reference.Offset);
+ if (!Memory)
+ return std::unexpected(Memory.error());
+ auto Pending =
+ Builder.buildInstr(targetOpcode(Restore ? InstructionOpcode::LD : InstructionOpcode::SD));
+ if (!Pending)
+ return std::unexpected(Pending.error());
+ auto Value = Restore ? Pending->addDef(Register::fromPhysical(Scratch))
+ : Pending->addUse(Register::fromPhysical(Scratch));
+ if (!Value)
+ return std::unexpected(Value.error());
+ if (auto Result = Pending->addUse(Register::fromPhysical(Reference.BaseRegister)); !Result)
+ return std::unexpected(Result.error());
+ if (auto Result = Pending->addImmediate(Reference.Offset); !Result)
+ return std::unexpected(Result.error());
+ if (auto Result = Pending->addMemOperand(**Memory); !Result)
+ return std::unexpected(Result.error());
+ auto Built = Pending->build();
+ if (!Built)
+ return std::unexpected(Built.error());
+ return {};
+ }
+};
+
+} // namespace
+
+const TargetFrameLowering &frameLowering() noexcept {
+ static const RISCVFrameLowering Info;
+ return Info;
+}
+
+} // namespace rcc::riscv
diff --git a/backend/lib/Target/RISCV/RISCVRegisterAllocation.cpp b/backend/lib/Target/RISCV/RISCVRegisterAllocation.cpp
index e437e18..81fc6dc 100644
--- a/backend/lib/Target/RISCV/RISCVRegisterAllocation.cpp
+++ b/backend/lib/Target/RISCV/RISCVRegisterAllocation.cpp
@@ -1,6 +1,7 @@
// SPDX-License-Identifier: GPL-3.0-or-later
#include "rcc/CodeGen/InstructionDescriptor.h"
+#include "rcc/CodeGen/MachineBasicBlock.h"
#include "rcc/CodeGen/MachineFunction.h"
#include "rcc/CodeGen/MachineIRBuilder.h"
#include "rcc/CodeGen/MachineInstr.h"
@@ -13,6 +14,7 @@
#include <algorithm>
#include <array>
+#include <bit>
#include <cstddef>
#include <cstdint>
#include <optional>
@@ -35,6 +37,20 @@ struct SpillForm final {
return RegisterClassID::fromValue(static_cast<std::uint32_t>(Value));
}
+[[nodiscard]] std::optional<std::int64_t> immediateValue(const MachineOperand &Operand) noexcept {
+ if (const auto *Immediate = Operand.getIf<std::int64_t>())
+ return *Immediate;
+ const auto *Integer = Operand.getIf<IntegerImmediateOperand>();
+ if (Integer == nullptr || Integer->Value.bitWidth() == 0 || Integer->Value.bitWidth() > 64) {
+ return std::nullopt;
+ }
+ std::uint64_t Raw = Integer->Value.words().empty() ? 0 : Integer->Value.words().front();
+ const std::uint32_t Width = Integer->Value.bitWidth();
+ if (Width < 64 && (Raw & (std::uint64_t{1} << (Width - 1U))) != 0)
+ Raw |= ~std::uint64_t{0} << Width;
+ return std::bit_cast<std::int64_t>(Raw);
+}
+
[[nodiscard]] constexpr std::optional<SpillForm> spillForm(RegisterClassID Class) noexcept {
if (Class == registerClassID(RegisterClass::GPR) ||
Class == registerClassID(RegisterClass::GPRC) ||
@@ -102,6 +118,74 @@ createStackMemoryOperand(MachineIRBuilder &Builder, FrameIndex StackSlot, Memory
class RISCVRegisterAllocationInfo final : public TargetRegisterAllocationInfo {
public:
+ [[nodiscard]] bool
+ isReservedForFunction(const MachineFunction &Function, PhysicalRegister Reg,
+ const TargetDescription &Target) const noexcept override {
+ std::uint64_t RequiredAlignment = 16;
+ RequiredAlignment = std::max(
+ RequiredAlignment, Function.frameInfo().state().MaxAlignment.value_or(Alignment{}).bytes());
+ for (const StackObject &Object : Function.frameInfo().objects()) {
+ if (!Object.isTombstone() && !Object.description()->IsDead) {
+ RequiredAlignment = std::max(RequiredAlignment, Object.description()->Align.bytes());
+ }
+ }
+ for (const MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr;
+ Block = Block->next()) {
+ for (const MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr;
+ Instruction = Instruction->next()) {
+ if (Instruction->opcode() != targetOpcode(InstructionOpcode::ADJCALLSTACKDOWN) &&
+ Instruction->opcode() != targetOpcode(InstructionOpcode::ADJCALLSTACKUP)) {
+ continue;
+ }
+ if (Instruction->operandCount() < 2)
+ continue;
+ const auto Requested = immediateValue(*Instruction->operandAt(1));
+ if (Requested && *Requested > 0 &&
+ std::has_single_bit(static_cast<std::uint64_t>(*Requested))) {
+ RequiredAlignment = std::max(RequiredAlignment, static_cast<std::uint64_t>(*Requested));
+ }
+ }
+ }
+ const bool RequiresRealignment = RequiredAlignment > 16;
+ if (Reg.number() == static_cast<std::uint32_t>(PhysicalRegisterID::X9))
+ return RequiresRealignment;
+ if (Reg.number() != static_cast<std::uint32_t>(PhysicalRegisterID::X8))
+ return false;
+ const MachineFrameState &State = Function.frameInfo().state();
+ bool HasCalls = State.HasCalls;
+ for (const MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr && !HasCalls;
+ Block = Block->next()) {
+ for (const MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr;
+ Instruction = Instruction->next()) {
+ const InstructionDescriptor *Descriptor =
+ Target.instructionDescriptor(Instruction->opcode());
+ if (Descriptor != nullptr && Descriptor->Flags.has(InstructionFlag::Call)) {
+ HasCalls = true;
+ break;
+ }
+ }
+ }
+ const bool HasVariableObject =
+ std::ranges::any_of(Function.frameInfo().objects(), [](const StackObject &Object) {
+ return !Object.isTombstone() && !Object.description()->IsDead && Object.isVariableSized();
+ });
+ if (State.IsFrameAddressTaken || State.HasOpaqueSPAdjustment || HasVariableObject ||
+ RequiresRealignment)
+ return true;
+ switch (State.FramePointerPolicy) {
+ case MachineFramePointerPolicy::All:
+ case MachineFramePointerPolicy::Reserved:
+ return true;
+ case MachineFramePointerPolicy::NonLeaf:
+ case MachineFramePointerPolicy::NonLeafNoReserve:
+ return HasCalls;
+ case MachineFramePointerPolicy::None:
+ case MachineFramePointerPolicy::Count:
+ return false;
+ }
+ return false;
+ }
+
[[nodiscard]] Expected<void> validateSpillReload(RegisterClassID Class,
const TargetDescription &Target) const override {
const RegisterClassDesc *Description = Target.registers().registerClass(Class);
generated by cgit v1.2.3 (git 2.46.0) at 2026-10-29 11:27:13 +0000