aboutsummaryrefslogtreecommitdiff
path: root/backend/lib/Target/RISCV/GISel
diff options
context:
space:
mode:
Diffstat (limited to 'backend/lib/Target/RISCV/GISel')
-rw-r--r--backend/lib/Target/RISCV/GISel/RISCVCallLowering.cpp1102
-rw-r--r--backend/lib/Target/RISCV/GISel/RISCVCallLowering.h49
-rw-r--r--backend/lib/Target/RISCV/GISel/RISCVGISelCombiner.cpp56
-rw-r--r--backend/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp1343
-rw-r--r--backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp1382
-rw-r--r--backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.h37
-rw-r--r--backend/lib/Target/RISCV/GISel/RISCVRegisterBankInfo.cpp388
-rw-r--r--backend/lib/Target/RISCV/GISel/RISCVRegisterBankInfo.h30
8 files changed, 4387 insertions, 0 deletions
diff --git a/backend/lib/Target/RISCV/GISel/RISCVCallLowering.cpp b/backend/lib/Target/RISCV/GISel/RISCVCallLowering.cpp
new file mode 100644
index 0000000..3909020
--- /dev/null
+++ b/backend/lib/Target/RISCV/GISel/RISCVCallLowering.cpp
@@ -0,0 +1,1102 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "RISCVCallLowering.h"
+
+#include "../RISCVCallingConv.h"
+#include "rcc/CodeGen/GenericOpcodes.h"
+#include "rcc/CodeGen/MachineBasicBlock.h"
+#include "rcc/CodeGen/MachineFrameInfo.h"
+#include "rcc/CodeGen/MachineFunction.h"
+#include "rcc/CodeGen/MachineIRBuilder.h"
+#include "rcc/CodeGen/MachineInstr.h"
+#include "rcc/CodeGen/MachineMemOperand.h"
+#include "rcc/CodeGen/MachineRegisterInfo.h"
+#include "rcc/CodeGen/TargetDescription.h"
+#include "rcc/Target/RISCV/RISCVTarget.h"
+
+#include <algorithm>
+#include <array>
+#include <cstddef>
+#include <cstdint>
+#include <limits>
+#include <optional>
+#include <span>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+#include <utility>
+#include <vector>
+
+namespace rcc::riscv {
+namespace {
+
+constexpr std::string_view PlanTarget = "riscv64-unknown-linux-gnu";
+constexpr std::uint32_t AllowedDirectCallFlags =
+ static_cast<std::uint32_t>(CallTargetFlag::DirectCall);
+
+[[nodiscard]] std::uint64_t stackObjectFlag(CallStackObjectFlag Flag) noexcept {
+ return static_cast<std::uint64_t>(Flag);
+}
+
+[[nodiscard]] Expected<Alignment> alignment(std::uint64_t Bytes) {
+ return Alignment::create(Bytes);
+}
+
+[[nodiscard]] Expected<std::uint64_t> alignTo(std::uint64_t Value, std::uint64_t AlignmentBytes) {
+ const std::uint64_t Remainder = Value % AlignmentBytes;
+ if (Remainder == 0)
+ return Value;
+ const std::uint64_t Increment = AlignmentBytes - Remainder;
+ if (Value > std::numeric_limits<std::uint64_t>::max() - Increment) {
+ return makeUnexpected(ErrorCode::Unsupported, "RISC-V ABI stack-area size overflows uint64");
+ }
+ return Value + Increment;
+}
+
+} // namespace
+
+Expected<CallLoweringPlan>
+RISCVCallLowering::planFormalArguments(const FormalArgumentRequest &Request) const {
+ auto Convention = riscvConvention(Request.Convention);
+ if (!Convention)
+ return std::unexpected(Convention.error());
+ auto OriginalCount = verifyValueSequence(Request.Arguments, true);
+ if (!OriginalCount)
+ return std::unexpected(OriginalCount.error());
+ if (Request.FixedArgumentCount != *OriginalCount) {
+ return makeUnexpected(ErrorCode::InvalidArgument,
+ "formal-argument request must identify every declared argument as fixed");
+ }
+ if (!Request.IsVariadic && Request.FixedArgumentCount != *OriginalCount) {
+ return makeUnexpected(ErrorCode::InvalidArgument,
+ "non-variadic formal request has an invalid fixed-argument boundary");
+ }
+ if (hasStructReturn(Request.Arguments) &&
+ (Request.Arguments.empty() || Request.Arguments.front().OriginalValue != 0 ||
+ !Request.Arguments.front().Flags.has(ABIValueFlag::StructReturn))) {
+ return makeUnexpected(ErrorCode::InvalidArgument,
+ "RISC-V sret must be the first formal argument");
+ }
+
+ auto Assignments = assignArguments(Request.Arguments, Request.FixedArgumentCount,
+ Request.IsVariadic, **Convention);
+ if (!Assignments)
+ return std::unexpected(Assignments.error());
+
+ VarArgLoweringState VarArgs;
+ if (Request.IsVariadic) {
+ VarArgs.IsVariadic = true;
+ VarArgs.FixedArgumentCount = Request.FixedArgumentCount;
+ VarArgs.FirstUnallocatedIntegerRegister = Assignments->FirstUnallocatedGPR;
+ VarArgs.OverflowAreaOffsetBytes = Assignments->StackArea.SizeBytes;
+ const std::size_t First = Assignments->FirstUnallocatedGPR;
+ const std::size_t RegisterCount = (*Convention)->IntegerArgumentRegisters.size();
+ VarArgs.RegisterSaveAreaSizeBytes = (RegisterCount - First) * XLenBytes;
+ VarArgs.RegisterSaveAreaOffsetBytes =
+ VarArgs.RegisterSaveAreaSizeBytes == 0
+ ? static_cast<std::int64_t>(VarArgs.OverflowAreaOffsetBytes)
+ : -static_cast<std::int64_t>(VarArgs.RegisterSaveAreaSizeBytes);
+ if (VarArgs.RegisterSaveAreaSizeBytes != 0 && First % 2 != 0)
+ VarArgs.RegisterSavePaddingBytes = XLenBytes;
+ for (std::size_t Index = First; Index < RegisterCount; ++Index) {
+ VarArgs.RegistersToSave.push_back(
+ {(*Convention)->IntegerArgumentRegisters[Index], (Index - First) * XLenBytes});
+ }
+ }
+
+ CallLoweringPlanDesc Plan;
+ Plan.Kind = CallLoweringPlanKind::FormalArguments;
+ Plan.Target = PlanTarget;
+ Plan.Convention = Request.Convention;
+ Plan.ArgumentAssignments = std::move(Assignments->Assignments);
+ Plan.StackArguments = Assignments->StackArea;
+ Plan.VarArgs = std::move(VarArgs);
+ return CallLoweringPlan::create(std::move(Plan));
+}
+
+Expected<CallLoweringPlan> RISCVCallLowering::planCall(const CallLoweringRequest &Request) const {
+ auto Convention = riscvConvention(Request.Convention);
+ if (!Convention)
+ return std::unexpected(Convention.error());
+ auto OriginalCount = verifyValueSequence(Request.Arguments, true);
+ if (!OriginalCount)
+ return std::unexpected(OriginalCount.error());
+ if ((!Request.IsVariadic && Request.FixedArgumentCount != *OriginalCount) ||
+ (Request.IsVariadic && Request.FixedArgumentCount > *OriginalCount)) {
+ return makeUnexpected(ErrorCode::InvalidArgument,
+ "call request has an invalid fixed/variadic argument boundary");
+ }
+ if (!Request.Callee.isValid())
+ return makeUnexpected(ErrorCode::InvalidArgument, "call request requires a valid callee");
+ if (hasStructReturn(Request.Arguments) &&
+ (Request.Arguments.empty() ||
+ !Request.Arguments.front().Flags.has(ABIValueFlag::StructReturn))) {
+ return makeUnexpected(ErrorCode::InvalidArgument,
+ "RISC-V sret must be the first call argument");
+ }
+ if (hasStructReturn(Request.Arguments) && !Request.Results.empty()) {
+ return makeUnexpected(ErrorCode::InvalidArgument,
+ "RISC-V sret calls cannot also request direct return values");
+ }
+
+ MachineCallTarget EffectiveCallee = Request.Callee;
+ switch (Request.Callee.kind()) {
+ case MachineCallTargetKind::GlobalSymbol: {
+ if ((Request.Callee.targetFlags() & ~AllowedDirectCallFlags) != 0) {
+ return makeUnexpected(ErrorCode::InvalidArgument,
+ "direct RISC-V call uses unknown target flags");
+ }
+ auto Callee = MachineCallTarget::globalSymbol(
+ std::string(Request.Callee.symbolName()),
+ Request.Callee.targetFlags() | static_cast<std::uint32_t>(CallTargetFlag::DirectCall));
+ if (!Callee)
+ return std::unexpected(Callee.error());
+ EffectiveCallee = std::move(*Callee);
+ break;
+ }
+ case MachineCallTargetKind::ExternalSymbol: {
+ if ((Request.Callee.targetFlags() & ~AllowedDirectCallFlags) != 0) {
+ return makeUnexpected(ErrorCode::InvalidArgument,
+ "direct RISC-V call uses unknown target flags");
+ }
+ auto Callee = MachineCallTarget::externalSymbol(
+ std::string(Request.Callee.symbolName()),
+ Request.Callee.targetFlags() | static_cast<std::uint32_t>(CallTargetFlag::DirectCall));
+ if (!Callee)
+ return std::unexpected(Callee.error());
+ EffectiveCallee = std::move(*Callee);
+ break;
+ }
+ case MachineCallTargetKind::IndirectRegister:
+ if (Request.Callee.targetFlags() != 0) {
+ return makeUnexpected(ErrorCode::InvalidArgument,
+ "indirect RISC-V calls cannot carry direct-call target flags");
+ }
+ break;
+ case MachineCallTargetKind::Invalid:
+ return makeUnexpected(ErrorCode::InvalidArgument, "call target is invalid");
+ }
+
+ const bool RequireResultRegisters = Request.TailKind == TailCallKind::NotTail;
+ auto ResultSequence = verifyValueSequence(Request.Results, RequireResultRegisters);
+ if (!ResultSequence)
+ return std::unexpected(ResultSequence.error());
+ auto Arguments = assignArguments(Request.Arguments, Request.FixedArgumentCount,
+ Request.IsVariadic, **Convention);
+ if (!Arguments)
+ return std::unexpected(Arguments.error());
+ auto Results = assignReturns(Request.Results, RequireResultRegisters, **Convention);
+ if (!Results)
+ return std::unexpected(Results.error());
+
+ auto Align16 = alignment(ABIStackAlignmentBytes);
+ if (!Align16)
+ return std::unexpected(Align16.error());
+ std::uint64_t CallFrameCursor = Arguments->StackArea.SizeBytes;
+ std::vector<ByValCopyAssignment> ByValCopies;
+ for (std::size_t Index = 0; Index < Arguments->Assignments.size(); ++Index) {
+ const CCAssignment &Assignment = Arguments->Assignments[Index];
+ if (!Assignment.CustomFlags.has(CCAssignmentFlag::ByVal))
+ continue;
+ const ABIValue &Value = Request.Arguments[Assignment.ValueIndex];
+ const std::uint64_t CopyAlignment =
+ std::min<std::uint64_t>(Value.ByValAlignment.bytes(), ABIStackAlignmentBytes);
+ auto CopyOffset = alignTo(CallFrameCursor, CopyAlignment);
+ if (!CopyOffset)
+ return std::unexpected(CopyOffset.error());
+ if (*CopyOffset > std::numeric_limits<std::uint64_t>::max() - Value.ByValSizeBytes) {
+ return makeUnexpected(ErrorCode::Unsupported, "RISC-V byval area overflows uint64");
+ }
+ auto CopyAlign = alignment(CopyAlignment);
+ if (!CopyAlign)
+ return std::unexpected(CopyAlign.error());
+ ByValCopies.push_back(
+ {Index, Value.Value, Value.Type, *CopyOffset, Value.ByValSizeBytes, *CopyAlign});
+ CallFrameCursor = *CopyOffset + Value.ByValSizeBytes;
+ }
+ auto CallFrameSize = alignTo(CallFrameCursor, ABIStackAlignmentBytes);
+ if (!CallFrameSize)
+ return std::unexpected(CallFrameSize.error());
+
+ TailCallKind Decision = TailCallKind::NotTail;
+ std::string RejectionReason;
+ if (Request.TailKind != TailCallKind::NotTail) {
+ auto Rejection = tailCallRejection(Request, *Arguments, *Results, ByValCopies, **Convention);
+ if (!Rejection)
+ return std::unexpected(Rejection.error());
+ RejectionReason = std::move(*Rejection);
+ if (RejectionReason.empty()) {
+ Decision = Request.TailKind;
+ } else if (Request.TailKind == TailCallKind::MustTail) {
+ return makeUnexpected(ErrorCode::Unsupported,
+ "musttail call is not ABI-compatible: " + RejectionReason);
+ } else {
+ auto NormalResults = verifyValueSequence(Request.Results, true);
+ if (!NormalResults)
+ return std::unexpected(NormalResults.error());
+ }
+ }
+
+ CallLoweringPlanDesc Plan;
+ Plan.Kind = CallLoweringPlanKind::Call;
+ Plan.Target = PlanTarget;
+ Plan.Convention = Request.Convention;
+ Plan.ArgumentAssignments = std::move(Arguments->Assignments);
+ Plan.ReturnAssignments = std::move(*Results);
+ Plan.StackArguments = Arguments->StackArea;
+ Plan.CallFrame = CallFrameRequirements{
+ Decision == TailCallKind::NotTail ? *CallFrameSize : Arguments->StackArea.SizeBytes, *Align16,
+ Decision == TailCallKind::NotTail, Decision == TailCallKind::NotTail,
+ Decision != TailCallKind::NotTail};
+ if (Request.IsVariadic) {
+ Plan.VarArgs.IsVariadic = true;
+ Plan.VarArgs.FixedArgumentCount = Request.FixedArgumentCount;
+ }
+ Plan.ByValCopies = std::move(ByValCopies);
+ Plan.PreservedMask = createCallPreservedMask((*Convention)->ID);
+ if (!Plan.PreservedMask)
+ return makeUnexpected(ErrorCode::InternalError, "RISC-V call-preserved mask is unavailable");
+ Plan.Callee = std::move(EffectiveCallee);
+ Plan.TailDecision = Decision;
+ Plan.TailRejectionReason = std::move(RejectionReason);
+ return CallLoweringPlan::create(std::move(Plan));
+}
+
+Expected<CallLoweringPlan>
+RISCVCallLowering::planReturn(const ReturnLoweringRequest &Request) const {
+ auto Convention = riscvConvention(Request.Convention);
+ if (!Convention)
+ return std::unexpected(Convention.error());
+ auto Values = assignReturns(Request.Values, true, **Convention);
+ if (!Values)
+ return std::unexpected(Values.error());
+
+ CallLoweringPlanDesc Plan;
+ Plan.Kind = CallLoweringPlanKind::Return;
+ Plan.Target = PlanTarget;
+ Plan.Convention = Request.Convention;
+ Plan.ReturnAssignments = std::move(*Values);
+ return CallLoweringPlan::create(std::move(Plan));
+}
+
+[[nodiscard]] CallValueTransform outgoingTransform(const CCAssignment &Assignment) noexcept {
+ if (Assignment.CustomFlags.has(CCAssignmentFlag::Bitcast))
+ return CallValueTransform::Bitcast;
+ switch (Assignment.Extension) {
+ case ABIExtensionKind::None:
+ return CallValueTransform::Full;
+ case ABIExtensionKind::AnyExtend:
+ return CallValueTransform::AnyExtend;
+ case ABIExtensionKind::SignExtend:
+ return CallValueTransform::SignExtend;
+ case ABIExtensionKind::ZeroExtend:
+ return CallValueTransform::ZeroExtend;
+ case ABIExtensionKind::Count:
+ break;
+ }
+ return CallValueTransform::Full;
+}
+
+[[nodiscard]] CallValueTransform incomingTransform(const CCAssignment &Assignment) noexcept {
+ if (Assignment.CustomFlags.has(CCAssignmentFlag::Bitcast))
+ return CallValueTransform::Bitcast;
+ if (Assignment.ValueType.scalarBitWidth() < Assignment.LocationType.scalarBitWidth())
+ return CallValueTransform::Truncate;
+ return CallValueTransform::Full;
+}
+
+[[nodiscard]] Expected<void> addUnsignedImmediate(MachineInstrBuilder &Builder,
+ std::uint64_t Value) {
+ auto Immediate = APInt::fromUnsigned(64, Value);
+ if (!Immediate)
+ return std::unexpected(Immediate.error());
+ return Builder.addIntegerImmediate(std::move(*Immediate));
+}
+
+[[nodiscard]] Expected<const MachineMemOperand *>
+createMemoryOperand(MachineFunction &Function, MemoryAccessFlags Accesses, std::uint64_t SizeBytes,
+ Alignment Align, MemoryBase Base, std::int64_t Offset = 0) {
+ auto Size = MemorySize::fixed(SizeBytes);
+ if (!Size)
+ return std::unexpected(Size.error());
+ MachineMemOperandDesc Description;
+ Description.Accesses = Accesses;
+ Description.Size = *Size;
+ Description.Align = Align;
+ Description.Base = std::move(Base);
+ Description.Offset = Offset;
+ return Function.createMemOperand(std::move(Description));
+}
+
+[[nodiscard]] Expected<Register> extendOutgoingRegister(MachineIRBuilder &Builder, Register Source,
+ const CCAssignment &Assignment) {
+ const std::uint32_t ValueWidth = Assignment.ValueType.scalarBitWidth();
+ const std::uint32_t LocationWidth = Assignment.LocationType.scalarBitWidth();
+ if (ValueWidth == LocationWidth)
+ return Source;
+ if (ValueWidth > LocationWidth) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V ABI register assignment narrows an outgoing value");
+ }
+
+ auto Extended = Builder.function().registerInfo().createVirtualRegister(Assignment.LocationType);
+ if (!Extended)
+ return std::unexpected(Extended.error());
+
+ Expected<MachineInstr *> Built = makeUnexpected(
+ ErrorCode::InternalError, "RISC-V ABI outgoing register extension was not initialized");
+ switch (outgoingTransform(Assignment)) {
+ case CallValueTransform::AnyExtend:
+ case CallValueTransform::Bitcast:
+ Built = Builder.buildAnyExt(*Extended, Source);
+ break;
+ case CallValueTransform::SignExtend:
+ Built = Builder.buildSExt(*Extended, Source);
+ break;
+ case CallValueTransform::ZeroExtend:
+ Built = Builder.buildZExt(*Extended, Source);
+ break;
+ case CallValueTransform::Full:
+ case CallValueTransform::Truncate:
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V ABI outgoing register assignment lacks an extension transform");
+ }
+ if (!Built)
+ return std::unexpected(Built.error());
+ return *Extended;
+}
+
+struct OutgoingRegisterValue final {
+ PhysicalRegister Location;
+ Register Value;
+};
+
+[[nodiscard]] Expected<OutgoingRegisterValue>
+prepareOutgoingRegisterAssignment(MachineIRBuilder &Builder, const CCAssignment &Assignment,
+ Register Source) {
+ if (!Assignment.LocationRegister) {
+ return makeUnexpected(ErrorCode::InternalError,
+ "RISC-V register assignment has no location register");
+ }
+ auto Extended = extendOutgoingRegister(Builder, Source, Assignment);
+ if (!Extended)
+ return std::unexpected(Extended.error());
+ return OutgoingRegisterValue{
+ .Location = *Assignment.LocationRegister,
+ .Value = *Extended,
+ };
+}
+
+[[nodiscard]] Expected<void>
+emitOutgoingRegisterAssignments(MachineIRBuilder &Builder,
+ std::span<const OutgoingRegisterValue> Assignments) {
+ for (const OutgoingRegisterValue &Assignment : Assignments) {
+ auto Copy = Builder.buildCopy(Register::fromPhysical(Assignment.Location), Assignment.Value);
+ if (!Copy)
+ return std::unexpected(Copy.error());
+ }
+ return {};
+}
+
+[[nodiscard]] Expected<void> emitIncomingRegisterAssignment(MachineIRBuilder &Builder,
+ const CCAssignment &Assignment) {
+ if (!Assignment.LocationRegister) {
+ return makeUnexpected(ErrorCode::InternalError,
+ "RISC-V register assignment has no location register");
+ }
+
+ const std::uint32_t ValueWidth = Assignment.ValueType.scalarBitWidth();
+ const std::uint32_t LocationWidth = Assignment.LocationType.scalarBitWidth();
+ if (ValueWidth == LocationWidth) {
+ auto Copy = Builder.buildCopy(Assignment.OriginalValue,
+ Register::fromPhysical(*Assignment.LocationRegister));
+ if (!Copy)
+ return std::unexpected(Copy.error());
+ return {};
+ }
+ if (ValueWidth > LocationWidth) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V ABI register assignment widens an incoming value");
+ }
+
+ auto LocationValue =
+ Builder.function().registerInfo().createVirtualRegister(Assignment.LocationType);
+ if (!LocationValue)
+ return std::unexpected(LocationValue.error());
+ auto Copy =
+ Builder.buildCopy(*LocationValue, Register::fromPhysical(*Assignment.LocationRegister));
+ if (!Copy)
+ return std::unexpected(Copy.error());
+ auto Truncation = Builder.buildTrunc(Assignment.OriginalValue, *LocationValue);
+ if (!Truncation)
+ return std::unexpected(Truncation.error());
+ return {};
+}
+
+[[nodiscard]] Expected<void> emitStackLoad(MachineIRBuilder &Builder,
+ const CCAssignment &Assignment, FrameIndex Object,
+ std::uint64_t ObjectOffset,
+ const MachineMemOperand &Memory) {
+ auto Instruction = Builder.buildInstr(instructionOpcode(InstructionOpcode::PseudoLOAD_STACK_ARG));
+ if (!Instruction)
+ return std::unexpected(Instruction.error());
+ auto Def = Instruction->addDef(Assignment.OriginalValue);
+ if (!Def)
+ return std::unexpected(Def.error());
+ auto Frame = Instruction->addFrameIndex(Object);
+ if (!Frame)
+ return std::unexpected(Frame.error());
+ auto Offset = addUnsignedImmediate(*Instruction, ObjectOffset);
+ if (!Offset)
+ return std::unexpected(Offset.error());
+ auto Transform =
+ Instruction->addImmediate(static_cast<std::int64_t>(incomingTransform(Assignment)));
+ if (!Transform)
+ return std::unexpected(Transform.error());
+ auto Mem = Instruction->addMemOperand(Memory);
+ if (!Mem)
+ return std::unexpected(Mem.error());
+ auto Built = Instruction->build();
+ if (!Built)
+ return std::unexpected(Built.error());
+ return {};
+}
+
+[[nodiscard]] Expected<void> emitStackStore(MachineIRBuilder &Builder, Register Source,
+ FrameIndex Object, std::uint64_t ObjectOffset,
+ const MachineMemOperand &Memory,
+ CallValueTransform Transform) {
+ auto Instruction =
+ Builder.buildInstr(instructionOpcode(InstructionOpcode::PseudoSTORE_STACK_ARG));
+ if (!Instruction)
+ return std::unexpected(Instruction.error());
+ auto Use = Instruction->addUse(Source);
+ if (!Use)
+ return std::unexpected(Use.error());
+ auto Frame = Instruction->addFrameIndex(Object);
+ if (!Frame)
+ return std::unexpected(Frame.error());
+ auto Offset = addUnsignedImmediate(*Instruction, ObjectOffset);
+ if (!Offset)
+ return std::unexpected(Offset.error());
+ auto Action = Instruction->addImmediate(static_cast<std::int64_t>(Transform));
+ if (!Action)
+ return std::unexpected(Action.error());
+ auto Mem = Instruction->addMemOperand(Memory);
+ if (!Mem)
+ return std::unexpected(Mem.error());
+ auto Built = Instruction->build();
+ if (!Built)
+ return std::unexpected(Built.error());
+ return {};
+}
+
+[[nodiscard]] Expected<void> ensureFunctionLiveIn(MachineRegisterInfo &Registers,
+ PhysicalRegister Reg,
+ std::optional<Register> InitialValue) {
+ const auto Existing =
+ std::ranges::find(Registers.liveIns(), Reg, &PhysicalRegisterLiveIn::PhysReg);
+ if (Existing == Registers.liveIns().end())
+ return Registers.addLiveIn(Reg, InitialValue);
+ if (InitialValue && Existing->InitialVirtualRegister &&
+ Existing->InitialVirtualRegister != InitialValue) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "physical formal register already maps to another virtual live-in");
+ }
+ return {};
+}
+
+[[nodiscard]] Expected<void> ensureBlockLiveIn(MachineBasicBlock &Block, PhysicalRegister Reg) {
+ return Block.addLiveIn(Reg, LaneMask::all());
+}
+
+[[nodiscard]] Expected<void> verifyPlanRegisters(const CallLoweringPlan &Plan,
+ const MachineFunction &Function,
+ const TargetDescription &Target) {
+ const MachineRegisterInfo &Registers = Function.registerInfo();
+ const auto Verify = [&Registers, &Target](const CCAssignment &Assignment) -> Expected<void> {
+ if (Assignment.LocationRegister &&
+ !Target.registers().isKnownPhysicalRegister(*Assignment.LocationRegister)) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "call-lowering plan names an unknown ABI location register");
+ }
+ const Register Reg = Assignment.OriginalValue;
+ if (Reg.isNoRegister())
+ return {};
+ if (Reg.isPhysical()) {
+ if (!Target.registers().isKnownPhysicalRegister(*Reg.physicalRegister())) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "call-lowering plan names an unknown physical value register");
+ }
+ return {};
+ }
+ auto Info = Registers.virtualRegisterInfo(Reg);
+ if (!Info)
+ return std::unexpected(Info.error());
+ if (!Info->get().Type || *Info->get().Type != Assignment.ValueType) {
+ return makeUnexpected(
+ ErrorCode::VerificationError,
+ "call-lowering plan register LowLevelType differs from MachineRegisterInfo");
+ }
+ return {};
+ };
+ for (const CCAssignment &Assignment : Plan.argumentAssignments()) {
+ auto Result = Verify(Assignment);
+ if (!Result)
+ return std::unexpected(Result.error());
+ }
+ for (const CCAssignment &Assignment : Plan.returnAssignments()) {
+ auto Result = Verify(Assignment);
+ if (!Result)
+ return std::unexpected(Result.error());
+ }
+ if (const MachineCallTarget *Callee = Plan.callee();
+ Callee != nullptr && Callee->kind() == MachineCallTargetKind::IndirectRegister) {
+ const Register CalleeReg = Callee->indirectRegister();
+ if (CalleeReg.isPhysical()) {
+ const std::optional<PhysicalRegister> Physical = CalleeReg.physicalRegister();
+ if (!Physical || !Target.registers().isKnownPhysicalRegister(*Physical)) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "indirect RISC-V callee names an unknown physical register");
+ }
+ } else {
+ auto Info = Registers.virtualRegisterInfo(CalleeReg);
+ if (!Info)
+ return std::unexpected(Info.error());
+ const std::optional<LowLevelType> Type = Info->get().Type;
+ if (!Type || ((!Type->isPointer() && !Type->isCategoryNeutralScalar()) ||
+ Type->scalarBitWidth() != 64)) {
+ return makeUnexpected(
+ ErrorCode::VerificationError,
+ "indirect RISC-V callee requires a 64-bit pointer or integer LowLevelType");
+ }
+ }
+ }
+ return {};
+}
+
+[[nodiscard]] Expected<void> verifyRequiredOpcodes(const MachineIRBuilder &Builder) {
+ static constexpr std::array Required{
+ InstructionOpcode::PseudoLOAD_STACK_ARG, InstructionOpcode::PseudoSTORE_STACK_ARG,
+ InstructionOpcode::PseudoBYVAL_COPY, InstructionOpcode::ADJCALLSTACKDOWN,
+ InstructionOpcode::ADJCALLSTACKUP, InstructionOpcode::PseudoCALL,
+ InstructionOpcode::PseudoTAIL, InstructionOpcode::PseudoRET,
+ };
+ for (const InstructionOpcode OpcodeValue : Required) {
+ if (Builder.target().instructionDescriptor(instructionOpcode(OpcodeValue)) == nullptr) {
+ return makeUnexpected(ErrorCode::InternalError,
+ "RISC-V target description lacks a call-lowering pseudo");
+ }
+ }
+ return {};
+}
+
+[[nodiscard]] Expected<void> verifyFormalLiveIns(const CallLoweringPlan &Plan,
+ const MachineFunction &Function) {
+ if (Plan.kind() != CallLoweringPlanKind::FormalArguments)
+ return {};
+
+ const auto LiveIns = Function.registerInfo().liveIns();
+ for (const CCAssignment &Assignment : Plan.argumentAssignments()) {
+ if (!Assignment.LocationRegister || !Assignment.OriginalValue.isVirtual())
+ continue;
+ const auto Existing =
+ std::ranges::find(LiveIns, *Assignment.LocationRegister, &PhysicalRegisterLiveIn::PhysReg);
+ if (Existing != LiveIns.end() && Existing->InitialVirtualRegister &&
+ *Existing->InitialVirtualRegister != Assignment.OriginalValue) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "physical formal register already maps to another virtual live-in");
+ }
+ }
+ return {};
+}
+
+[[nodiscard]] Expected<void> preflightPlan(MachineIRBuilder &Builder,
+ const CallLoweringPlan &Plan) {
+ auto Convention = riscvConvention(Plan.convention());
+ if (!Convention)
+ return std::unexpected(Convention.error());
+ if (Plan.target() != PlanTarget) {
+ return makeUnexpected(ErrorCode::InvalidArgument,
+ "call-lowering plan belongs to another target or ABI");
+ }
+ auto InsertPoint = Builder.verifyInsertPoint();
+ if (!InsertPoint)
+ return std::unexpected(InsertPoint.error());
+ auto Opcodes = verifyRequiredOpcodes(Builder);
+ if (!Opcodes)
+ return std::unexpected(Opcodes.error());
+ auto Registers = verifyPlanRegisters(Plan, Builder.function(), Builder.target());
+ if (!Registers)
+ return std::unexpected(Registers.error());
+ auto LiveIns = verifyFormalLiveIns(Plan, Builder.function());
+ if (!LiveIns)
+ return std::unexpected(LiveIns.error());
+ for (const CCAssignment &Assignment : Plan.argumentAssignments()) {
+ if (Assignment.StackOffsetBytes &&
+ *Assignment.StackOffsetBytes >
+ static_cast<std::uint64_t>(std::numeric_limits<std::int64_t>::max())) {
+ return makeUnexpected(ErrorCode::Unsupported,
+ "RISC-V stack assignment offset exceeds the frame-index range");
+ }
+ }
+ for (const ByValCopyAssignment &Copy : Plan.byValCopies()) {
+ if (Copy.StackOffsetBytes >
+ static_cast<std::uint64_t>(std::numeric_limits<std::int64_t>::max())) {
+ return makeUnexpected(ErrorCode::Unsupported,
+ "RISC-V byval offset exceeds the frame-index range");
+ }
+ }
+ if (Plan.kind() != CallLoweringPlanKind::Return && Builder.function().frameInfo().isFrozen()) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "call lowering cannot modify a frozen machine frame");
+ }
+ if (Plan.kind() == CallLoweringPlanKind::FormalArguments &&
+ (Builder.function().firstBlock() == nullptr ||
+ Builder.insertionBlock() != Builder.function().firstBlock())) {
+ return makeUnexpected(ErrorCode::InvalidArgument,
+ "formal arguments must be lowered in the entry machine block");
+ }
+ return {};
+}
+
+[[nodiscard]] Expected<void> updateMaximumFrameAlignment(MachineFrameState &State,
+ Alignment Required) {
+ if (!State.MaxAlignment || State.MaxAlignment->bytes() < Required.bytes())
+ State.MaxAlignment = Required;
+ return {};
+}
+
+[[nodiscard]] Expected<void> emitFormalArguments(MachineIRBuilder &Builder,
+ const CallLoweringPlan &Plan) {
+ MachineFunction &Function = Builder.function();
+ MachineBasicBlock &Entry = *Function.firstBlock();
+ MachineRegisterInfo &Registers = Function.registerInfo();
+ MachineFrameInfo &Frame = Function.frameInfo();
+ Alignment RequiredFrameAlignment;
+
+ for (const CCAssignment &Assignment : Plan.argumentAssignments()) {
+ if (Assignment.LocationRegister) {
+ const PhysicalRegister Location = *Assignment.LocationRegister;
+ const std::optional<Register> Initial =
+ Assignment.OriginalValue.isVirtual() ? std::optional<Register>(Assignment.OriginalValue)
+ : std::nullopt;
+ auto FunctionLiveIn = ensureFunctionLiveIn(Registers, Location, Initial);
+ if (!FunctionLiveIn)
+ return std::unexpected(FunctionLiveIn.error());
+ auto BlockLiveIn = ensureBlockLiveIn(Entry, Location);
+ if (!BlockLiveIn)
+ return std::unexpected(BlockLiveIn.error());
+ auto Copy = emitIncomingRegisterAssignment(Builder, Assignment);
+ if (!Copy)
+ return std::unexpected(Copy.error());
+ continue;
+ }
+
+ if (!Assignment.StackOffsetBytes) {
+ return makeUnexpected(ErrorCode::InternalError,
+ "verified formal stack assignment has no stack offset");
+ }
+ StackObjectDesc Object;
+ Object.Kind = StackObjectKind::FixedABI;
+ Object.Size = Assignment.StackSlotSizeBytes;
+ Object.Align = Assignment.LocationAlignment;
+ Object.Offset = static_cast<std::int64_t>(*Assignment.StackOffsetBytes);
+ Object.IsImmutable = true;
+ Object.TargetFlags = stackObjectFlag(CallStackObjectFlag::IncomingArgument);
+ auto FrameObject = Frame.createObject(std::move(Object));
+ if (!FrameObject)
+ return std::unexpected(FrameObject.error());
+ auto Memory =
+ createMemoryOperand(Function, {MemoryAccessFlag::Load}, Assignment.LocationSizeBytes,
+ Assignment.LocationAlignment, MemoryBase::frameIndex(*FrameObject));
+ if (!Memory)
+ return std::unexpected(Memory.error());
+ auto Load = emitStackLoad(Builder, Assignment, *FrameObject, 0, **Memory);
+ if (!Load)
+ return std::unexpected(Load.error());
+ if (RequiredFrameAlignment.bytes() < Assignment.LocationAlignment.bytes())
+ RequiredFrameAlignment = Assignment.LocationAlignment;
+ }
+
+ const VarArgLoweringState &VarArgs = Plan.varArgs();
+ if (!VarArgs.IsVariadic) {
+ if (RequiredFrameAlignment.bytes() == 1)
+ return {};
+ MachineFrameState State = Frame.state();
+ auto AlignmentUpdate = updateMaximumFrameAlignment(State, RequiredFrameAlignment);
+ if (!AlignmentUpdate)
+ return std::unexpected(AlignmentUpdate.error());
+ return Frame.setState(std::move(State));
+ }
+
+ auto Align8 = alignment(XLenBytes);
+ auto Align16 = alignment(ABIStackAlignmentBytes);
+ if (!Align8 || !Align16)
+ return makeUnexpected(ErrorCode::InternalError, "RISC-V ABI primitive type facts are invalid");
+
+ FrameIndex SaveObject;
+ if (VarArgs.RegisterSaveAreaSizeBytes == 0) {
+ StackObjectDesc Object;
+ Object.Kind = StackObjectKind::FixedABI;
+ Object.Size = XLenBytes;
+ Object.Align = *Align8;
+ Object.Offset = VarArgs.RegisterSaveAreaOffsetBytes;
+ Object.IsImmutable = true;
+ Object.TargetFlags = stackObjectFlag(CallStackObjectFlag::VarArgSaveArea);
+ auto Created = Frame.createObject(std::move(Object));
+ if (!Created)
+ return std::unexpected(Created.error());
+ SaveObject = *Created;
+ } else {
+ StackObjectDesc Object;
+ Object.Kind = StackObjectKind::FixedABI;
+ Object.Size = VarArgs.RegisterSaveAreaSizeBytes;
+ Object.Align = VarArgs.FirstUnallocatedIntegerRegister % 2 == 0 ? *Align16 : *Align8;
+ Object.Offset = VarArgs.RegisterSaveAreaOffsetBytes;
+ Object.TargetFlags = stackObjectFlag(CallStackObjectFlag::VarArgSaveArea);
+ auto Created = Frame.createObject(std::move(Object));
+ if (!Created)
+ return std::unexpected(Created.error());
+ SaveObject = *Created;
+
+ if (VarArgs.RegisterSavePaddingBytes != 0) {
+ StackObjectDesc Padding;
+ Padding.Kind = StackObjectKind::FixedABI;
+ Padding.Size = VarArgs.RegisterSavePaddingBytes;
+ Padding.Align = *Align8;
+ Padding.Offset = VarArgs.RegisterSaveAreaOffsetBytes -
+ static_cast<std::int64_t>(VarArgs.RegisterSavePaddingBytes);
+ Padding.IsImmutable = true;
+ Padding.TargetFlags = stackObjectFlag(CallStackObjectFlag::VarArgAlignmentPadding);
+ auto CreatedPadding = Frame.createObject(std::move(Padding));
+ if (!CreatedPadding)
+ return std::unexpected(CreatedPadding.error());
+ }
+ }
+
+ for (const VarArgRegisterSave Save : VarArgs.RegistersToSave) {
+ auto FunctionLiveIn = ensureFunctionLiveIn(Registers, Save.Reg, std::nullopt);
+ if (!FunctionLiveIn)
+ return std::unexpected(FunctionLiveIn.error());
+ auto BlockLiveIn = ensureBlockLiveIn(Entry, Save.Reg);
+ if (!BlockLiveIn)
+ return std::unexpected(BlockLiveIn.error());
+ auto Memory = createMemoryOperand(Function, {MemoryAccessFlag::Store}, XLenBytes, *Align8,
+ MemoryBase::frameIndex(SaveObject),
+ static_cast<std::int64_t>(Save.OffsetBytes));
+ if (!Memory)
+ return std::unexpected(Memory.error());
+ auto Store = emitStackStore(Builder, Register::fromPhysical(Save.Reg), SaveObject,
+ Save.OffsetBytes, **Memory, CallValueTransform::Full);
+ if (!Store)
+ return std::unexpected(Store.error());
+ }
+
+ MachineFrameState State = Frame.state();
+ const Alignment VarArgAlignment =
+ RequiredFrameAlignment.bytes() > Align16->bytes() ? RequiredFrameAlignment : *Align16;
+ auto AlignmentUpdate = updateMaximumFrameAlignment(State, VarArgAlignment);
+ if (!AlignmentUpdate)
+ return std::unexpected(AlignmentUpdate.error());
+ return Frame.setState(std::move(State));
+}
+
+[[nodiscard]] Expected<void> emitCallFrameAdjustment(MachineIRBuilder &Builder,
+ InstructionOpcode OpcodeValue,
+ const CallFrameRequirements &Frame) {
+ if (Frame.SizeBytes > static_cast<std::uint64_t>(std::numeric_limits<std::int64_t>::max())) {
+ return makeUnexpected(ErrorCode::Unsupported,
+ "RISC-V call-frame size does not fit a MIR immediate");
+ }
+ auto Instruction = Builder.buildInstr(instructionOpcode(OpcodeValue));
+ if (!Instruction)
+ return std::unexpected(Instruction.error());
+ auto Size = Instruction->addImmediate(static_cast<std::int64_t>(Frame.SizeBytes));
+ if (!Size)
+ return std::unexpected(Size.error());
+ auto PreparedBytes = Instruction->addImmediate(0);
+ if (!PreparedBytes)
+ return std::unexpected(PreparedBytes.error());
+ auto Built = Instruction->build();
+ if (!Built)
+ return std::unexpected(Built.error());
+ return {};
+}
+
+[[nodiscard]] Expected<std::unordered_map<std::size_t, Register>>
+emitByValCopies(MachineIRBuilder &Builder, const CallLoweringPlan &Plan) {
+ MachineFunction &Function = Builder.function();
+ MachineFrameInfo &Frame = Function.frameInfo();
+ std::unordered_map<std::size_t, Register> Replacements;
+ for (const ByValCopyAssignment Copy : Plan.byValCopies()) {
+ StackObjectDesc Object;
+ Object.Kind = StackObjectKind::Target;
+ Object.Size = Copy.SizeBytes;
+ Object.Align = Copy.Align;
+ Object.LocalOffset = static_cast<std::int64_t>(Copy.StackOffsetBytes);
+ Object.TargetFlags = stackObjectFlag(CallStackObjectFlag::ByValCopy);
+ auto FrameObject = Frame.createObject(std::move(Object));
+ if (!FrameObject)
+ return std::unexpected(FrameObject.error());
+ auto Temporary = Function.registerInfo().createVirtualRegister(Copy.PointerType);
+ if (!Temporary)
+ return std::unexpected(Temporary.error());
+
+ auto SourceMemory = createMemoryOperand(Function, {MemoryAccessFlag::Load}, Copy.SizeBytes,
+ Copy.Align, MemoryBase{});
+ if (!SourceMemory)
+ return std::unexpected(SourceMemory.error());
+ auto DestinationMemory =
+ createMemoryOperand(Function, {MemoryAccessFlag::Store}, Copy.SizeBytes, Copy.Align,
+ MemoryBase::frameIndex(*FrameObject));
+ if (!DestinationMemory)
+ return std::unexpected(DestinationMemory.error());
+
+ auto Instruction = Builder.buildInstr(instructionOpcode(InstructionOpcode::PseudoBYVAL_COPY));
+ if (!Instruction)
+ return std::unexpected(Instruction.error());
+ auto Def = Instruction->addDef(*Temporary);
+ if (!Def)
+ return std::unexpected(Def.error());
+ auto Use = Instruction->addUse(Copy.Source);
+ if (!Use)
+ return std::unexpected(Use.error());
+ auto FrameOperand = Instruction->addFrameIndex(*FrameObject);
+ if (!FrameOperand)
+ return std::unexpected(FrameOperand.error());
+ auto Size = addUnsignedImmediate(*Instruction, Copy.SizeBytes);
+ if (!Size)
+ return std::unexpected(Size.error());
+ auto Align = addUnsignedImmediate(*Instruction, Copy.Align.bytes());
+ if (!Align)
+ return std::unexpected(Align.error());
+ auto SourceMem = Instruction->addMemOperand(**SourceMemory);
+ if (!SourceMem)
+ return std::unexpected(SourceMem.error());
+ auto DestinationMem = Instruction->addMemOperand(**DestinationMemory);
+ if (!DestinationMem)
+ return std::unexpected(DestinationMem.error());
+ auto Built = Instruction->build();
+ if (!Built)
+ return std::unexpected(Built.error());
+ Replacements.emplace(Copy.ArgumentAssignmentIndex, *Temporary);
+ }
+ return Replacements;
+}
+
+[[nodiscard]] Expected<void>
+emitOutgoingArguments(MachineIRBuilder &Builder, const CallLoweringPlan &Plan,
+ const std::unordered_map<std::size_t, Register> &Replacements) {
+ MachineFunction &Function = Builder.function();
+ MachineFrameInfo &Frame = Function.frameInfo();
+ const auto Assignments = Plan.argumentAssignments();
+ std::vector<OutgoingRegisterValue> RegisterAssignments;
+ RegisterAssignments.reserve(Assignments.size());
+ for (std::size_t Index = 0; Index < Assignments.size(); ++Index) {
+ const CCAssignment &Assignment = Assignments[Index];
+ const auto Replacement = Replacements.find(Index);
+ const Register Source =
+ Replacement == Replacements.end() ? Assignment.OriginalValue : Replacement->second;
+ if (Assignment.LocationRegister) {
+ auto Prepared = prepareOutgoingRegisterAssignment(Builder, Assignment, Source);
+ if (!Prepared)
+ return std::unexpected(Prepared.error());
+ RegisterAssignments.push_back(*Prepared);
+ continue;
+ }
+
+ if (!Assignment.StackOffsetBytes) {
+ return makeUnexpected(ErrorCode::InternalError,
+ "verified outgoing stack assignment has no stack offset");
+ }
+ StackObjectDesc Object;
+ Object.Size = Assignment.StackSlotSizeBytes;
+ Object.Align = Assignment.LocationAlignment;
+ Object.TargetFlags = stackObjectFlag(CallStackObjectFlag::OutgoingArgument);
+ if (Plan.tailDecision() == TailCallKind::NotTail) {
+ Object.Kind = StackObjectKind::Target;
+ Object.LocalOffset = static_cast<std::int64_t>(*Assignment.StackOffsetBytes);
+ } else {
+ Object.Kind = StackObjectKind::FixedABI;
+ Object.Offset = static_cast<std::int64_t>(*Assignment.StackOffsetBytes);
+ }
+ auto FrameObject = Frame.createObject(std::move(Object));
+ if (!FrameObject)
+ return std::unexpected(FrameObject.error());
+ auto Memory =
+ createMemoryOperand(Function, {MemoryAccessFlag::Store}, Assignment.LocationSizeBytes,
+ Assignment.LocationAlignment, MemoryBase::frameIndex(*FrameObject));
+ if (!Memory)
+ return std::unexpected(Memory.error());
+ auto Store =
+ emitStackStore(Builder, Source, *FrameObject, 0, **Memory, outgoingTransform(Assignment));
+ if (!Store)
+ return std::unexpected(Store.error());
+ }
+ return emitOutgoingRegisterAssignments(
+ Builder, std::span<const OutgoingRegisterValue>(RegisterAssignments));
+}
+
+[[nodiscard]] Expected<void> addCallTarget(MachineInstrBuilder &Instruction,
+ const MachineCallTarget &Callee) {
+ switch (Callee.kind()) {
+ case MachineCallTargetKind::GlobalSymbol:
+ return Instruction.addGlobalSymbol(std::string(Callee.symbolName()), 0, Callee.targetFlags());
+ case MachineCallTargetKind::ExternalSymbol:
+ return Instruction.addExternalSymbol(std::string(Callee.symbolName()), 0, Callee.targetFlags());
+ case MachineCallTargetKind::IndirectRegister:
+ return Instruction.addUse(Callee.indirectRegister());
+ case MachineCallTargetKind::Invalid:
+ break;
+ }
+ return makeUnexpected(ErrorCode::InternalError,
+ "verified call-lowering plan has an invalid callee");
+}
+
+[[nodiscard]] Expected<PhysicalRegister> requiredLocationRegister(const CCAssignment &Assignment) {
+ if (!Assignment.LocationRegister) {
+ return makeUnexpected(ErrorCode::InternalError,
+ "verified return assignment has no location register");
+ }
+ return *Assignment.LocationRegister;
+}
+
+[[nodiscard]] Expected<void> emitCallInstruction(MachineIRBuilder &Builder,
+ const CallLoweringPlan &Plan) {
+ const bool IsTail = Plan.tailDecision() != TailCallKind::NotTail;
+ auto Instruction = Builder.buildInstr(
+ instructionOpcode(IsTail ? InstructionOpcode::PseudoTAIL : InstructionOpcode::PseudoCALL));
+ if (!Instruction)
+ return std::unexpected(Instruction.error());
+ auto Callee = addCallTarget(*Instruction, *Plan.callee());
+ if (!Callee)
+ return std::unexpected(Callee.error());
+ auto Mask = Instruction->addRegisterMask(Plan.preservedMask());
+ if (!Mask)
+ return std::unexpected(Mask.error());
+
+ std::unordered_set<PhysicalRegister> DynamicUses;
+ for (const CCAssignment &Assignment : Plan.argumentAssignments()) {
+ if (Assignment.LocationRegister && DynamicUses.insert(*Assignment.LocationRegister).second) {
+ auto Use = Instruction->addImplicitUse(Register::fromPhysical(*Assignment.LocationRegister));
+ if (!Use)
+ return std::unexpected(Use.error());
+ }
+ }
+ std::unordered_set<PhysicalRegister> DynamicDefs;
+ if (!IsTail) {
+ for (const CCAssignment &Assignment : Plan.returnAssignments()) {
+ if (Assignment.LocationRegister && DynamicDefs.insert(*Assignment.LocationRegister).second) {
+ auto Def =
+ Instruction->addImplicitDef(Register::fromPhysical(*Assignment.LocationRegister));
+ if (!Def)
+ return std::unexpected(Def.error());
+ }
+ }
+ }
+ auto Built = Instruction->build();
+ if (!Built)
+ return std::unexpected(Built.error());
+ return {};
+}
+
+[[nodiscard]] Expected<void> emitCall(MachineIRBuilder &Builder, const CallLoweringPlan &Plan) {
+ if (Plan.callFrame().EmitSetup) {
+ auto Down =
+ emitCallFrameAdjustment(Builder, InstructionOpcode::ADJCALLSTACKDOWN, Plan.callFrame());
+ if (!Down)
+ return std::unexpected(Down.error());
+ }
+
+ auto Replacements = emitByValCopies(Builder, Plan);
+ if (!Replacements)
+ return std::unexpected(Replacements.error());
+ auto Arguments = emitOutgoingArguments(Builder, Plan, *Replacements);
+ if (!Arguments)
+ return std::unexpected(Arguments.error());
+ auto Call = emitCallInstruction(Builder, Plan);
+ if (!Call)
+ return std::unexpected(Call.error());
+
+ if (Plan.callFrame().EmitDestroy) {
+ auto Up = emitCallFrameAdjustment(Builder, InstructionOpcode::ADJCALLSTACKUP, Plan.callFrame());
+ if (!Up)
+ return std::unexpected(Up.error());
+ }
+ if (Plan.tailDecision() == TailCallKind::NotTail) {
+ for (const CCAssignment &Assignment : Plan.returnAssignments()) {
+ auto Copy = emitIncomingRegisterAssignment(Builder, Assignment);
+ if (!Copy)
+ return std::unexpected(Copy.error());
+ }
+ }
+
+ MachineFrameInfo &Frame = Builder.function().frameInfo();
+ MachineFrameState State = Frame.state();
+ State.HasCalls = true;
+ State.HasTailCall = State.HasTailCall || Plan.tailDecision() != TailCallKind::NotTail;
+ State.AdjustsStack =
+ State.AdjustsStack || (Plan.callFrame().EmitSetup && Plan.callFrame().SizeBytes != 0);
+ State.HasMustTailInVarArgFunc =
+ State.HasMustTailInVarArgFunc ||
+ (Plan.tailDecision() == TailCallKind::MustTail && Plan.varArgs().IsVariadic);
+ if (Plan.tailDecision() == TailCallKind::NotTail) {
+ State.MaxCallFrameSize =
+ std::max(State.MaxCallFrameSize.value_or(0), Plan.callFrame().SizeBytes);
+ }
+ auto AlignmentUpdate = updateMaximumFrameAlignment(State, Plan.callFrame().Align);
+ if (!AlignmentUpdate)
+ return std::unexpected(AlignmentUpdate.error());
+ return Frame.setState(std::move(State));
+}
+
+[[nodiscard]] Expected<void> emitReturn(MachineIRBuilder &Builder, const CallLoweringPlan &Plan) {
+ std::vector<OutgoingRegisterValue> RegisterAssignments;
+ RegisterAssignments.reserve(Plan.returnAssignments().size());
+ for (const CCAssignment &Assignment : Plan.returnAssignments()) {
+ auto Prepared =
+ prepareOutgoingRegisterAssignment(Builder, Assignment, Assignment.OriginalValue);
+ if (!Prepared)
+ return std::unexpected(Prepared.error());
+ RegisterAssignments.push_back(*Prepared);
+ }
+ auto Copies = emitOutgoingRegisterAssignments(
+ Builder, std::span<const OutgoingRegisterValue>(RegisterAssignments));
+ if (!Copies)
+ return std::unexpected(Copies.error());
+
+ auto Return = Builder.buildInstr(instructionOpcode(InstructionOpcode::PseudoRET));
+ if (!Return)
+ return std::unexpected(Return.error());
+ for (const CCAssignment &Assignment : Plan.returnAssignments()) {
+ auto Location = requiredLocationRegister(Assignment);
+ if (!Location)
+ return std::unexpected(Location.error());
+ auto Use = Return->addImplicitUse(Register::fromPhysical(*Location));
+ if (!Use)
+ return std::unexpected(Use.error());
+ }
+ auto Built = Return->build();
+ if (!Built)
+ return std::unexpected(Built.error());
+ return {};
+}
+
+Expected<void> RISCVCallLowering::emitPlan(MachineIRBuilder &Builder,
+ const CallLoweringPlan &Plan) const {
+ auto Preflight = preflightPlan(Builder, Plan);
+ if (!Preflight)
+ return std::unexpected(Preflight.error());
+ switch (Plan.kind()) {
+ case CallLoweringPlanKind::FormalArguments:
+ return emitFormalArguments(Builder, Plan);
+ case CallLoweringPlanKind::Call:
+ return emitCall(Builder, Plan);
+ case CallLoweringPlanKind::Return:
+ return emitReturn(Builder, Plan);
+ }
+ return makeUnexpected(ErrorCode::InternalError, "call-lowering plan kind is invalid");
+}
+
+} // namespace rcc::riscv
diff --git a/backend/lib/Target/RISCV/GISel/RISCVCallLowering.h b/backend/lib/Target/RISCV/GISel/RISCVCallLowering.h
new file mode 100644
index 0000000..bc03fe8
--- /dev/null
+++ b/backend/lib/Target/RISCV/GISel/RISCVCallLowering.h
@@ -0,0 +1,49 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef RCC_LIB_TARGET_RISCV_GISEL_RISCVCALLLOWERING_H
+#define RCC_LIB_TARGET_RISCV_GISEL_RISCVCALLLOWERING_H
+
+#include "../RISCVCallingConv.h"
+#include "rcc/CodeGen/TargetCallLowering.h"
+#include "rcc/Target/RISCV/RISCVInstructions.h"
+
+#include <cstdint>
+
+namespace rcc::riscv {
+
+enum class CallTargetFlag : std::uint32_t {
+ DirectCall = 1U << 0U,
+};
+
+enum class CallStackObjectFlag : std::uint64_t {
+ IncomingArgument = 1ULL << 0U,
+ OutgoingArgument = 1ULL << 1U,
+ ByValCopy = 1ULL << 2U,
+ VarArgSaveArea = 1ULL << 3U,
+ VarArgAlignmentPadding = 1ULL << 4U,
+};
+
+enum class CallValueTransform : std::uint8_t {
+ Full,
+ AnyExtend,
+ SignExtend,
+ ZeroExtend,
+ Truncate,
+ Bitcast,
+};
+
+class RISCVCallLowering final : public TargetCallLowering {
+public:
+ [[nodiscard]] Expected<CallLoweringPlan>
+ planFormalArguments(const FormalArgumentRequest &Request) const override;
+ [[nodiscard]] Expected<CallLoweringPlan>
+ planCall(const CallLoweringRequest &Request) const override;
+ [[nodiscard]] Expected<CallLoweringPlan>
+ planReturn(const ReturnLoweringRequest &Request) const override;
+ [[nodiscard]] Expected<void> emitPlan(MachineIRBuilder &Builder,
+ const CallLoweringPlan &Plan) const override;
+};
+
+} // namespace rcc::riscv
+
+#endif // RCC_LIB_TARGET_RISCV_GISEL_RISCVCALLLOWERING_H
diff --git a/backend/lib/Target/RISCV/GISel/RISCVGISelCombiner.cpp b/backend/lib/Target/RISCV/GISel/RISCVGISelCombiner.cpp
new file mode 100644
index 0000000..bf67c0a
--- /dev/null
+++ b/backend/lib/Target/RISCV/GISel/RISCVGISelCombiner.cpp
@@ -0,0 +1,56 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "RISCV.h"
+#include "rcc/CodeGen/GlobalISel/GIMatchTableExecutor.h"
+#include "rcc/CodeGen/MachineFunction.h"
+#include "rcc/CodeGen/MachineInstr.h"
+#include "rcc/CodeGen/MachineRegisterInfo.h"
+#include "rcc/CodeGen/TargetDescription.h"
+
+#include <string>
+
+namespace rcc::riscv {
+namespace {
+
+#define GET_GLOBALISEL_COMBINER_TABLE
+#include "RISCVGenGlobalISelCombiner.inc"
+#undef GET_GLOBALISEL_COMBINER_TABLE
+
+class RISCVGISelCombinerExecutor final : public GIMatchTableExecutor {
+protected:
+ [[nodiscard]] Expected<bool>
+ matchesTargetPredicate(std::string_view Name, const MachineInstr &Instruction,
+ const MachineFunction &Function,
+ const TargetDescription &Target) const override {
+ if (Name != "sext_cheaper_than_zext")
+ return GIMatchTableExecutor::matchesTargetPredicate(Name, Instruction, Function, Target);
+ const MachineOperand *Destination = Instruction.operandAt(0);
+ const MachineOperand *Source = Instruction.operandAt(1);
+ const Register *DestinationReg =
+ Destination == nullptr ? nullptr : Destination->getIf<Register>();
+ const Register *SourceReg = Source == nullptr ? nullptr : Source->getIf<Register>();
+ if (DestinationReg == nullptr || SourceReg == nullptr || !DestinationReg->isVirtual() ||
+ !SourceReg->isVirtual())
+ return false;
+ auto DestinationType = Function.registerInfo().getType(*DestinationReg);
+ auto SourceType = Function.registerInfo().getType(*SourceReg);
+ if (!DestinationType || !SourceType)
+ return false;
+ return DestinationType->isCategoryNeutralScalar() && SourceType->isCategoryNeutralScalar() &&
+ DestinationType->scalarBitWidth() == 64 && SourceType->scalarBitWidth() == 32;
+ }
+};
+
+} // namespace
+
+const GIMatchTable &giselCombinerTable() noexcept {
+ static constexpr GIMatchTable Table = generatedGlobalISelCombinerMatchTable();
+ return Table;
+}
+
+const GIMatchTableExecutor &giselCombinerExecutor() noexcept {
+ static const RISCVGISelCombinerExecutor Executor;
+ return Executor;
+}
+
+} // namespace rcc::riscv
diff --git a/backend/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp b/backend/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp
new file mode 100644
index 0000000..4504fa7
--- /dev/null
+++ b/backend/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp
@@ -0,0 +1,1343 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "RISCV.h"
+#include "RISCVImmediateMaterialization.h"
+#include "RISCVInstrInfo.h"
+#include "RISCVRegisterBankInfo.h"
+#include "rcc/CodeGen/GenericOpcodes.h"
+#include "rcc/CodeGen/GlobalISel/GISelValueTracking.h"
+#include "rcc/CodeGen/GlobalISel/InstructionSelector.h"
+#include "rcc/CodeGen/GlobalObject.h"
+#include "rcc/CodeGen/LowLevelType.h"
+#include "rcc/CodeGen/MachineBasicBlock.h"
+#include "rcc/CodeGen/MachineFunction.h"
+#include "rcc/CodeGen/MachineIRBuilder.h"
+#include "rcc/CodeGen/MachineInstr.h"
+#include "rcc/CodeGen/MachineMemOperand.h"
+#include "rcc/CodeGen/MachineOperand.h"
+#include "rcc/CodeGen/MachineRegisterInfo.h"
+#include "rcc/CodeGen/TargetDescription.h"
+#include "rcc/Target/RISCV/RISCVSubtarget.h"
+#include "rcc/Target/RISCV/RISCVTarget.h"
+
+#include <algorithm>
+#include <array>
+#include <bit>
+#include <cstdint>
+#include <initializer_list>
+#include <limits>
+#include <optional>
+#include <ranges>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace rcc::riscv {
+namespace {
+
+constexpr std::uint32_t AbsoluteHighFlag = 32;
+constexpr std::uint32_t AbsoluteLowFlag = 64;
+
+#define GET_GLOBALISEL_MATCH_TABLE
+#include "RISCVGenGlobalISel.inc"
+#undef GET_GLOBALISEL_MATCH_TABLE
+
+[[nodiscard]] std::optional<LowLevelType> registerType(const MachineFunction &Function,
+ const MachineOperand &Operand) {
+ const Register *Reg = Operand.getIf<Register>();
+ if (Reg == nullptr || !Reg->isVirtual())
+ return std::nullopt;
+ auto Info = Function.registerInfo().virtualRegisterInfo(*Reg);
+ if (!Info)
+ return std::nullopt;
+ return Info->get().Type;
+}
+
+[[nodiscard]] std::optional<LowLevelType> resultType(const MachineFunction &Function,
+ const MachineInstr &Instruction) {
+ const MachineOperand *Result = Instruction.operandAt(0);
+ return Result == nullptr ? std::nullopt : registerType(Function, *Result);
+}
+
+[[nodiscard]] std::optional<LowLevelType>
+operandType(const MachineFunction &Function, const MachineInstr &Instruction, std::size_t Index) {
+ const MachineOperand *Operand = Instruction.operandAt(Index);
+ return Operand == nullptr ? std::nullopt : registerType(Function, *Operand);
+}
+
+[[nodiscard]] bool isSupportedFloatingScalar(LowLevelType Type) noexcept {
+ return Type.isCategoryNeutralScalar() &&
+ (Type.scalarBitWidth() == 32 || Type.scalarBitWidth() == 64);
+}
+
+[[nodiscard]] Expected<InstructionOpcode> floatingCompareOpcode(FloatingPredicate Predicate,
+ std::uint32_t BitWidth) {
+ if (BitWidth == 32) {
+ switch (Predicate) {
+ case FloatingPredicate::OrderedEqual:
+ return InstructionOpcode::FEQ_S;
+ case FloatingPredicate::OrderedLessThan:
+ return InstructionOpcode::FLT_S;
+ case FloatingPredicate::OrderedLessEqual:
+ return InstructionOpcode::FLE_S;
+ default:
+ break;
+ }
+ } else if (BitWidth == 64) {
+ switch (Predicate) {
+ case FloatingPredicate::OrderedEqual:
+ return InstructionOpcode::FEQ_D;
+ case FloatingPredicate::OrderedLessThan:
+ return InstructionOpcode::FLT_D;
+ case FloatingPredicate::OrderedLessEqual:
+ return InstructionOpcode::FLE_D;
+ default:
+ break;
+ }
+ }
+ return makeUnexpected(ErrorCode::Unsupported,
+ "RISC-V floating comparison predicate is not directly supported");
+}
+
+// Convert a floating predicate by swapping or inverting it to one supported by
+// a scalar RISC-V comparison instruction.
+[[nodiscard]] bool legalizeFloatingPredicate(Register &Left, Register &Right,
+ FloatingPredicate &Predicate,
+ bool &NeedsInvert) noexcept {
+ switch (Predicate) {
+ case FloatingPredicate::OrderedGreaterThan:
+ Predicate = FloatingPredicate::OrderedLessThan;
+ std::swap(Left, Right);
+ return true;
+ case FloatingPredicate::OrderedGreaterEqual:
+ Predicate = FloatingPredicate::OrderedLessEqual;
+ std::swap(Left, Right);
+ return true;
+ case FloatingPredicate::UnorderedNotEqual:
+ Predicate = FloatingPredicate::OrderedEqual;
+ NeedsInvert = true;
+ return true;
+ case FloatingPredicate::UnorderedGreaterThan:
+ Predicate = FloatingPredicate::OrderedLessEqual;
+ NeedsInvert = true;
+ return true;
+ case FloatingPredicate::UnorderedGreaterEqual:
+ Predicate = FloatingPredicate::OrderedLessThan;
+ NeedsInvert = true;
+ return true;
+ case FloatingPredicate::UnorderedLessThan:
+ Predicate = FloatingPredicate::OrderedLessEqual;
+ NeedsInvert = true;
+ std::swap(Left, Right);
+ return true;
+ case FloatingPredicate::UnorderedLessEqual:
+ Predicate = FloatingPredicate::OrderedLessThan;
+ NeedsInvert = true;
+ std::swap(Left, Right);
+ return true;
+ default:
+ return false;
+ }
+}
+
+[[nodiscard]] bool isSupportedIntegerType(LowLevelType Type) noexcept {
+ return (Type.isCategoryNeutralScalar() && Type.scalarBitWidth() != 0 &&
+ Type.scalarBitWidth() <= 64) ||
+ (Type.isPointer() && Type.scalarBitWidth() == 64);
+}
+
+[[nodiscard]] Expected<void> insertPhysicalRegister(MachineInstr &Instruction, std::size_t Index,
+ PhysicalRegister Reg,
+ RegisterOperandFlag Flag) {
+ auto Operand = MachineOperand::registerOperand(Register::fromPhysical(Reg), {Flag});
+ if (!Operand)
+ return std::unexpected(Operand.error());
+ return Instruction.insertOperand(Index, std::move(*Operand));
+}
+
+[[nodiscard]] Expected<std::int64_t> signedImmediate(const MachineOperand &Operand) {
+ if (const std::int64_t *Value = Operand.getIf<std::int64_t>())
+ return *Value;
+ const auto *Integer = Operand.getIf<IntegerImmediateOperand>();
+ if (Integer == nullptr || Integer->Value.bitWidth() == 0 || Integer->Value.bitWidth() > 64)
+ return makeUnexpected(ErrorCode::Unsupported,
+ "RISC-V constant selection requires an immediate no wider than 64 bits");
+ std::uint64_t Raw = Integer->Value.words().empty() ? 0 : Integer->Value.words().front();
+ const std::uint32_t Width = Integer->Value.bitWidth();
+ if (Width < 64 && (Raw & (std::uint64_t{1} << (Width - 1U))) != 0)
+ Raw |= ~std::uint64_t{0} << Width;
+ return std::bit_cast<std::int64_t>(Raw);
+}
+
+[[nodiscard]] std::optional<std::int64_t>
+directIntegerConstant(const MachineRegisterInfo &RegisterInfo, Register ValueRegister) {
+ if (!ValueRegister.isVirtual())
+ return std::nullopt;
+ const MachineInstr *Definition = RegisterInfo.getVRegDef(ValueRegister);
+ if (Definition == nullptr || Definition->opcode() != opcode(GenericOpcode::G_CONSTANT))
+ return std::nullopt;
+ const MachineOperand *ValueOperand = Definition->operandAt(1);
+ if (ValueOperand == nullptr)
+ return std::nullopt;
+ auto Value = signedImmediate(*ValueOperand);
+ return Value ? std::optional<std::int64_t>(*Value) : std::nullopt;
+}
+
+struct SelectedAddress final {
+ MachineOperand Base;
+ std::int64_t Offset = 0;
+};
+
+[[nodiscard]] std::optional<SelectedAddress> selectAddress(const MachineInstr &Instruction,
+ const MachineFunction &Function,
+ std::uint32_t OperandIndex) {
+ const MachineOperand *Root = Instruction.operandAt(OperandIndex);
+ const Register *RootRegister = Root == nullptr ? nullptr : Root->getIf<Register>();
+ if (RootRegister == nullptr || !RootRegister->isVirtual())
+ return std::nullopt;
+
+ const MachineRegisterInfo &Registers = Function.registerInfo();
+ const MachineInstr *RootDefinition = Registers.getVRegDef(*RootRegister);
+ if (RootDefinition != nullptr &&
+ RootDefinition->opcode() == opcode(GenericOpcode::G_FRAME_INDEX)) {
+ const MachineOperand *FrameIndex = RootDefinition->operandAt(1);
+ if (FrameIndex == nullptr)
+ return std::nullopt;
+ return SelectedAddress{.Base = FrameIndex->cloneDetached(), .Offset = 0};
+ }
+
+ if (RootDefinition != nullptr && RootDefinition->opcode() == opcode(GenericOpcode::G_PTR_ADD)) {
+ const MachineOperand *Base = RootDefinition->operandAt(1);
+ const MachineOperand *Offset = RootDefinition->operandAt(2);
+ const Register *BaseRegister = Base == nullptr ? nullptr : Base->getIf<Register>();
+ const Register *OffsetRegister = Offset == nullptr ? nullptr : Offset->getIf<Register>();
+ if (BaseRegister != nullptr && OffsetRegister != nullptr) {
+ const std::optional<std::int64_t> Constant =
+ directIntegerConstant(Registers, *OffsetRegister);
+ if (Constant && *Constant >= -2048 && *Constant <= 2047) {
+ const MachineInstr *BaseDefinition =
+ BaseRegister->isVirtual() ? Registers.getVRegDef(*BaseRegister) : nullptr;
+ if (BaseDefinition != nullptr &&
+ BaseDefinition->opcode() == opcode(GenericOpcode::G_FRAME_INDEX)) {
+ const MachineOperand *FrameIndex = BaseDefinition->operandAt(1);
+ if (FrameIndex == nullptr)
+ return std::nullopt;
+ return SelectedAddress{.Base = FrameIndex->cloneDetached(), .Offset = *Constant};
+ }
+ return SelectedAddress{.Base = Base->cloneDetached(), .Offset = *Constant};
+ }
+ }
+ }
+
+ return SelectedAddress{.Base = Root->cloneDetached(), .Offset = 0};
+}
+
+struct SelectedIntegerCondition final {
+ Register Left;
+ Register Right;
+ BranchConditionCode Code = BranchConditionCode::NotEqual;
+};
+
+[[nodiscard]] Expected<std::optional<SelectedIntegerCondition>>
+selectIntegerCondition(const MachineInstr &Instruction, const MachineFunction &Function,
+ std::uint32_t OperandIndex) {
+ const MachineOperand *ConditionOperand = Instruction.operandAt(OperandIndex);
+ const Register *Condition =
+ ConditionOperand == nullptr ? nullptr : ConditionOperand->getIf<Register>();
+ if (Condition == nullptr)
+ return std::optional<SelectedIntegerCondition>{};
+
+ auto Zero = physicalRegister(PhysicalRegisterID::X0);
+ if (!Zero)
+ return std::unexpected(Zero.error());
+ SelectedIntegerCondition Result{
+ .Left = *Condition,
+ .Right = Register::fromPhysical(*Zero),
+ .Code = BranchConditionCode::NotEqual,
+ };
+ const MachineInstr *Comparison =
+ Condition->isVirtual() ? Function.registerInfo().getVRegDef(*Condition) : nullptr;
+ if (Comparison == nullptr || Comparison->opcode() != opcode(GenericOpcode::G_ICMP))
+ return std::optional<SelectedIntegerCondition>{Result};
+
+ const MachineOperand *PredicateMO = Comparison->operandAt(1);
+ const auto *Predicate = PredicateMO == nullptr ? nullptr : PredicateMO->getIf<PredicateOperand>();
+ const IntegerPredicate *Integer = Predicate == nullptr ? nullptr : Predicate->integerPredicate();
+ const MachineOperand *LeftOperand = Comparison->operandAt(2);
+ const MachineOperand *RightOperand = Comparison->operandAt(3);
+ const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf<Register>();
+ const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf<Register>();
+ if (Integer == nullptr || Left == nullptr || Right == nullptr) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V integer comparison has malformed branch operands");
+ }
+ Result.Left = *Left;
+ Result.Right = *Right;
+
+ if (const std::optional<std::int64_t> Constant =
+ directIntegerConstant(Function.registerInfo(), *Right)) {
+ if (*Integer == IntegerPredicate::SignedGreaterThan && *Constant == -1) {
+ Result.Right = Register::fromPhysical(*Zero);
+ Result.Code = BranchConditionCode::SignedGreaterEqual;
+ return std::optional<SelectedIntegerCondition>{Result};
+ }
+ if (*Integer == IntegerPredicate::SignedLessThan && *Constant == 1) {
+ Result.Right = Result.Left;
+ Result.Left = Register::fromPhysical(*Zero);
+ Result.Code = BranchConditionCode::SignedGreaterEqual;
+ return std::optional<SelectedIntegerCondition>{Result};
+ }
+ }
+
+ switch (*Integer) {
+ case IntegerPredicate::Equal:
+ Result.Code = BranchConditionCode::Equal;
+ break;
+ case IntegerPredicate::NotEqual:
+ Result.Code = BranchConditionCode::NotEqual;
+ break;
+ case IntegerPredicate::SignedLessThan:
+ Result.Code = BranchConditionCode::SignedLessThan;
+ break;
+ case IntegerPredicate::SignedGreaterEqual:
+ Result.Code = BranchConditionCode::SignedGreaterEqual;
+ break;
+ case IntegerPredicate::UnsignedLessThan:
+ Result.Code = BranchConditionCode::UnsignedLessThan;
+ break;
+ case IntegerPredicate::UnsignedGreaterEqual:
+ Result.Code = BranchConditionCode::UnsignedGreaterEqual;
+ break;
+ case IntegerPredicate::SignedGreaterThan:
+ std::swap(Result.Left, Result.Right);
+ Result.Code = BranchConditionCode::SignedLessThan;
+ break;
+ case IntegerPredicate::SignedLessEqual:
+ std::swap(Result.Left, Result.Right);
+ Result.Code = BranchConditionCode::SignedGreaterEqual;
+ break;
+ case IntegerPredicate::UnsignedGreaterThan:
+ std::swap(Result.Left, Result.Right);
+ Result.Code = BranchConditionCode::UnsignedLessThan;
+ break;
+ case IntegerPredicate::UnsignedLessEqual:
+ std::swap(Result.Left, Result.Right);
+ Result.Code = BranchConditionCode::UnsignedGreaterEqual;
+ break;
+ case IntegerPredicate::Count:
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V select has an invalid integer predicate");
+ }
+ return std::optional<SelectedIntegerCondition>{Result};
+}
+
+class RISCVInstructionSelector final : public InstructionSelector {
+public:
+ RISCVInstructionSelector(const RISCVSubtarget &Subtarget,
+ const RISCVRegisterBankInfo &RegisterBanks)
+ : Subtarget(Subtarget), RegisterBanks(RegisterBanks) {}
+
+ [[nodiscard]] Expected<bool> matchesTargetPredicate(std::string_view Name, const MachineInstr &,
+ const MachineFunction &,
+ const TargetDescription &) const override {
+ if (Name == "a")
+ return Subtarget.features().has(RISCVExtension::A);
+ if (Name == "zba")
+ return Subtarget.features().has(RISCVExtension::Zba);
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V generated GlobalISel rule uses unknown target predicate '" +
+ std::string(Name) + "'");
+ }
+
+ [[nodiscard]] Expected<void>
+ constrainSelectedInstruction(MachineInstr &Instruction, MachineFunction &Function,
+ const TargetDescription &Target) const override {
+ return RegisterBanks.constrainSelectedInstruction(Instruction, Function, Target);
+ }
+
+ [[nodiscard]] Expected<std::optional<GIMatchComplexOperand>>
+ renderComplexOperand(std::string_view Name, const MachineInstr &Instruction,
+ const MachineFunction &Function, const TargetDescription &Target,
+ std::uint32_t OperandIndex) const override {
+ if (Name == "addr_reg_imm_base" || Name == "addr_reg_imm_offset") {
+ std::optional<SelectedAddress> Address = selectAddress(Instruction, Function, OperandIndex);
+ if (!Address)
+ return std::optional<GIMatchComplexOperand>{};
+ if (Name == "addr_reg_imm_offset") {
+ return GIMatchComplexOperand{
+ .Operand = MachineOperand::immediate(Address->Offset),
+ .Instruction = std::nullopt,
+ };
+ }
+ return GIMatchComplexOperand{
+ .Operand = std::move(Address->Base),
+ .Instruction = std::nullopt,
+ };
+ }
+ if (Name == "select_condition_lhs" || Name == "select_condition_rhs" ||
+ Name == "select_condition_code") {
+ auto Condition = selectIntegerCondition(Instruction, Function, OperandIndex);
+ if (!Condition)
+ return std::unexpected(Condition.error());
+ if (!*Condition)
+ return std::optional<GIMatchComplexOperand>{};
+ if (Name == "select_condition_code") {
+ return GIMatchComplexOperand{
+ .Operand = MachineOperand::immediate(static_cast<std::int64_t>((*Condition)->Code)),
+ .Instruction = std::nullopt,
+ };
+ }
+ const Register Value =
+ Name == "select_condition_lhs" ? (*Condition)->Left : (*Condition)->Right;
+ auto Operand = MachineOperand::registerOperand(Value, {RegisterOperandFlag::Use});
+ if (!Operand)
+ return std::unexpected(Operand.error());
+ return GIMatchComplexOperand{
+ .Operand = std::move(*Operand),
+ .Instruction = std::nullopt,
+ };
+ }
+ if (Name == "fcopysign_to_s32" || Name == "fcopysign_to_s64") {
+ const MachineOperand *SourceOperand = Instruction.operandAt(OperandIndex);
+ const Register *Source =
+ SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>();
+ if (Source == nullptr || !Source->isVirtual())
+ return std::optional<GIMatchComplexOperand>{};
+ const std::uint32_t DestinationWidth = Name == "fcopysign_to_s32" ? 32U : 64U;
+ auto DestinationType = LowLevelType::makeCategoryNeutralScalar(DestinationWidth);
+ if (!DestinationType)
+ return std::unexpected(DestinationType.error());
+ auto SourceUse = MachineOperand::registerOperand(*Source, {RegisterOperandFlag::Use});
+ if (!SourceUse)
+ return std::unexpected(SourceUse.error());
+ std::vector<MachineOperand> Operands;
+ Operands.push_back(std::move(*SourceUse));
+ Operands.push_back(MachineOperand::immediate(DestinationWidth == 32 ? 7 : 0));
+ return GIMatchComplexOperand{
+ .Operand = std::nullopt,
+ .Instruction =
+ GIMatchComplexInstruction{
+ .ResultOpcode =
+ instructionOpcode(DestinationWidth == 32 ? InstructionOpcode::FCVT_S_D
+ : InstructionOpcode::FCVT_D_S),
+ .ResultType = *DestinationType,
+ .Operands = std::move(Operands),
+ },
+ };
+ }
+ const std::uint64_t ShiftWidth = Name == "shift_mask_xlen" ? 64U
+ : Name == "shift_mask_32" ? 32U
+ : 0U;
+ if (ShiftWidth == 0)
+ return std::optional<GIMatchComplexOperand>{};
+ const MachineOperand *Root = Instruction.operandAt(OperandIndex);
+ const Register *RootRegister = Root == nullptr ? nullptr : Root->getIf<Register>();
+ if (RootRegister == nullptr || !RootRegister->isVirtual())
+ return std::optional<GIMatchComplexOperand>{};
+
+ Register ShiftRegister = *RootRegister;
+ const auto Definition = [&Function](Register Reg) -> const MachineInstr * {
+ return Reg.isVirtual() ? Function.registerInfo().getVRegDef(Reg) : nullptr;
+ };
+ const auto UseOperand = [](Register Reg) -> Expected<MachineOperand> {
+ return MachineOperand::registerOperand(Reg, {RegisterOperandFlag::Use});
+ };
+ const auto DirectResult = [&UseOperand](Register Reg) -> Expected<GIMatchComplexOperand> {
+ auto Operand = UseOperand(Reg);
+ if (!Operand)
+ return std::unexpected(Operand.error());
+ return GIMatchComplexOperand{
+ .Operand = std::move(*Operand),
+ .Instruction = std::nullopt,
+ };
+ };
+
+ if (const MachineInstr *Extension = Definition(ShiftRegister);
+ Extension != nullptr && Extension->opcode() == opcode(GenericOpcode::G_ZEXT)) {
+ const MachineOperand *SourceOperand = Extension->operandAt(1);
+ const Register *Source =
+ SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>();
+ if (Source != nullptr)
+ ShiftRegister = *Source;
+ }
+
+ const std::uint64_t ShiftMask = ShiftWidth - 1U;
+ if (const MachineInstr *And = Definition(ShiftRegister);
+ And != nullptr && And->opcode() == opcode(GenericOpcode::G_AND)) {
+ const MachineOperand *SourceOperand = And->operandAt(1);
+ const MachineOperand *MaskOperand = And->operandAt(2);
+ const Register *Source =
+ SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>();
+ const Register *Mask = MaskOperand == nullptr ? nullptr : MaskOperand->getIf<Register>();
+ const auto MaskValue =
+ Mask == nullptr ? std::nullopt : directIntegerConstant(Function.registerInfo(), *Mask);
+ if (Source != nullptr && MaskValue) {
+ const std::uint64_t RawMask = std::bit_cast<std::uint64_t>(*MaskValue);
+ bool ElideMask = (RawMask & ShiftMask) == ShiftMask;
+ if (!ElideMask) {
+ GISelValueTracking ValueTracking(Function, Target);
+ auto KnownZeroes = ValueTracking.getKnownZeroes(*Source);
+ if (!KnownZeroes)
+ return std::unexpected(KnownZeroes.error());
+ ElideMask = ((RawMask | KnownZeroes->word(0)) & ShiftMask) == ShiftMask;
+ }
+ if (ElideMask)
+ ShiftRegister = *Source;
+ }
+ }
+
+ const MachineInstr *ShiftDefinition = Definition(ShiftRegister);
+ if (ShiftDefinition == nullptr)
+ return DirectResult(ShiftRegister);
+ const MachineOperand *LeftOperand = ShiftDefinition->operandAt(1);
+ const MachineOperand *RightOperand = ShiftDefinition->operandAt(2);
+ const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf<Register>();
+ const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf<Register>();
+
+ if (ShiftDefinition->opcode() == opcode(GenericOpcode::G_ADD) && Left != nullptr &&
+ Right != nullptr) {
+ const auto Immediate = directIntegerConstant(Function.registerInfo(), *Right);
+ if (Immediate != std::nullopt && *Immediate != 0 &&
+ std::bit_cast<std::uint64_t>(*Immediate) % ShiftWidth == 0)
+ return DirectResult(*Left);
+ }
+
+ if (ShiftDefinition->opcode() != opcode(GenericOpcode::G_SUB) || Left == nullptr ||
+ Right == nullptr)
+ return DirectResult(ShiftRegister);
+ const auto Immediate = directIntegerConstant(Function.registerInfo(), *Left);
+ if (!Immediate || *Immediate == 0)
+ return DirectResult(ShiftRegister);
+
+ const std::uint64_t Remainder = std::bit_cast<std::uint64_t>(*Immediate) % ShiftWidth;
+ if (Remainder != 0 && Remainder != ShiftMask)
+ return DirectResult(ShiftRegister);
+ auto Type = Function.registerInfo().getType(*Right);
+ if (!Type)
+ return std::unexpected(Type.error());
+ std::vector<MachineOperand> Operands;
+ if (Remainder == 0) {
+ auto Zero = physicalRegister(PhysicalRegisterID::X0);
+ if (!Zero)
+ return std::unexpected(Zero.error());
+ auto ZeroOperand = UseOperand(Register::fromPhysical(*Zero));
+ auto SourceOperand = UseOperand(*Right);
+ if (!ZeroOperand)
+ return std::unexpected(ZeroOperand.error());
+ if (!SourceOperand)
+ return std::unexpected(SourceOperand.error());
+ Operands.push_back(std::move(*ZeroOperand));
+ Operands.push_back(std::move(*SourceOperand));
+ return GIMatchComplexOperand{
+ .Operand = std::nullopt,
+ .Instruction =
+ GIMatchComplexInstruction{
+ .ResultOpcode = instructionOpcode(InstructionOpcode::SUBW),
+ .ResultType = *Type,
+ .Operands = std::move(Operands),
+ },
+ };
+ }
+
+ auto SourceOperand = UseOperand(*Right);
+ if (!SourceOperand)
+ return std::unexpected(SourceOperand.error());
+ Operands.push_back(std::move(*SourceOperand));
+ Operands.push_back(MachineOperand::immediate(-1));
+ return GIMatchComplexOperand{
+ .Operand = std::nullopt,
+ .Instruction =
+ GIMatchComplexInstruction{
+ .ResultOpcode = instructionOpcode(InstructionOpcode::XORI),
+ .ResultType = *Type,
+ .Operands = std::move(Operands),
+ },
+ };
+ }
+
+ [[nodiscard]] Expected<void> select(MachineInstr &Instruction, MachineFunction &Function,
+ const TargetDescription &Target) const override {
+ auto Generated = GIMatchTableExecutor::select(generatedGlobalISelMatchTable(), Instruction,
+ Function, Target);
+ if (!Generated)
+ return std::unexpected(Generated.error());
+ if (*Generated)
+ return {};
+ if (Instruction.opcode() == opcode(GenericOpcode::COPY))
+ return selectCopy(Instruction, Function, Target);
+ const GenericOpcode Generic = static_cast<GenericOpcode>(Instruction.opcode().value());
+ const std::optional<LowLevelType> Type = resultType(Function, Instruction);
+
+ switch (Generic) {
+ case GenericOpcode::G_CONSTANT:
+ return selectIntegerConstant(Instruction, Function, Target, Type);
+ case GenericOpcode::G_FCONSTANT:
+ return selectFloatingConstant(Instruction, Function, Target, Type);
+ case GenericOpcode::G_ASSERT_SEXT:
+ case GenericOpcode::G_ASSERT_ZEXT:
+ case GenericOpcode::G_ASSERT_ALIGN:
+ if (Instruction.operandCount() == 3) {
+ auto EraseImmediate = Instruction.eraseOperand(2);
+ if (!EraseImmediate)
+ return std::unexpected(EraseImmediate.error());
+ }
+ return Instruction.setOpcode(opcode(GenericOpcode::COPY));
+ case GenericOpcode::G_FREEZE:
+ case GenericOpcode::G_CONSTANT_FOLD_BARRIER:
+ case GenericOpcode::G_PTRTOINT:
+ case GenericOpcode::G_INTTOPTR:
+ case GenericOpcode::G_ANYEXT:
+ case GenericOpcode::G_TRUNC:
+ return Instruction.setOpcode(opcode(GenericOpcode::COPY));
+ case GenericOpcode::G_FCMP:
+ return selectFloatingCompare(Instruction, Function, Target);
+ case GenericOpcode::G_FENCE:
+ return selectFence(Instruction);
+ case GenericOpcode::G_GLOBAL_VALUE:
+ case GenericOpcode::G_JUMP_TABLE:
+ case GenericOpcode::G_CONSTANT_POOL:
+ return selectGlobalAddress(Instruction, Function, Target);
+ case GenericOpcode::G_PHI:
+ return Instruction.setOpcode(opcode(GenericOpcode::PHI));
+ case GenericOpcode::COPY:
+ case GenericOpcode::IMPLICIT_DEF:
+ case GenericOpcode::PHI:
+ case GenericOpcode::REG_SEQUENCE:
+ case GenericOpcode::INSERT_SUBREG:
+ case GenericOpcode::EXTRACT_SUBREG:
+ case GenericOpcode::SUBREG_TO_REG:
+ case GenericOpcode::KILL:
+ case GenericOpcode::BUNDLE:
+ case GenericOpcode::INLINEASM:
+ case GenericOpcode::CORE_PSEUDO_END:
+ break;
+ default:
+ return makeUnexpected(ErrorCode::Unsupported,
+ "RISC-V instruction selector has no rule for generic opcode");
+ }
+ return makeUnexpected(ErrorCode::Unsupported,
+ "RISC-V instruction selector has no rule for generic opcode");
+ }
+
+private:
+ [[nodiscard]] Expected<const MachineMemOperand *> createGOTLoad(MachineFunction &Function,
+ LowLevelType Type) const {
+ auto Size = MemorySize::fixed(Type.minimumBitWidth() / 8U);
+ if (!Size)
+ return std::unexpected(Size.error());
+ auto Alignment = rcc::Alignment::create(Type.minimumBitWidth() / 8U);
+ if (!Alignment)
+ return std::unexpected(Alignment.error());
+ MachineMemOperandDesc Description;
+ Description.Accesses = {MemoryAccessFlag::Load, MemoryAccessFlag::Dereferenceable,
+ MemoryAccessFlag::Invariant};
+ Description.Size = *Size;
+ Description.MemoryType = Type;
+ Description.Align = *Alignment;
+ Description.Base = MemoryBase::got();
+ return Function.createMemOperand(std::move(Description));
+ }
+
+ [[nodiscard]] Expected<void> selectGlobalAddress(MachineInstr &Instruction,
+ MachineFunction &Function,
+ const TargetDescription &Target) const {
+ const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+ const MachineOperand *SymbolOperand = Instruction.operandAt(1);
+ const Register *Destination =
+ DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+ const std::optional<LowLevelType> Type = resultType(Function, Instruction);
+ if (Destination == nullptr || SymbolOperand == nullptr || !Type || !Type->isPointer()) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V address selection has malformed generic operands");
+ }
+ auto DestinationConstraint = constrainBuilderRegisters(Function, Target, {*Destination});
+ if (!DestinationConstraint)
+ return std::unexpected(DestinationConstraint.error());
+
+ GlobalValueProperties Properties;
+ if (Instruction.opcode() == opcode(GenericOpcode::G_GLOBAL_VALUE)) {
+ const auto *Global = SymbolOperand->getIf<GlobalSymbolOperand>();
+ if (Global == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V global address has no global symbol operand");
+ Properties = Global->Properties;
+ if (Properties.ThreadLocal) {
+ return makeUnexpected(ErrorCode::Unsupported,
+ "RISC-V GlobalISel does not support thread-local global addresses");
+ }
+ }
+
+ const bool TaggedGlobals = Subtarget.features().has(RISCVSubtargetOption::TaggedGlobals);
+ const bool UseGOT =
+ Subtarget.relocationModel() == RelocationModel::PositionIndependent || TaggedGlobals;
+ const auto ReplaceWithPseudo = [&](InstructionOpcode OpcodeValue) -> Expected<void> {
+ auto SetOpcode = Instruction.setOpcode(instructionOpcode(OpcodeValue));
+ if (!SetOpcode)
+ return std::unexpected(SetOpcode.error());
+ return constrainSelectedInstruction(Instruction, Function, Target);
+ };
+ const auto BuildGOTLoad = [&]() -> Expected<void> {
+ auto Memory = createGOTLoad(Function, *Type);
+ if (!Memory)
+ return std::unexpected(Memory.error());
+ MachineIRBuilder Builder(Function, Target);
+ auto Point = Builder.setInsertPointBefore(Instruction);
+ if (!Point)
+ return std::unexpected(Point.error());
+ auto Pending = Builder.buildInstr(instructionOpcode(InstructionOpcode::PseudoLGA));
+ if (!Pending)
+ return std::unexpected(Pending.error());
+ auto Def = Pending->addDef(*Destination);
+ if (!Def)
+ return std::unexpected(Def.error());
+ auto Symbol = Pending->addOperand(SymbolOperand->cloneDetached());
+ if (!Symbol)
+ return std::unexpected(Symbol.error());
+ auto MemoryAdded = Pending->addMemOperand(**Memory);
+ if (!MemoryAdded)
+ return std::unexpected(MemoryAdded.error());
+ auto Result = Pending->build();
+ if (!Result)
+ return std::unexpected(Result.error());
+ auto Flags = propagateSelectionFlags(**Result, Instruction, Target);
+ if (!Flags)
+ return std::unexpected(Flags.error());
+ auto Constraint = constrainSelectedInstruction(**Result, Function, Target);
+ if (!Constraint)
+ return std::unexpected(Constraint.error());
+ return Instruction.eraseFromParent();
+ };
+
+ if (UseGOT) {
+ if (Properties.DSOLocal && !TaggedGlobals)
+ return ReplaceWithPseudo(InstructionOpcode::PseudoLLA);
+ return BuildGOTLoad();
+ }
+
+ if (Subtarget.codeModel() == CodeModel::Medium) {
+ if (Properties.ExternalWeak)
+ return BuildGOTLoad();
+ return ReplaceWithPseudo(InstructionOpcode::PseudoLLA);
+ }
+ if (Subtarget.codeModel() != CodeModel::Small) {
+ return makeUnexpected(ErrorCode::Unsupported,
+ "RISC-V GlobalISel does not support this global address code model");
+ }
+
+ auto Temporary = createTemporary(Function, *Type, Target);
+ if (!Temporary)
+ return std::unexpected(Temporary.error());
+ MachineIRBuilder Builder(Function, Target);
+ auto Point = Builder.setInsertPointBefore(Instruction);
+ if (!Point)
+ return std::unexpected(Point.error());
+ auto HighPending = Builder.buildInstr(instructionOpcode(InstructionOpcode::LUI));
+ if (!HighPending)
+ return std::unexpected(HighPending.error());
+ auto HighDef = HighPending->addDef(*Temporary);
+ if (!HighDef)
+ return std::unexpected(HighDef.error());
+ MachineOperand HighSymbol = SymbolOperand->cloneDetached();
+ HighSymbol.setTargetFlags(AbsoluteHighFlag);
+ auto HighOperand = HighPending->addOperand(std::move(HighSymbol));
+ if (!HighOperand)
+ return std::unexpected(HighOperand.error());
+ auto High = HighPending->build();
+ if (!High)
+ return std::unexpected(High.error());
+ auto HighFlags = propagateSelectionFlags(**High, Instruction, Target);
+ if (!HighFlags)
+ return std::unexpected(HighFlags.error());
+ auto HighConstraint = constrainSelectedInstruction(**High, Function, Target);
+ if (!HighConstraint)
+ return std::unexpected(HighConstraint.error());
+
+ auto LowPending = Builder.buildInstr(instructionOpcode(InstructionOpcode::ADDI));
+ if (!LowPending)
+ return std::unexpected(LowPending.error());
+ auto LowDef = LowPending->addDef(*Destination);
+ if (!LowDef)
+ return std::unexpected(LowDef.error());
+ auto LowSource = LowPending->addUse(*Temporary);
+ if (!LowSource)
+ return std::unexpected(LowSource.error());
+ MachineOperand LowSymbol = SymbolOperand->cloneDetached();
+ LowSymbol.setTargetFlags(AbsoluteLowFlag);
+ auto LowOperand = LowPending->addOperand(std::move(LowSymbol));
+ if (!LowOperand)
+ return std::unexpected(LowOperand.error());
+ auto Low = LowPending->build();
+ if (!Low)
+ return std::unexpected(Low.error());
+ auto LowFlags = propagateSelectionFlags(**Low, Instruction, Target);
+ if (!LowFlags)
+ return std::unexpected(LowFlags.error());
+ auto LowConstraint = constrainSelectedInstruction(**Low, Function, Target);
+ if (!LowConstraint)
+ return std::unexpected(LowConstraint.error());
+ return Instruction.eraseFromParent();
+ }
+
+ [[nodiscard]] Expected<void> selectCopy(MachineInstr &Instruction, MachineFunction &Function,
+ const TargetDescription &Target) const {
+ const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+ const Register *Destination =
+ DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+ if (Destination == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V COPY selection requires a register destination");
+ if (!Destination->isVirtual())
+ return {};
+ return RegisterBanks.constrainRegister(Function, *Destination, Target);
+ }
+
+ [[nodiscard]] Expected<Register>
+ createTemporary(MachineFunction &Function, LowLevelType Type, const TargetDescription &Target,
+ std::optional<RegisterBankID> Bank = std::nullopt) const {
+ auto Temporary = Function.registerInfo().createVirtualRegister(Type);
+ if (!Temporary)
+ return std::unexpected(Temporary.error());
+ if (Bank) {
+ auto SetBank = Function.registerInfo().setRegisterBank(*Temporary, *Bank);
+ if (!SetBank)
+ return std::unexpected(SetBank.error());
+ }
+ auto Constraint = RegisterBanks.constrainRegister(Function, *Temporary, Target);
+ if (!Constraint)
+ return std::unexpected(Constraint.error());
+ return *Temporary;
+ }
+
+ [[nodiscard]] Expected<void>
+ constrainBuilderRegisters(MachineFunction &Function, const TargetDescription &Target,
+ std::initializer_list<Register> Registers) const {
+ for (Register Reg : Registers) {
+ if (!Reg.isVirtual())
+ continue;
+ auto Constraint = RegisterBanks.constrainRegister(Function, Reg, Target);
+ if (!Constraint)
+ return std::unexpected(Constraint.error());
+ }
+ return {};
+ }
+
+ [[nodiscard]] Expected<void> replaceImmediate(MachineInstr &Instruction, std::size_t Index,
+ std::int64_t Value) const {
+ if (Instruction.operandAt(Index) == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V instruction is missing an immediate operand");
+ auto Erased = Instruction.eraseOperand(Index);
+ if (!Erased)
+ return std::unexpected(Erased.error());
+ return Instruction.insertOperand(Index, MachineOperand::immediate(Value));
+ }
+
+ [[nodiscard]] Expected<MachineInstr *>
+ buildImmediateAfter(MachineIRBuilder &Builder, MachineInstr &Anchor, InstructionOpcode Opcode,
+ Register Def, Register Source, std::int64_t Value) const {
+ auto Point = Builder.setInsertPointAfter(Anchor);
+ if (!Point)
+ return std::unexpected(Point.error());
+ auto RegisterConstraints =
+ constrainBuilderRegisters(Builder.function(), Builder.target(), {Def, Source});
+ if (!RegisterConstraints)
+ return std::unexpected(RegisterConstraints.error());
+ auto Pending = Builder.buildInstr(instructionOpcode(Opcode));
+ if (!Pending)
+ return std::unexpected(Pending.error());
+ auto DefResult = Pending->addDef(Def);
+ if (!DefResult)
+ return std::unexpected(DefResult.error());
+ auto SourceResult = Pending->addUse(Source);
+ if (!SourceResult)
+ return std::unexpected(SourceResult.error());
+ auto ImmediateResult = Pending->addImmediate(Value);
+ if (!ImmediateResult)
+ return std::unexpected(ImmediateResult.error());
+ auto Built = Pending->build();
+ if (!Built)
+ return std::unexpected(Built.error());
+ auto FlagPropagation = propagateSelectionFlags(**Built, Anchor, Builder.target());
+ if (!FlagPropagation)
+ return std::unexpected(FlagPropagation.error());
+ auto Constraint = constrainSelectedInstruction(**Built, Builder.function(), Builder.target());
+ if (!Constraint)
+ return std::unexpected(Constraint.error());
+ return *Built;
+ }
+
+ [[nodiscard]] Expected<MachineInstr *>
+ buildImmediateOnlyAfter(MachineIRBuilder &Builder, MachineInstr &Anchor, InstructionOpcode Opcode,
+ Register Def, std::int64_t Value) const {
+ auto Point = Builder.setInsertPointAfter(Anchor);
+ if (!Point)
+ return std::unexpected(Point.error());
+ auto RegisterConstraints =
+ constrainBuilderRegisters(Builder.function(), Builder.target(), {Def});
+ if (!RegisterConstraints)
+ return std::unexpected(RegisterConstraints.error());
+ auto Pending = Builder.buildInstr(instructionOpcode(Opcode));
+ if (!Pending)
+ return std::unexpected(Pending.error());
+ auto DefResult = Pending->addDef(Def);
+ if (!DefResult)
+ return std::unexpected(DefResult.error());
+ auto ImmediateResult = Pending->addImmediate(Value);
+ if (!ImmediateResult)
+ return std::unexpected(ImmediateResult.error());
+ auto Built = Pending->build();
+ if (!Built)
+ return std::unexpected(Built.error());
+ auto FlagPropagation = propagateSelectionFlags(**Built, Anchor, Builder.target());
+ if (!FlagPropagation)
+ return std::unexpected(FlagPropagation.error());
+ auto Constraint = constrainSelectedInstruction(**Built, Builder.function(), Builder.target());
+ if (!Constraint)
+ return std::unexpected(Constraint.error());
+ return *Built;
+ }
+
+ [[nodiscard]] Expected<MachineInstr *>
+ buildImmediateBefore(MachineIRBuilder &Builder, MachineInstr &Anchor, InstructionOpcode Opcode,
+ Register Def, Register Source, std::int64_t Value) const {
+ auto Point = Builder.setInsertPointBefore(Anchor);
+ if (!Point)
+ return std::unexpected(Point.error());
+ auto RegisterConstraints =
+ constrainBuilderRegisters(Builder.function(), Builder.target(), {Def, Source});
+ if (!RegisterConstraints)
+ return std::unexpected(RegisterConstraints.error());
+ auto Pending = Builder.buildInstr(instructionOpcode(Opcode));
+ if (!Pending)
+ return std::unexpected(Pending.error());
+ auto DefResult = Pending->addDef(Def);
+ if (!DefResult)
+ return std::unexpected(DefResult.error());
+ auto SourceResult = Pending->addUse(Source);
+ if (!SourceResult)
+ return std::unexpected(SourceResult.error());
+ auto ImmediateResult = Pending->addImmediate(Value);
+ if (!ImmediateResult)
+ return std::unexpected(ImmediateResult.error());
+ auto Built = Pending->build();
+ if (!Built)
+ return std::unexpected(Built.error());
+ auto FlagPropagation = propagateSelectionFlags(**Built, Anchor, Builder.target());
+ if (!FlagPropagation)
+ return std::unexpected(FlagPropagation.error());
+ auto Constraint = constrainSelectedInstruction(**Built, Builder.function(), Builder.target());
+ if (!Constraint)
+ return std::unexpected(Constraint.error());
+ return *Built;
+ }
+
+ [[nodiscard]] Expected<void> selectFloatingCompare(MachineInstr &Instruction,
+ MachineFunction &Function,
+ const TargetDescription &Target) const {
+ const MachineOperand *PredicateMO = Instruction.operandAt(1);
+ const auto *Predicate =
+ PredicateMO == nullptr ? nullptr : PredicateMO->getIf<PredicateOperand>();
+ const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+ const MachineOperand *LeftOperand = Instruction.operandAt(2);
+ const MachineOperand *RightOperand = Instruction.operandAt(3);
+ const Register *Destination =
+ DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+ const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf<Register>();
+ const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf<Register>();
+ const std::optional<LowLevelType> FloatType = operandType(Function, Instruction, 2);
+ const std::optional<LowLevelType> ResultType = resultType(Function, Instruction);
+ if (Predicate == nullptr || Predicate->floatingPredicate() == nullptr ||
+ Destination == nullptr || Left == nullptr || Right == nullptr || !FloatType || !ResultType)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V floating comparison has invalid operands");
+ FloatingPredicate Condition = *Predicate->floatingPredicate();
+ const Register DestinationRegister = *Destination;
+ Register LeftRegister = *Left;
+ Register RightRegister = *Right;
+ MachineIRBuilder Builder(Function, Target);
+
+ const auto BuildBinary = [&](InstructionOpcode OpcodeValue, Register Def, Register LeftValue,
+ Register RightValue) -> Expected<MachineInstr *> {
+ auto Point = Builder.setInsertPointBefore(Instruction);
+ if (!Point)
+ return std::unexpected(Point.error());
+ auto Registers = constrainBuilderRegisters(Function, Target, {Def, LeftValue, RightValue});
+ if (!Registers)
+ return std::unexpected(Registers.error());
+ auto Pending = Builder.buildInstr(instructionOpcode(OpcodeValue));
+ if (!Pending)
+ return std::unexpected(Pending.error());
+ auto DefResult = Pending->addDef(Def);
+ if (!DefResult)
+ return std::unexpected(DefResult.error());
+ auto LeftResult = Pending->addUse(LeftValue);
+ if (!LeftResult)
+ return std::unexpected(LeftResult.error());
+ auto RightResult = Pending->addUse(RightValue);
+ if (!RightResult)
+ return std::unexpected(RightResult.error());
+ auto Built = Pending->build();
+ if (!Built)
+ return std::unexpected(Built.error());
+ auto Constraint = constrainSelectedInstruction(**Built, Function, Target);
+ if (!Constraint)
+ return std::unexpected(Constraint.error());
+ return *Built;
+ };
+ const auto BuildImmediate = [&](InstructionOpcode OpcodeValue, Register Def, Register Source,
+ std::int64_t Immediate) -> Expected<MachineInstr *> {
+ auto Point = Builder.setInsertPointBefore(Instruction);
+ if (!Point)
+ return std::unexpected(Point.error());
+ auto Registers = constrainBuilderRegisters(Function, Target, {Def, Source});
+ if (!Registers)
+ return std::unexpected(Registers.error());
+ auto Pending = Builder.buildInstr(instructionOpcode(OpcodeValue));
+ if (!Pending)
+ return std::unexpected(Pending.error());
+ auto DefResult = Pending->addDef(Def);
+ if (!DefResult)
+ return std::unexpected(DefResult.error());
+ auto SourceResult = Pending->addUse(Source);
+ if (!SourceResult)
+ return std::unexpected(SourceResult.error());
+ auto ImmediateResult = Pending->addImmediate(Immediate);
+ if (!ImmediateResult)
+ return std::unexpected(ImmediateResult.error());
+ auto Built = Pending->build();
+ if (!Built)
+ return std::unexpected(Built.error());
+ auto Constraint = constrainSelectedInstruction(**Built, Function, Target);
+ if (!Constraint)
+ return std::unexpected(Constraint.error());
+ return *Built;
+ };
+
+ Register TemporaryResult = DestinationRegister;
+ bool NeedsInvert = false;
+ if (legalizeFloatingPredicate(LeftRegister, RightRegister, Condition, NeedsInvert)) {
+ if (NeedsInvert) {
+ auto Temporary = createTemporary(Function, *ResultType, Target);
+ if (!Temporary)
+ return std::unexpected(Temporary.error());
+ TemporaryResult = *Temporary;
+ }
+ auto OpcodeValue = floatingCompareOpcode(Condition, FloatType->scalarBitWidth());
+ if (!OpcodeValue)
+ return std::unexpected(OpcodeValue.error());
+ auto Comparison = BuildBinary(*OpcodeValue, TemporaryResult, LeftRegister, RightRegister);
+ if (!Comparison)
+ return std::unexpected(Comparison.error());
+ } else if (Condition == FloatingPredicate::OrderedNotEqual ||
+ Condition == FloatingPredicate::UnorderedEqual) {
+ NeedsInvert = Condition == FloatingPredicate::UnorderedEqual;
+ auto First = createTemporary(Function, *ResultType, Target);
+ auto Second = createTemporary(Function, *ResultType, Target);
+ if (!First || !Second)
+ return std::unexpected((!First ? First.error() : Second.error()));
+ auto LessOpcode =
+ floatingCompareOpcode(FloatingPredicate::OrderedLessThan, FloatType->scalarBitWidth());
+ if (!LessOpcode)
+ return std::unexpected(LessOpcode.error());
+ auto FirstComparison = BuildBinary(*LessOpcode, *First, LeftRegister, RightRegister);
+ if (!FirstComparison)
+ return std::unexpected(FirstComparison.error());
+ auto SecondComparison = BuildBinary(*LessOpcode, *Second, RightRegister, LeftRegister);
+ if (!SecondComparison)
+ return std::unexpected(SecondComparison.error());
+ if (NeedsInvert) {
+ auto Temporary = createTemporary(Function, *ResultType, Target);
+ if (!Temporary)
+ return std::unexpected(Temporary.error());
+ TemporaryResult = *Temporary;
+ }
+ auto Combined = BuildBinary(InstructionOpcode::OR, TemporaryResult, *First, *Second);
+ if (!Combined)
+ return std::unexpected(Combined.error());
+ } else if (Condition == FloatingPredicate::Ordered ||
+ Condition == FloatingPredicate::Unordered) {
+ NeedsInvert = Condition == FloatingPredicate::Unordered;
+ auto EqualOpcode =
+ floatingCompareOpcode(FloatingPredicate::OrderedEqual, FloatType->scalarBitWidth());
+ if (!EqualOpcode)
+ return std::unexpected(EqualOpcode.error());
+ if (NeedsInvert) {
+ auto Temporary = createTemporary(Function, *ResultType, Target);
+ if (!Temporary)
+ return std::unexpected(Temporary.error());
+ TemporaryResult = *Temporary;
+ }
+ if (LeftRegister == RightRegister) {
+ auto Comparison = BuildBinary(*EqualOpcode, TemporaryResult, LeftRegister, LeftRegister);
+ if (!Comparison)
+ return std::unexpected(Comparison.error());
+ } else {
+ auto First = createTemporary(Function, *ResultType, Target);
+ auto Second = createTemporary(Function, *ResultType, Target);
+ if (!First || !Second)
+ return std::unexpected((!First ? First.error() : Second.error()));
+ auto FirstComparison = BuildBinary(*EqualOpcode, *First, LeftRegister, LeftRegister);
+ if (!FirstComparison)
+ return std::unexpected(FirstComparison.error());
+ auto SecondComparison = BuildBinary(*EqualOpcode, *Second, RightRegister, RightRegister);
+ if (!SecondComparison)
+ return std::unexpected(SecondComparison.error());
+ auto Combined = BuildBinary(InstructionOpcode::AND, TemporaryResult, *First, *Second);
+ if (!Combined)
+ return std::unexpected(Combined.error());
+ }
+ } else {
+ return makeUnexpected(ErrorCode::Unsupported,
+ "RISC-V floating comparison predicate has no LLVM selection");
+ }
+
+ if (NeedsInvert) {
+ auto Inverted =
+ BuildImmediate(InstructionOpcode::XORI, DestinationRegister, TemporaryResult, 1);
+ if (!Inverted)
+ return std::unexpected(Inverted.error());
+ }
+ return Instruction.eraseFromParent();
+ }
+
+ [[nodiscard]] Expected<void> selectFence(MachineInstr &Instruction) const {
+ const auto Ordering = signedImmediate(*Instruction.operandAt(0));
+ if (!Ordering)
+ return std::unexpected(Ordering.error());
+ std::int64_t Pred = 0;
+ std::int64_t Succ = 0;
+ switch (*Ordering) {
+ case 4:
+ Pred = 1;
+ Succ = 3;
+ break;
+ case 5:
+ Pred = 3;
+ Succ = 2;
+ break;
+ case 6:
+ case 7:
+ Pred = 3;
+ Succ = 3;
+ break;
+ default:
+ return makeUnexpected(ErrorCode::Unsupported, "RISC-V fence ordering is unsupported");
+ }
+ auto SetPred = replaceImmediate(Instruction, 0, Pred);
+ if (!SetPred)
+ return std::unexpected(SetPred.error());
+ auto SetSucc = replaceImmediate(Instruction, 1, Succ);
+ if (!SetSucc)
+ return std::unexpected(SetSucc.error());
+ return Instruction.setOpcode(instructionOpcode(InstructionOpcode::FENCE));
+ }
+
+ [[nodiscard]] Expected<void> selectIntegerConstant(MachineInstr &Instruction,
+ MachineFunction &Function,
+ const TargetDescription &Target,
+ std::optional<LowLevelType> Type) const {
+ if (!Type || !isSupportedIntegerType(*Type)) {
+ return makeUnexpected(ErrorCode::Unsupported,
+ "RISC-V constant selection requires an i1-i64 or pointer result");
+ }
+ const MachineOperand *ImmediateOperand = Instruction.operandAt(1);
+ if (ImmediateOperand == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V generic constant has no immediate operand");
+ auto ImmediateValue = signedImmediate(*ImmediateOperand);
+ if (!ImmediateValue)
+ return std::unexpected(ImmediateValue.error());
+ const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+ const Register *DestinationValue =
+ DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+ if (DestinationValue == nullptr || !DestinationValue->isVirtual()) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V generic constant must define a virtual register");
+ }
+ const Register Destination = *DestinationValue;
+ auto DestinationConstraint = constrainBuilderRegisters(Function, Target, {Destination});
+ if (!DestinationConstraint)
+ return std::unexpected(DestinationConstraint.error());
+ const std::int64_t Value = *ImmediateValue;
+ auto Zero = physicalRegister(PhysicalRegisterID::X0);
+ if (!Zero)
+ return std::unexpected(Zero.error());
+ if (Value == 0) {
+ auto EraseImmediate = Instruction.eraseOperand(1);
+ if (!EraseImmediate)
+ return std::unexpected(EraseImmediate.error());
+ auto Source = insertPhysicalRegister(Instruction, 1, *Zero, RegisterOperandFlag::Use);
+ if (!Source)
+ return std::unexpected(Source.error());
+ return Instruction.setOpcode(opcode(GenericOpcode::COPY));
+ }
+
+ ImmediateMaterializationSequence Sequence =
+ generateRV64IntegerMaterialization(Value, Subtarget.features());
+ if (Sequence.empty()) {
+ return makeUnexpected(ErrorCode::InternalError,
+ "RISC-V integer materialization produced an empty sequence");
+ }
+ Register Current = Destination;
+ if (Sequence.size() != 1) {
+ auto Temporary = createTemporary(Function, *Type, Target);
+ if (!Temporary)
+ return std::unexpected(Temporary.error());
+ Current = *Temporary;
+ }
+
+ const auto SelectHead = [&](const ImmediateMaterializationInstruction &Step,
+ Register Definition) -> Expected<void> {
+ auto SetOpcode = Instruction.setOpcode(instructionOpcode(Step.Opcode));
+ if (!SetOpcode)
+ return std::unexpected(SetOpcode.error());
+ auto SetDestination = Instruction.operandAt(0)->setRegister(Definition);
+ if (!SetDestination)
+ return std::unexpected(SetDestination.error());
+ auto EraseImmediate = Instruction.eraseOperand(1);
+ if (!EraseImmediate)
+ return std::unexpected(EraseImmediate.error());
+ if (Step.OperandKind == ImmediateMaterializationOperandKind::Immediate)
+ return Instruction.addOperand(MachineOperand::immediate(Step.Immediate));
+ auto Source = insertPhysicalRegister(Instruction, 1, *Zero, RegisterOperandFlag::Use);
+ if (!Source)
+ return std::unexpected(Source.error());
+ return Instruction.addOperand(MachineOperand::immediate(Step.Immediate));
+ };
+ auto Head = SelectHead(Sequence.front(), Current);
+ if (!Head)
+ return std::unexpected(Head.error());
+
+ MachineIRBuilder Builder(Function, Target);
+ MachineInstr *Anchor = &Instruction;
+ for (std::size_t Index = 1; Index < Sequence.size(); ++Index) {
+ const bool IsFinal = Index + 1U == Sequence.size();
+ Register Next = Destination;
+ if (!IsFinal) {
+ auto Temporary = createTemporary(Function, *Type, Target);
+ if (!Temporary)
+ return std::unexpected(Temporary.error());
+ Next = *Temporary;
+ }
+ const ImmediateMaterializationInstruction &Step = Sequence[Index];
+ Expected<MachineInstr *> Built =
+ Step.OperandKind == ImmediateMaterializationOperandKind::Immediate
+ ? buildImmediateOnlyAfter(Builder, *Anchor, Step.Opcode, Next, Step.Immediate)
+ : buildImmediateAfter(Builder, *Anchor, Step.Opcode, Next, Current, Step.Immediate);
+ if (!Built)
+ return std::unexpected(Built.error());
+ Anchor = *Built;
+ Current = Next;
+ }
+ return {};
+ }
+
+ [[nodiscard]] Expected<void> selectFloatingConstant(MachineInstr &Instruction,
+ MachineFunction &Function,
+ const TargetDescription &Target,
+ std::optional<LowLevelType> Type) const {
+ if (!Type || !isSupportedFloatingScalar(*Type))
+ return makeUnexpected(ErrorCode::Unsupported, "RISC-V floating constant requires f32 or f64");
+ const MachineOperand *Immediate = Instruction.operandAt(1);
+ const auto *Floating =
+ Immediate == nullptr ? nullptr : Immediate->getIf<FloatingImmediateOperand>();
+ if (Floating == nullptr || Floating->Bits.bitWidth() != Type->scalarBitWidth())
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V floating constant has mismatched bit semantics");
+ const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+ const Register *Destination =
+ DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+ if (Destination == nullptr || !Destination->isVirtual())
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V floating constant must define a virtual register");
+ const Register DestinationRegister = *Destination;
+
+ const std::uint32_t BitWidth = Type->scalarBitWidth();
+ const std::uint64_t Bits = Floating->Bits.words().empty() ? 0 : Floating->Bits.words().front();
+ const InstructionOpcode Move =
+ BitWidth == 32 ? InstructionOpcode::FMV_W_X : InstructionOpcode::FMV_D_X;
+ if (Bits == 0) {
+ auto Zero = physicalRegister(PhysicalRegisterID::X0);
+ if (!Zero)
+ return std::unexpected(Zero.error());
+ auto EraseImmediate = Instruction.eraseOperand(1);
+ if (!EraseImmediate)
+ return std::unexpected(EraseImmediate.error());
+ auto Source = insertPhysicalRegister(Instruction, 1, *Zero, RegisterOperandFlag::Use);
+ if (!Source)
+ return std::unexpected(Source.error());
+ return Instruction.setOpcode(instructionOpcode(Move));
+ }
+
+ auto Integer = APInt::fromUnsigned(BitWidth, Bits);
+ if (!Integer)
+ return std::unexpected(Integer.error());
+ auto IntegerType = LowLevelType::makeCategoryNeutralScalar(64);
+ if (!IntegerType)
+ return std::unexpected(IntegerType.error());
+ auto Temporary = createTemporary(Function, *IntegerType, Target);
+ if (!Temporary)
+ return std::unexpected(Temporary.error());
+
+ auto SetOpcode = Instruction.setOpcode(opcode(GenericOpcode::G_CONSTANT));
+ if (!SetOpcode)
+ return std::unexpected(SetOpcode.error());
+ auto SetDestination = Instruction.operandAt(0)->setRegister(*Temporary);
+ if (!SetDestination)
+ return std::unexpected(SetDestination.error());
+ auto EraseImmediate = Instruction.eraseOperand(1);
+ if (!EraseImmediate)
+ return std::unexpected(EraseImmediate.error());
+ auto AddInteger = Instruction.addOperand(MachineOperand::integerImmediate(*Integer));
+ if (!AddInteger)
+ return std::unexpected(AddInteger.error());
+ auto IntegerSelection = selectIntegerConstant(Instruction, Function, Target, *IntegerType);
+ if (!IntegerSelection)
+ return std::unexpected(IntegerSelection.error());
+
+ MachineInstr *IntegerDefinition = nullptr;
+ for (MachineOperand &DefOperand : Function.registerInfo().defOperands(*Temporary)) {
+ IntegerDefinition = DefOperand.parent();
+ break;
+ }
+ if (IntegerDefinition == nullptr)
+ return makeUnexpected(ErrorCode::InternalError,
+ "RISC-V floating constant lost its integer materialization");
+ MachineIRBuilder Builder(Function, Target);
+ auto InsertPoint = Builder.setInsertPointAfter(*IntegerDefinition);
+ if (!InsertPoint)
+ return std::unexpected(InsertPoint.error());
+ auto RegisterConstraints =
+ constrainBuilderRegisters(Function, Target, {DestinationRegister, *Temporary});
+ if (!RegisterConstraints)
+ return std::unexpected(RegisterConstraints.error());
+ auto MoveInstruction = Builder.buildInstr(instructionOpcode(Move));
+ if (!MoveInstruction)
+ return std::unexpected(MoveInstruction.error());
+ auto MoveDef = MoveInstruction->addDef(DestinationRegister);
+ if (!MoveDef)
+ return std::unexpected(MoveDef.error());
+ auto MoveUse = MoveInstruction->addUse(*Temporary);
+ if (!MoveUse)
+ return std::unexpected(MoveUse.error());
+ auto BuiltMove = MoveInstruction->build();
+ if (!BuiltMove)
+ return std::unexpected(BuiltMove.error());
+ auto Constraint = constrainSelectedInstruction(**BuiltMove, Function, Target);
+ if (!Constraint)
+ return std::unexpected(Constraint.error());
+ return {};
+ }
+
+ const RISCVSubtarget &Subtarget;
+ const RISCVRegisterBankInfo &RegisterBanks;
+};
+
+} // namespace
+
+InstructionSelector *createRISCVInstructionSelector(const RISCVSubtarget &Subtarget,
+ const RISCVRegisterBankInfo &RegisterBanks) {
+ return new RISCVInstructionSelector(Subtarget, RegisterBanks);
+}
+
+} // namespace rcc::riscv
diff --git a/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp b/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
new file mode 100644
index 0000000..1e10b16
--- /dev/null
+++ b/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
@@ -0,0 +1,1382 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "RISCVLegalizerInfo.h"
+
+#include "RISCVImmediateMaterialization.h"
+#include "rcc/CodeGen/GenericOpcodes.h"
+#include "rcc/CodeGen/GlobalISel/GISelChangeObserver.h"
+#include "rcc/CodeGen/GlobalISel/LegalizerHelper.h"
+#include "rcc/CodeGen/GlobalISel/Utils.h"
+#include "rcc/CodeGen/LowLevelType.h"
+#include "rcc/CodeGen/MachineBasicBlock.h"
+#include "rcc/CodeGen/MachineFrameInfo.h"
+#include "rcc/CodeGen/MachineFunction.h"
+#include "rcc/CodeGen/MachineIRBuilder.h"
+#include "rcc/CodeGen/MachineInstr.h"
+#include "rcc/CodeGen/MachineJumpTableInfo.h"
+#include "rcc/CodeGen/MachineMemOperand.h"
+#include "rcc/CodeGen/MachineOperand.h"
+#include "rcc/CodeGen/MachineRegisterInfo.h"
+#include "rcc/CodeGen/TargetDescription.h"
+#include "rcc/IR/DataLayout.h"
+#include "rcc/Target/RISCV/RISCVFunctionInfo.h"
+#include "rcc/Target/RISCV/RISCVSubtarget.h"
+#include "rcc/Target/RISCV/RISCVTarget.h"
+
+#include <algorithm>
+#include <array>
+#include <bit>
+#include <cstdint>
+#include <limits>
+#include <optional>
+#include <ranges>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace rcc::riscv {
+namespace {
+
+[[nodiscard]] std::optional<LowLevelType> registerType(const MachineFunction &Function,
+ Register Reg) {
+ if (!Reg.isVirtual())
+ return std::nullopt;
+ auto Info = Function.registerInfo().virtualRegisterInfo(Reg);
+ if (!Info)
+ return std::nullopt;
+ return Info->get().Type;
+}
+
+[[nodiscard]] std::optional<LowLevelType> registerType(const MachineFunction &Function,
+ const MachineOperand &Operand) {
+ const Register *Reg = Operand.getIf<Register>();
+ return Reg == nullptr ? std::nullopt : registerType(Function, *Reg);
+}
+
+[[nodiscard]] std::optional<LowLevelType> resultType(const MachineFunction &Function,
+ const MachineInstr &Instruction) {
+ const MachineOperand *Result = Instruction.operandAt(0);
+ return Result == nullptr ? std::nullopt : registerType(Function, *Result);
+}
+
+[[nodiscard]] Expected<Opcode> getRISCVWOpcode(GenericOpcode Generic) {
+ switch (Generic) {
+ case GenericOpcode::G_ASHR:
+ return instructionOpcode(InstructionOpcode::G_SRAW);
+ case GenericOpcode::G_LSHR:
+ return instructionOpcode(InstructionOpcode::G_SRLW);
+ case GenericOpcode::G_SHL:
+ return instructionOpcode(InstructionOpcode::G_SLLW);
+ case GenericOpcode::G_SDIV:
+ return instructionOpcode(InstructionOpcode::G_DIVW);
+ case GenericOpcode::G_UDIV:
+ return instructionOpcode(InstructionOpcode::G_DIVUW);
+ case GenericOpcode::G_UREM:
+ return instructionOpcode(InstructionOpcode::G_REMUW);
+ case GenericOpcode::G_ROTL:
+ return instructionOpcode(InstructionOpcode::G_ROLW);
+ case GenericOpcode::G_ROTR:
+ return instructionOpcode(InstructionOpcode::G_RORW);
+ case GenericOpcode::G_CTLZ:
+ return instructionOpcode(InstructionOpcode::G_CLZW);
+ case GenericOpcode::G_CTTZ:
+ return instructionOpcode(InstructionOpcode::G_CTZW);
+ case GenericOpcode::G_CTLS:
+ return instructionOpcode(InstructionOpcode::G_CLSW);
+ case GenericOpcode::G_FPTOSI:
+ return instructionOpcode(InstructionOpcode::G_FCVT_W_RV64);
+ case GenericOpcode::G_FPTOUI:
+ return instructionOpcode(InstructionOpcode::G_FCVT_WU_RV64);
+ default:
+ return makeUnexpected(ErrorCode::InvalidArgument,
+ "RISC-V word opcode mapping received an unexpected opcode");
+ }
+}
+
+[[nodiscard]] Expected<APInt>
+extendIntegerImmediate(const APInt &Value, std::uint32_t DestinationWidth, bool SignExtend) {
+ if (DestinationWidth == 0 || DestinationWidth > 64)
+ return makeUnexpected(ErrorCode::Unsupported,
+ "RISC-V constant extension requires a destination no wider than 64 bits");
+
+ std::uint64_t Raw = Value.words().empty() ? 0 : Value.words().front();
+ const std::uint32_t SourceWidth = Value.bitWidth();
+ if (SignExtend && Value.isNegative() && SourceWidth < 64)
+ Raw |= ~std::uint64_t{0} << SourceWidth;
+ if (DestinationWidth < 64)
+ Raw &= (std::uint64_t{1} << DestinationWidth) - 1U;
+ return APInt::create(DestinationWidth, std::span<const std::uint64_t>(&Raw, 1));
+}
+
+[[nodiscard]] Expected<void>
+foldConstantExtension(MachineInstr &Instruction, MachineFunction &Function, GenericOpcode Generic) {
+ const MachineOperand *SourceOperand = Instruction.operandAt(1);
+ const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+ const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>();
+ const Register *Destination =
+ DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+ if (Source == nullptr || Destination == nullptr || !Source->isVirtual() ||
+ !Destination->isVirtual()) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V constant extension requires virtual-register operands");
+ }
+ const Register SourceRegister = *Source;
+ const Register DestinationRegister = *Destination;
+ const MachineOperand *SourceDefinition = Function.registerInfo().getUniqueVRegDef(SourceRegister);
+ const MachineInstr *SourceDef =
+ SourceDefinition == nullptr ? nullptr : SourceDefinition->parent();
+ if (SourceDef == nullptr || SourceDef->opcode() != opcode(GenericOpcode::G_CONSTANT))
+ return makeUnexpected(ErrorCode::Unsupported,
+ "RISC-V constant extension requires a single constant definition");
+ const MachineOperand *ConstantOperand = SourceDef->operandAt(1);
+ const auto *Integer =
+ ConstantOperand == nullptr ? nullptr : ConstantOperand->getIf<IntegerImmediateOperand>();
+ if (Integer == nullptr)
+ return makeUnexpected(ErrorCode::Unsupported,
+ "RISC-V constant extension requires an integer constant");
+ const auto DestinationType = registerType(Function, *Destination);
+ if (!DestinationType)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V constant extension destination has no type");
+ const std::uint32_t DestinationWidth = DestinationType->scalarBitWidth();
+ const bool SignExtend = Generic != GenericOpcode::G_ZEXT;
+ auto Extended = extendIntegerImmediate(Integer->Value, DestinationWidth, SignExtend);
+ if (!Extended)
+ return std::unexpected(Extended.error());
+
+ if (Function.registerInfo().hasOneUse(SourceRegister)) {
+ MachineInstr *Def = const_cast<MachineInstr *>(SourceDef);
+ if (auto Replaced = Def->operandAt(0)->setRegister(DestinationRegister); !Replaced)
+ return std::unexpected(Replaced.error());
+ auto Erased = Def->eraseOperand(1);
+ if (!Erased)
+ return std::unexpected(Erased.error());
+ auto Added = Def->addOperand(MachineOperand::integerImmediate(std::move(*Extended)));
+ if (!Added)
+ return std::unexpected(Added.error());
+ return Instruction.eraseFromParent();
+ }
+
+ auto SetOpcode = Instruction.setOpcode(opcode(GenericOpcode::G_CONSTANT));
+ if (!SetOpcode)
+ return std::unexpected(SetOpcode.error());
+ while (Instruction.operandCount() > 1) {
+ auto Erased = Instruction.eraseOperand(1);
+ if (!Erased)
+ return std::unexpected(Erased.error());
+ }
+ auto Added = Instruction.addOperand(MachineOperand::integerImmediate(std::move(*Extended)));
+ if (!Added)
+ return std::unexpected(Added.error());
+
+ if (Function.registerInfo().useEmpty(SourceRegister)) {
+ MachineInstr *Def = const_cast<MachineInstr *>(SourceDef);
+ auto Removed = Def->eraseFromParent();
+ if (!Removed)
+ return std::unexpected(Removed.error());
+ // Keep the source register record as a tombstone-free historical vreg until
+ // the legalizer worklist transaction has completed.
+ }
+ return {};
+}
+
+[[nodiscard]] Expected<void> lowerConstantToPool(MachineInstr &Instruction,
+ MachineFunction &Function,
+ const TargetDescription &Target) {
+ const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+ const Register *Destination =
+ DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+ const MachineOperand *ImmediateOperand = Instruction.operandAt(1);
+ const auto *Immediate =
+ ImmediateOperand == nullptr ? nullptr : ImmediateOperand->getIf<IntegerImmediateOperand>();
+ if (Destination == nullptr || !Destination->isVirtual() || Immediate == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V constant-pool legalization has invalid operands");
+ const auto DestinationType = registerType(Function, *Destination);
+ if (!DestinationType || DestinationType->scalarBitWidth() != 64)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V constant-pool legalization requires a 64-bit destination");
+ auto Value = GenericConstantData::integer(Immediate->Value);
+ if (!Value)
+ return std::unexpected(Value.error());
+ auto Align = Alignment::create(8);
+ if (!Align)
+ return std::unexpected(Align.error());
+ MachineConstantPoolEntryDesc Description;
+ Description.Value = std::move(*Value);
+ Description.Size = 8;
+ Description.Align = *Align;
+ auto PoolIndex = Function.constantPool().addEntry(std::move(Description));
+ if (!PoolIndex)
+ return std::unexpected(PoolIndex.error());
+
+ auto PointerType = LowLevelType::makePointer(0, 64);
+ if (!PointerType)
+ return std::unexpected(PointerType.error());
+ auto Address = Function.registerInfo().createVirtualRegister(*PointerType);
+ if (!Address)
+ return std::unexpected(Address.error());
+ MachineIRBuilder Builder(Function, Target);
+ auto Point = Builder.setInsertPointBefore(Instruction);
+ if (!Point)
+ return std::unexpected(Point.error());
+ auto AddressInstruction = Builder.buildInstr(opcode(GenericOpcode::G_CONSTANT_POOL));
+ if (!AddressInstruction)
+ return std::unexpected(AddressInstruction.error());
+ auto AddressDef = AddressInstruction->addDef(*Address);
+ if (!AddressDef)
+ return std::unexpected(AddressDef.error());
+ auto AddressPool = AddressInstruction->addConstantPool(*PoolIndex);
+ if (!AddressPool)
+ return std::unexpected(AddressPool.error());
+ auto BuiltAddress = AddressInstruction->build();
+ if (!BuiltAddress)
+ return std::unexpected(BuiltAddress.error());
+
+ MachineMemOperandDesc MemoryDescription;
+ MemoryDescription.Accesses = {MemoryAccessFlag::Load};
+ MemoryDescription.Size = *MemorySize::fixed(8);
+ MemoryDescription.MemoryType = *DestinationType;
+ MemoryDescription.Align = *Align;
+ MemoryDescription.Base = MemoryBase::constantPoolPseudo();
+ auto Memory = Function.createMemOperand(std::move(MemoryDescription));
+ if (!Memory)
+ return std::unexpected(Memory.error());
+ auto SetOpcode = Instruction.setOpcode(opcode(GenericOpcode::G_LOAD));
+ if (!SetOpcode)
+ return std::unexpected(SetOpcode.error());
+ while (Instruction.operandCount() > 1) {
+ auto Erased = Instruction.eraseOperand(1);
+ if (!Erased)
+ return std::unexpected(Erased.error());
+ }
+ auto AddressUse =
+ MachineOperand::registerOperand(*Address, RegisterOperandFlags{RegisterOperandFlag::Use});
+ if (!AddressUse)
+ return std::unexpected(AddressUse.error());
+ auto AddedAddress = Instruction.addOperand(std::move(*AddressUse));
+ if (!AddedAddress)
+ return std::unexpected(AddedAddress.error());
+ auto AddedMemory = Instruction.addMemOperand(**Memory);
+ if (!AddedMemory)
+ return std::unexpected(AddedMemory.error());
+ return {};
+}
+
+[[nodiscard]] Expected<std::int64_t> signedImmediate(const MachineOperand &Operand) {
+ if (const std::int64_t *Value = Operand.getIf<std::int64_t>())
+ return *Value;
+ const auto *Integer = Operand.getIf<IntegerImmediateOperand>();
+ if (Integer == nullptr || Integer->Value.bitWidth() == 0 || Integer->Value.bitWidth() > 64)
+ return makeUnexpected(ErrorCode::Unsupported,
+ "RISC-V constant selection requires an immediate no wider than 64 bits");
+ std::uint64_t Raw = Integer->Value.words().empty() ? 0 : Integer->Value.words().front();
+ const std::uint32_t Width = Integer->Value.bitWidth();
+ if (Width < 64 && (Raw & (std::uint64_t{1} << (Width - 1U))) != 0)
+ Raw |= ~std::uint64_t{0} << Width;
+ return std::bit_cast<std::int64_t>(Raw);
+}
+
+} // namespace
+
+RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &Subtarget) : Subtarget(Subtarget) {
+ const LowLevelType S1 = *LowLevelType::makeCategoryNeutralScalar(1);
+ const LowLevelType S8 = *LowLevelType::makeCategoryNeutralScalar(8);
+ const LowLevelType S16 = *LowLevelType::makeCategoryNeutralScalar(16);
+ const LowLevelType S32 = *LowLevelType::makeCategoryNeutralScalar(32);
+ const LowLevelType S64 = *LowLevelType::makeCategoryNeutralScalar(64);
+ const LowLevelType S128 = *LowLevelType::makeCategoryNeutralScalar(128);
+ const LowLevelType P0 = *LowLevelType::makePointer(0, 64);
+ const bool HasM = Subtarget.features().has(RISCVExtension::M);
+ const bool HasA = Subtarget.features().has(RISCVExtension::A);
+ const bool HasF = Subtarget.features().has(RISCVExtension::F);
+ const bool HasD = Subtarget.features().has(RISCVExtension::D);
+ const bool HasZbb = Subtarget.features().has(RISCVExtension::Zbb);
+
+ getActionDefinitionsBuilder({opcode(GenericOpcode::G_ASSERT_SEXT),
+ opcode(GenericOpcode::G_ASSERT_ZEXT),
+ opcode(GenericOpcode::G_ASSERT_ALIGN)})
+ .alwaysLegal();
+
+ getActionDefinitionsBuilder({opcode(GenericOpcode::G_ADD), opcode(GenericOpcode::G_SUB)})
+ .legalFor({S64})
+ .customFor({S32})
+ .widenScalarToNextPow2(0)
+ .clampScalar(0, S64, S64);
+
+ getActionDefinitionsBuilder(
+ {opcode(GenericOpcode::G_AND), opcode(GenericOpcode::G_OR), opcode(GenericOpcode::G_XOR)})
+ .legalFor({S64})
+ .widenScalarToNextPow2(0)
+ .clampScalar(0, S64, S64);
+
+ getActionDefinitionsBuilder({opcode(GenericOpcode::G_UADDE), opcode(GenericOpcode::G_UADDO),
+ opcode(GenericOpcode::G_USUBE), opcode(GenericOpcode::G_USUBO),
+ opcode(GenericOpcode::G_READ_REGISTER),
+ opcode(GenericOpcode::G_WRITE_REGISTER)})
+ .lower();
+ getActionDefinitionsBuilder({opcode(GenericOpcode::G_SADDE), opcode(GenericOpcode::G_SADDO),
+ opcode(GenericOpcode::G_SSUBE), opcode(GenericOpcode::G_SSUBO)})
+ .minScalar(0, S64)
+ .lower();
+ getActionDefinitionsBuilder({opcode(GenericOpcode::G_UADDSAT), opcode(GenericOpcode::G_SADDSAT),
+ opcode(GenericOpcode::G_USUBSAT), opcode(GenericOpcode::G_SSUBSAT),
+ opcode(GenericOpcode::G_SSHLSAT), opcode(GenericOpcode::G_USHLSAT)})
+ .lower();
+
+ getActionDefinitionsBuilder(
+ {opcode(GenericOpcode::G_SHL), opcode(GenericOpcode::G_ASHR), opcode(GenericOpcode::G_LSHR)})
+ .legalFor({{S64, S64}})
+ .customFor({{S32, S32}})
+ .widenScalarToNextPow2(0)
+ .clampScalar(1, S64, S64)
+ .clampScalar(0, S64, S64);
+
+ getActionDefinitionsBuilder({opcode(GenericOpcode::G_ZEXT), opcode(GenericOpcode::G_SEXT),
+ opcode(GenericOpcode::G_ANYEXT)})
+ .legalFor({{S32, S16}, {S64, S16}, {S64, S32}})
+ .maxScalar(0, S64);
+ getActionDefinitionsBuilder(opcode(GenericOpcode::G_TRUNC)).alwaysLegal();
+ getActionDefinitionsBuilder(opcode(GenericOpcode::G_SEXT_INREG))
+ .customFor({S64})
+ .clampScalar(0, S64, S64)
+ .lower();
+
+ for (const GenericOpcode Generic :
+ {GenericOpcode::G_MERGE_VALUES, GenericOpcode::G_UNMERGE_VALUES}) {
+ const std::size_t BigTypeIndex = Generic == GenericOpcode::G_MERGE_VALUES ? 0 : 1;
+ const std::size_t LittleTypeIndex = Generic == GenericOpcode::G_MERGE_VALUES ? 1 : 0;
+ getActionDefinitionsBuilder(opcode(Generic))
+ .widenScalarToNextPow2(LittleTypeIndex, 64)
+ .widenScalarToNextPow2(BigTypeIndex, 64)
+ .clampScalar(LittleTypeIndex, S64, S64)
+ .clampScalar(BigTypeIndex, S64, S64);
+ }
+ getActionDefinitionsBuilder({opcode(GenericOpcode::G_FSHL), opcode(GenericOpcode::G_FSHR)})
+ .lower();
+ getActionDefinitionsBuilder({opcode(GenericOpcode::G_ROTR), opcode(GenericOpcode::G_ROTL)})
+ .legalFor(HasZbb, {{S64, S64}})
+ .customFor(HasZbb, {{S32, S32}})
+ .lower();
+ getActionDefinitionsBuilder(opcode(GenericOpcode::G_BITREVERSE)).maxScalar(0, S64).lower();
+ auto &ByteSwapActions = getActionDefinitionsBuilder(opcode(GenericOpcode::G_BSWAP));
+ if (HasZbb)
+ ByteSwapActions.legalFor({S64}).clampScalar(0, S64, S64);
+ else
+ ByteSwapActions.maxScalar(0, S64).lower();
+
+ auto &CountZerosActions =
+ getActionDefinitionsBuilder({opcode(GenericOpcode::G_CTLZ), opcode(GenericOpcode::G_CTTZ)});
+ auto &CountZerosPoisonActions = getActionDefinitionsBuilder(
+ {opcode(GenericOpcode::G_CTLZ_ZERO_POISON), opcode(GenericOpcode::G_CTTZ_ZERO_POISON)});
+ if (HasZbb) {
+ CountZerosActions.legalFor({{S64, S64}})
+ .customFor({{S32, S32}})
+ .clampScalar(0, S32, S64)
+ .widenScalarToNextPow2(0)
+ .scalarSameSizeAs(1, 0);
+ } else {
+ CountZerosActions.maxScalar(0, S64).scalarSameSizeAs(1, 0).lower();
+ CountZerosPoisonActions.maxScalar(0, S64).scalarSameSizeAs(1, 0);
+ }
+ CountZerosPoisonActions.lower();
+
+ getActionDefinitionsBuilder(opcode(GenericOpcode::G_CTLS))
+ .maxScalar(0, S64)
+ .scalarSameSizeAs(1, 0)
+ .lower();
+
+ auto &CountPopulationActions = getActionDefinitionsBuilder(opcode(GenericOpcode::G_CTPOP));
+ if (HasZbb) {
+ CountPopulationActions.legalFor({{S64, S64}}).clampScalar(0, S64, S64).scalarSameSizeAs(1, 0);
+ } else {
+ CountPopulationActions.widenScalarToNextPow2(0, 8)
+ .clampScalar(0, S8, S64)
+ .scalarSameSizeAs(1, 0)
+ .lower();
+ }
+
+ getActionDefinitionsBuilder(opcode(GenericOpcode::G_CONSTANT))
+ .legalFor({P0})
+ .customFor({S64})
+ .widenScalarToNextPow2(0)
+ .clampScalar(0, S64, S64);
+ getActionDefinitionsBuilder(opcode(GenericOpcode::G_FREEZE))
+ .legalFor({S16, S32, S64, P0})
+ .widenScalarToNextPow2(0)
+ .clampScalar(0, S16, S64);
+ getActionDefinitionsBuilder(
+ {opcode(GenericOpcode::G_IMPLICIT_DEF), opcode(GenericOpcode::G_CONSTANT_FOLD_BARRIER)})
+ .legalFor({S32, S64, P0})
+ .widenScalarToNextPow2(0)
+ .clampScalar(0, S32, S64);
+
+ getActionDefinitionsBuilder(opcode(GenericOpcode::G_ICMP))
+ .legalFor({{S64, S64}, {S64, P0}})
+ .widenScalarToNextPow2(1, 8)
+ .clampScalar(1, S64, S64)
+ .clampScalar(0, S64, S64);
+ getActionDefinitionsBuilder(opcode(GenericOpcode::G_SELECT))
+ .legalFor({{S32, S64}, {S64, S64}, {P0, S64}})
+ .widenScalarToNextPow2(0)
+ .clampScalar(0, S32, S64)
+ .clampScalar(1, S64, S64);
+
+ auto &LoadRules = getActionDefinitionsBuilder(opcode(GenericOpcode::G_LOAD));
+ auto &StoreRules = getActionDefinitionsBuilder(opcode(GenericOpcode::G_STORE));
+ auto &ExtLoadRules = getActionDefinitionsBuilder(
+ {opcode(GenericOpcode::G_SEXTLOAD), opcode(GenericOpcode::G_ZEXTLOAD)});
+ const auto GetScalarMemAlign = [&Subtarget](std::uint64_t Size) {
+ return Subtarget.features().has(RISCVSubtargetOption::UnalignedScalarMemory) ? 8 : Size;
+ };
+ LoadRules
+ .legalForTypesWithMemDesc({{S16, P0, S8, GetScalarMemAlign(8)},
+ {S32, P0, S8, GetScalarMemAlign(8)},
+ {S16, P0, S16, GetScalarMemAlign(16)},
+ {S32, P0, S16, GetScalarMemAlign(16)},
+ {S32, P0, S32, GetScalarMemAlign(32)},
+ {S64, P0, S8, GetScalarMemAlign(8)},
+ {S64, P0, S16, GetScalarMemAlign(16)},
+ {S64, P0, S32, GetScalarMemAlign(32)},
+ {S64, P0, S64, GetScalarMemAlign(64)},
+ {P0, P0, S64, GetScalarMemAlign(64)}})
+ .widenScalarToNextPow2(0, 8)
+ .lowerIfMemSizeNotByteSizePow2()
+ .clampScalar(0, S16, S64)
+ .lower();
+ StoreRules
+ .legalForTypesWithMemDesc({{S16, P0, S8, GetScalarMemAlign(8)},
+ {S32, P0, S8, GetScalarMemAlign(8)},
+ {S16, P0, S16, GetScalarMemAlign(16)},
+ {S32, P0, S16, GetScalarMemAlign(16)},
+ {S32, P0, S32, GetScalarMemAlign(32)},
+ {S64, P0, S8, GetScalarMemAlign(8)},
+ {S64, P0, S16, GetScalarMemAlign(16)},
+ {S64, P0, S32, GetScalarMemAlign(32)},
+ {S64, P0, S64, GetScalarMemAlign(64)},
+ {P0, P0, S64, GetScalarMemAlign(64)}})
+ .clampScalar(0, S16, S64)
+ .lowerIfMemSizeNotByteSizePow2()
+ .lower();
+ ExtLoadRules
+ .legalForTypesWithMemDesc({{S64, P0, S8, GetScalarMemAlign(8)},
+ {S64, P0, S16, GetScalarMemAlign(16)},
+ {S64, P0, S32, GetScalarMemAlign(32)}})
+ .widenScalarToNextPow2(0)
+ .clampScalar(0, S64, S64)
+ .lower();
+ getActionDefinitionsBuilder(opcode(GenericOpcode::G_FPEXTLOAD))
+ .legalForTypesWithMemDesc({{S64, P0, S32, GetScalarMemAlign(32)}})
+ .lower();
+ getActionDefinitionsBuilder(opcode(GenericOpcode::G_FPTRUNCSTORE))
+ .legalForTypesWithMemDesc({{S64, P0, S32, GetScalarMemAlign(32)}})
+ .lower();
+
+ getActionDefinitionsBuilder({opcode(GenericOpcode::G_PTR_ADD), opcode(GenericOpcode::G_PTRMASK)})
+ .legalFor({{P0, S64}});
+ getActionDefinitionsBuilder(opcode(GenericOpcode::G_PTRTOINT))
+ .legalFor({{S64, P0}})
+ .clampScalar(0, S64, S64);
+ getActionDefinitionsBuilder(opcode(GenericOpcode::G_INTTOPTR))
+ .legalFor({{P0, S64}})
+ .clampScalar(1, S64, S64);
+ getActionDefinitionsBuilder(opcode(GenericOpcode::G_BR)).alwaysLegal();
+ getActionDefinitionsBuilder(opcode(GenericOpcode::G_BRCOND)).legalFor({S64}).minScalar(0, S64);
+ getActionDefinitionsBuilder(opcode(GenericOpcode::G_BRJT)).customFor({{P0, S64}});
+ getActionDefinitionsBuilder(opcode(GenericOpcode::G_BRINDIRECT)).legalFor({P0});
+ getActionDefinitionsBuilder(opcode(GenericOpcode::G_PHI))
+ .legalFor({P0, S32, S64})
+ .widenScalarToNextPow2(0)
+ .clampScalar(0, S32, S64);
+ getActionDefinitionsBuilder(
+ {opcode(GenericOpcode::G_GLOBAL_VALUE), opcode(GenericOpcode::G_JUMP_TABLE),
+ opcode(GenericOpcode::G_CONSTANT_POOL), opcode(GenericOpcode::G_FRAME_INDEX)})
+ .legalFor({P0});
+
+ if (HasM) {
+ getActionDefinitionsBuilder(opcode(GenericOpcode::G_MUL))
+ .legalFor({S64})
+ .widenScalarToNextPow2(0)
+ .clampScalar(0, S64, S64);
+ getActionDefinitionsBuilder({opcode(GenericOpcode::G_SMULH), opcode(GenericOpcode::G_UMULH)})
+ .legalFor({S64})
+ .lower();
+ getActionDefinitionsBuilder({opcode(GenericOpcode::G_SMULO), opcode(GenericOpcode::G_UMULO)})
+ .minScalar(0, S64)
+ .lower();
+ } else {
+ getActionDefinitionsBuilder(opcode(GenericOpcode::G_MUL))
+ .libcallFor({S64, S128})
+ .widenScalarToNextPow2(0)
+ .clampScalar(0, S64, S128);
+ getActionDefinitionsBuilder({opcode(GenericOpcode::G_SMULH), opcode(GenericOpcode::G_UMULH)})
+ .lowerFor({S64});
+ getActionDefinitionsBuilder({opcode(GenericOpcode::G_SMULO), opcode(GenericOpcode::G_UMULO)})
+ .minScalar(0, S64)
+ // Widen s64 to s128 so one libcall provides both the low result and
+ // the high half needed for the overflow check.
+ .widenScalarIf(
+ [S64](const LegalizationQuery &Query) {
+ return !Query.Types.empty() && Query.Types[0] == S64;
+ },
+ [S128](const LegalizationQuery &) { return std::pair{std::size_t{0}, S128}; })
+ .lower();
+ }
+ if (HasM) {
+ getActionDefinitionsBuilder({opcode(GenericOpcode::G_SDIV), opcode(GenericOpcode::G_UDIV),
+ opcode(GenericOpcode::G_UREM)})
+ .legalFor({S64})
+ .customFor({S32})
+ .libcallFor({S128})
+ .clampScalar(0, S32, S128)
+ .widenScalarToNextPow2(0);
+ getActionDefinitionsBuilder(opcode(GenericOpcode::G_SREM))
+ .legalFor({S64})
+ .libcallFor({S128})
+ .clampScalar(0, S64, S128)
+ .widenScalarToNextPow2(0);
+ } else {
+ getActionDefinitionsBuilder({opcode(GenericOpcode::G_SDIV), opcode(GenericOpcode::G_UDIV),
+ opcode(GenericOpcode::G_SREM), opcode(GenericOpcode::G_UREM)})
+ .libcallFor({S64, S128})
+ .clampScalar(0, S64, S128)
+ .widenScalarToNextPow2(0);
+ }
+ getActionDefinitionsBuilder({opcode(GenericOpcode::G_SDIVREM), opcode(GenericOpcode::G_UDIVREM)})
+ .lower();
+ getActionDefinitionsBuilder(opcode(GenericOpcode::G_ABS)).lower();
+ getActionDefinitionsBuilder({opcode(GenericOpcode::G_ABDS), opcode(GenericOpcode::G_ABDU)})
+ .minScalar(HasZbb, 0, S64)
+ .lower();
+ getActionDefinitionsBuilder({opcode(GenericOpcode::G_UMAX), opcode(GenericOpcode::G_UMIN),
+ opcode(GenericOpcode::G_SMAX), opcode(GenericOpcode::G_SMIN)})
+ .legalFor(HasZbb, {S64})
+ .minScalar(HasZbb, 0, S64)
+ .lower();
+ getActionDefinitionsBuilder({opcode(GenericOpcode::G_SCMP), opcode(GenericOpcode::G_UCMP)})
+ .lower();
+ getActionDefinitionsBuilder({opcode(GenericOpcode::G_MEMCPY), opcode(GenericOpcode::G_MEMMOVE),
+ opcode(GenericOpcode::G_MEMSET)})
+ .libcall();
+ getActionDefinitionsBuilder(opcode(GenericOpcode::G_MEMCPY_INLINE)).lower();
+ getActionDefinitionsBuilder({opcode(GenericOpcode::G_DYN_STACKALLOC),
+ opcode(GenericOpcode::G_STACKSAVE),
+ opcode(GenericOpcode::G_STACKRESTORE)})
+ .lower();
+
+ getActionDefinitionsBuilder(
+ {opcode(GenericOpcode::G_FADD), opcode(GenericOpcode::G_FSUB), opcode(GenericOpcode::G_FMUL),
+ opcode(GenericOpcode::G_FDIV), opcode(GenericOpcode::G_FMA), opcode(GenericOpcode::G_FSQRT),
+ opcode(GenericOpcode::G_FMAXNUM), opcode(GenericOpcode::G_FMINNUM),
+ opcode(GenericOpcode::G_FMAXIMUMNUM), opcode(GenericOpcode::G_FMINIMUMNUM)})
+ .legalFor(HasF, {S32})
+ .legalFor(HasD, {S64})
+ .libcallFor({S32, S64})
+ .libcallFor({S128});
+ getActionDefinitionsBuilder({opcode(GenericOpcode::G_FNEG), opcode(GenericOpcode::G_FABS)})
+ .legalFor(HasF, {S32})
+ .legalFor(HasD, {S64})
+ .lowerFor({S32, S64, S128});
+ getActionDefinitionsBuilder(opcode(GenericOpcode::G_FREM))
+ .libcallFor({S32, S64, S128})
+ .minScalar(0, S32)
+ .scalarize(0);
+ getActionDefinitionsBuilder(opcode(GenericOpcode::G_FCOPYSIGN))
+ .legalFor(HasF, {{S32, S32}})
+ .legalFor(HasD, {{S64, S64}, {S32, S64}, {S64, S32}})
+ .lower();
+ getActionDefinitionsBuilder(opcode(GenericOpcode::G_FPTRUNC))
+ .legalFor(HasD, {{S32, S64}})
+ .libcallFor({{S32, S64}, {S32, S128}, {S64, S128}});
+ getActionDefinitionsBuilder(opcode(GenericOpcode::G_FPEXT))
+ .legalFor(HasD, {{S64, S32}})
+ .libcallFor({{S64, S32}, {S128, S32}, {S128, S64}});
+ getActionDefinitionsBuilder(opcode(GenericOpcode::G_FCMP))
+ .legalFor(HasF, {{S64, S32}})
+ .legalFor(HasD, {{S64, S64}})
+ .clampScalar(0, S64, S64)
+ .libcallFor({{S64, S32}, {S64, S64}, {S64, S128}});
+ getActionDefinitionsBuilder(opcode(GenericOpcode::G_IS_FPCLASS))
+ .customFor(HasF, {{S1, S32}})
+ .customFor(HasD, {{S1, S64}})
+ .lower();
+ getActionDefinitionsBuilder(opcode(GenericOpcode::G_FCONSTANT))
+ .legalFor(HasF, {S32})
+ .legalFor(HasD, {S64})
+ .customFor({S32, S64})
+ .lowerFor({S64, S128});
+ getActionDefinitionsBuilder({opcode(GenericOpcode::G_FPTOSI), opcode(GenericOpcode::G_FPTOUI)})
+ .legalFor(HasF, {{S64, S32}})
+ .legalFor(HasD, {{S64, S64}})
+ .customFor(HasF, {{S32, S32}})
+ .customFor(HasD, {{S32, S64}})
+ .widenScalarToNextPow2(0)
+ .minScalar(0, S32)
+ .libcallFor({{S32, S32},
+ {S64, S32},
+ {S32, S64},
+ {S64, S64},
+ {S32, S128},
+ {S64, S128},
+ {S128, S32},
+ {S128, S64},
+ {S128, S128}});
+ getActionDefinitionsBuilder({opcode(GenericOpcode::G_SITOFP), opcode(GenericOpcode::G_UITOFP)})
+ .legalFor(HasF, {{S32, S64}})
+ .legalFor(HasD, {{S64, S64}})
+ .widenScalarToNextPow2(1)
+ .widenScalarIf(
+ [HasF, HasD](const LegalizationQuery &Query) {
+ if (Query.Types.size() < 2 || !Query.Types[0].isCategoryNeutralScalar() ||
+ !Query.Types[1].isCategoryNeutralScalar() ||
+ Query.Types[1].scalarBitWidth() >= 64) {
+ return false;
+ }
+ return (HasF && Query.Types[0].scalarBitWidth() == 32) ||
+ (HasD && Query.Types[0].scalarBitWidth() == 64);
+ },
+ [S64](const LegalizationQuery &) { return std::pair{std::size_t{1}, S64}; })
+ .minScalar(1, S32)
+ .libcallFor({{S32, S32},
+ {S64, S32},
+ {S32, S64},
+ {S64, S64},
+ {S128, S32},
+ {S128, S64},
+ {S32, S128},
+ {S64, S128},
+ {S128, S128}});
+ getActionDefinitionsBuilder({opcode(GenericOpcode::G_FCEIL), opcode(GenericOpcode::G_FFLOOR),
+ opcode(GenericOpcode::G_FRINT), opcode(GenericOpcode::G_FNEARBYINT),
+ opcode(GenericOpcode::G_INTRINSIC_TRUNC),
+ opcode(GenericOpcode::G_INTRINSIC_ROUND),
+ opcode(GenericOpcode::G_INTRINSIC_ROUNDEVEN)})
+ .libcallFor({S32, S64, S128});
+ getActionDefinitionsBuilder({opcode(GenericOpcode::G_FCOS), opcode(GenericOpcode::G_FSIN),
+ opcode(GenericOpcode::G_FTAN), opcode(GenericOpcode::G_FPOW),
+ opcode(GenericOpcode::G_FLOG), opcode(GenericOpcode::G_FLOG2),
+ opcode(GenericOpcode::G_FLOG10), opcode(GenericOpcode::G_FEXP),
+ opcode(GenericOpcode::G_FEXP2), opcode(GenericOpcode::G_FEXP10),
+ opcode(GenericOpcode::G_FACOS), opcode(GenericOpcode::G_FASIN),
+ opcode(GenericOpcode::G_FATAN), opcode(GenericOpcode::G_FATAN2),
+ opcode(GenericOpcode::G_FCOSH), opcode(GenericOpcode::G_FSINH),
+ opcode(GenericOpcode::G_FTANH), opcode(GenericOpcode::G_FMODF)})
+ .libcallFor({S32, S64, S128});
+ getActionDefinitionsBuilder({opcode(GenericOpcode::G_FPOWI), opcode(GenericOpcode::G_FLDEXP)})
+ .libcallFor({{S32, S32}, {S64, S32}, {S128, S32}});
+ getActionDefinitionsBuilder(opcode(GenericOpcode::G_FCANONICALIZE))
+ .legalFor(HasF, {S32})
+ .legalFor(HasD, {S64});
+
+ getActionDefinitionsBuilder(opcode(GenericOpcode::G_VASTART)).customFor({P0});
+ getActionDefinitionsBuilder(opcode(GenericOpcode::G_VAARG)).clampScalar(0, S64, S64).lower();
+ getActionDefinitionsBuilder(opcode(GenericOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS))
+ .lowerIf([S8, S16, S32, S64, P0](const LegalizationQuery &Query) {
+ return Query.Types.size() >= 3 &&
+ (Query.Types[0] == S8 || Query.Types[0] == S16 || Query.Types[0] == S32 ||
+ Query.Types[0] == S64) &&
+ Query.Types[2] == P0;
+ });
+ getActionDefinitionsBuilder(opcode(GenericOpcode::G_ATOMIC_CMPXCHG))
+ .legalFor(HasA, {{S64, P0}})
+ .libcallFor(!HasA, {{S8, P0}, {S16, P0}, {S32, P0}, {S64, P0}})
+ .clampScalar(0, S64, S64);
+ getActionDefinitionsBuilder(opcode(GenericOpcode::G_ATOMICRMW_ADD))
+ .legalFor(HasA, {{S64, P0}})
+ .libcallFor(!HasA, {{S8, P0}, {S16, P0}, {S32, P0}, {S64, P0}})
+ .clampScalar(0, S64, S64);
+ getActionDefinitionsBuilder(opcode(GenericOpcode::G_ATOMICRMW_SUB))
+ .libcallFor(!HasA, {{S8, P0}, {S16, P0}, {S32, P0}, {S64, P0}})
+ .clampScalar(0, S64, S64)
+ .lower();
+ getActionDefinitionsBuilder(
+ {opcode(GenericOpcode::G_ATOMICRMW_XCHG), opcode(GenericOpcode::G_ATOMICRMW_AND),
+ opcode(GenericOpcode::G_ATOMICRMW_OR), opcode(GenericOpcode::G_ATOMICRMW_XOR),
+ opcode(GenericOpcode::G_ATOMICRMW_MAX), opcode(GenericOpcode::G_ATOMICRMW_MIN),
+ opcode(GenericOpcode::G_ATOMICRMW_UMAX), opcode(GenericOpcode::G_ATOMICRMW_UMIN)})
+ .legalFor(HasA, {{S64, P0}})
+ .libcallFor(!HasA, {{S8, P0}, {S16, P0}, {S32, P0}, {S64, P0}})
+ .clampScalar(0, S64, S64);
+ getActionDefinitionsBuilder(
+ {opcode(GenericOpcode::G_INTRINSIC), opcode(GenericOpcode::G_INTRINSIC_W_SIDE_EFFECTS)})
+ .alwaysLegal();
+ getActionDefinitionsBuilder(opcode(GenericOpcode::G_FENCE)).alwaysLegal();
+ getActionDefinitionsBuilder({opcode(GenericOpcode::G_TRAP), opcode(GenericOpcode::G_DEBUGTRAP),
+ opcode(GenericOpcode::G_UBSANTRAP)})
+ .alwaysLegal();
+}
+
+Expected<void> RISCVLegalizerInfo::legalizeBRJT(MachineInstr &Instruction,
+ MachineIRBuilder &MIRBuilder) const {
+ if (Instruction.operandCount() != 3)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V G_BRJT requires table pointer, table index, and index operands");
+
+ const MachineOperand *TablePointerOperand = Instruction.operandAt(0);
+ const MachineOperand *TableIndexOperand = Instruction.operandAt(1);
+ const MachineOperand *IndexOperand = Instruction.operandAt(2);
+ const Register *TablePointer =
+ TablePointerOperand == nullptr ? nullptr : TablePointerOperand->getIf<Register>();
+ const JumpTableOperand *TableIndex =
+ TableIndexOperand == nullptr ? nullptr : TableIndexOperand->getIf<JumpTableOperand>();
+ const Register *Index = IndexOperand == nullptr ? nullptr : IndexOperand->getIf<Register>();
+ if (TablePointer == nullptr || TableIndex == nullptr || Index == nullptr ||
+ !TablePointer->isVirtual() || !Index->isVirtual()) {
+ return makeUnexpected(
+ ErrorCode::VerificationError,
+ "RISC-V G_BRJT requires virtual-register table pointer and index operands");
+ }
+
+ MachineFunction &Function = MIRBuilder.function();
+ const MachineJumpTableEntry *TableEntry = Function.jumpTableInfo().table(TableIndex->Index);
+ const MachineJumpTableDesc *Table = TableEntry == nullptr ? nullptr : TableEntry->description();
+ if (Table == nullptr) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V G_BRJT references an undefined jump table");
+ }
+ if (!std::has_single_bit(Table->EntrySize)) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V G_BRJT requires a power-of-two jump table entry size");
+ }
+
+ auto PointerType = Function.registerInfo().getType(*TablePointer);
+ if (!PointerType)
+ return std::unexpected(PointerType.error());
+ auto IndexType = Function.registerInfo().getType(*Index);
+ if (!IndexType)
+ return std::unexpected(IndexType.error());
+ if (!PointerType->isPointer() || !IndexType->isCategoryNeutralScalar()) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V G_BRJT requires scalar pointer and integer index types");
+ }
+ const std::optional<std::uint32_t> AddressSpace = PointerType->addressSpace();
+ if (!AddressSpace) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V G_BRJT table pointer has no address space");
+ }
+
+ auto EntryType = LowLevelType::makeCategoryNeutralScalar(
+ static_cast<std::uint64_t>(Table->EntrySize) * std::uint64_t{8});
+ if (!EntryType)
+ return std::unexpected(EntryType.error());
+ auto EntryMemorySize = MemorySize::fixed(Table->EntrySize);
+ if (!EntryMemorySize)
+ return std::unexpected(EntryMemorySize.error());
+
+ MachineMemOperandDesc LoadDescription;
+ LoadDescription.Accesses = {MemoryAccessFlag::Load};
+ LoadDescription.Size = *EntryMemorySize;
+ LoadDescription.MemoryType = *EntryType;
+ LoadDescription.Align = Table->Align;
+ LoadDescription.AddressSpace = *AddressSpace;
+ LoadDescription.Base = MemoryBase::jumpTablePseudo();
+ auto LoadMemory = Function.createMemOperand(std::move(LoadDescription));
+ if (!LoadMemory)
+ return std::unexpected(LoadMemory.error());
+
+ auto ShiftAmount = Function.registerInfo().createVirtualRegister(*IndexType);
+ if (!ShiftAmount)
+ return std::unexpected(ShiftAmount.error());
+ if (auto Built = MIRBuilder.buildConstant(
+ *ShiftAmount,
+ static_cast<std::int64_t>(std::countr_zero(static_cast<unsigned int>(Table->EntrySize))));
+ !Built) {
+ return std::unexpected(Built.error());
+ }
+
+ auto ScaledIndex = Function.registerInfo().createVirtualRegister(*IndexType);
+ if (!ScaledIndex)
+ return std::unexpected(ScaledIndex.error());
+ if (auto Built = MIRBuilder.buildShl(*ScaledIndex, *Index, *ShiftAmount); !Built)
+ return std::unexpected(Built.error());
+
+ auto EntryAddress = Function.registerInfo().createVirtualRegister(*PointerType);
+ if (!EntryAddress)
+ return std::unexpected(EntryAddress.error());
+ auto AddressBuilder = MIRBuilder.buildInstr(opcode(GenericOpcode::G_PTR_ADD));
+ if (!AddressBuilder)
+ return std::unexpected(AddressBuilder.error());
+ if (auto Added = AddressBuilder->addDef(*EntryAddress); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = AddressBuilder->addUse(*TablePointer); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = AddressBuilder->addUse(*ScaledIndex); !Added)
+ return std::unexpected(Added.error());
+ if (auto Built = AddressBuilder->build(); !Built)
+ return std::unexpected(Built.error());
+
+ Register Target;
+ switch (Table->Encoding) {
+ case JumpTableEntryEncoding::Custom32: {
+ auto LoadedOffset = Function.registerInfo().createVirtualRegister(*IndexType);
+ if (!LoadedOffset)
+ return std::unexpected(LoadedOffset.error());
+ auto LoadBuilder = MIRBuilder.buildInstr(opcode(GenericOpcode::G_SEXTLOAD));
+ if (!LoadBuilder)
+ return std::unexpected(LoadBuilder.error());
+ if (auto Added = LoadBuilder->addDef(*LoadedOffset); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = LoadBuilder->addUse(*EntryAddress); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = LoadBuilder->addMemOperand(**LoadMemory); !Added)
+ return std::unexpected(Added.error());
+ if (auto Built = LoadBuilder->build(); !Built)
+ return std::unexpected(Built.error());
+
+ auto TargetPointer = Function.registerInfo().createVirtualRegister(*PointerType);
+ if (!TargetPointer)
+ return std::unexpected(TargetPointer.error());
+ if (auto Built = MIRBuilder.buildIntToPtr(*TargetPointer, *LoadedOffset); !Built)
+ return std::unexpected(Built.error());
+ Target = *TargetPointer;
+ break;
+ }
+ case JumpTableEntryEncoding::LabelDifference32: {
+ auto LoadedOffset = Function.registerInfo().createVirtualRegister(*IndexType);
+ if (!LoadedOffset)
+ return std::unexpected(LoadedOffset.error());
+ auto LoadBuilder = MIRBuilder.buildInstr(opcode(GenericOpcode::G_SEXTLOAD));
+ if (!LoadBuilder)
+ return std::unexpected(LoadBuilder.error());
+ if (auto Added = LoadBuilder->addDef(*LoadedOffset); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = LoadBuilder->addUse(*EntryAddress); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = LoadBuilder->addMemOperand(**LoadMemory); !Added)
+ return std::unexpected(Added.error());
+ if (auto Built = LoadBuilder->build(); !Built)
+ return std::unexpected(Built.error());
+
+ auto TargetPointer = Function.registerInfo().createVirtualRegister(*PointerType);
+ if (!TargetPointer)
+ return std::unexpected(TargetPointer.error());
+ auto TargetBuilder = MIRBuilder.buildInstr(opcode(GenericOpcode::G_PTR_ADD));
+ if (!TargetBuilder)
+ return std::unexpected(TargetBuilder.error());
+ if (auto Added = TargetBuilder->addDef(*TargetPointer); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = TargetBuilder->addUse(*TablePointer); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = TargetBuilder->addUse(*LoadedOffset); !Added)
+ return std::unexpected(Added.error());
+ if (auto Built = TargetBuilder->build(); !Built)
+ return std::unexpected(Built.error());
+ Target = *TargetPointer;
+ break;
+ }
+ case JumpTableEntryEncoding::BlockAddress: {
+ auto TargetPointer = Function.registerInfo().createVirtualRegister(*PointerType);
+ if (!TargetPointer)
+ return std::unexpected(TargetPointer.error());
+ auto LoadBuilder = MIRBuilder.buildInstr(opcode(GenericOpcode::G_LOAD));
+ if (!LoadBuilder)
+ return std::unexpected(LoadBuilder.error());
+ if (auto Added = LoadBuilder->addDef(*TargetPointer); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = LoadBuilder->addUse(*EntryAddress); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = LoadBuilder->addMemOperand(**LoadMemory); !Added)
+ return std::unexpected(Added.error());
+ if (auto Built = LoadBuilder->build(); !Built)
+ return std::unexpected(Built.error());
+ Target = *TargetPointer;
+ break;
+ }
+ default:
+ return makeUnexpected(ErrorCode::Unsupported,
+ "RISC-V G_BRJT has an unsupported jump table encoding");
+ }
+
+ auto BranchBuilder = MIRBuilder.buildInstr(opcode(GenericOpcode::G_BRINDIRECT));
+ if (!BranchBuilder)
+ return std::unexpected(BranchBuilder.error());
+ if (auto Added = BranchBuilder->addUse(Target); !Added)
+ return std::unexpected(Added.error());
+ if (auto Built = BranchBuilder->build(); !Built)
+ return std::unexpected(Built.error());
+
+ return Instruction.eraseFromParent();
+}
+
+Expected<void> RISCVLegalizerInfo::legalizeVAStart(MachineInstr &Instruction,
+ MachineIRBuilder &MIRBuilder) const {
+ if (Instruction.operandCount() != 1 || Instruction.memOperands().size() != 1 ||
+ Instruction.memOperands().front() == nullptr) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V G_VASTART requires one pointer operand and one memory operand");
+ }
+
+ MachineFunction &Function = MIRBuilder.function();
+ const RISCVFunctionInfo *FunctionInfo = Function.getTargetInfo<RISCVFunctionInfo>();
+ if (FunctionInfo == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V G_VASTART requires RISC-V function information");
+
+ const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+ const Register *Destination =
+ DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+ if (Destination == nullptr || !Destination->isVirtual()) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V G_VASTART requires a virtual-register pointer operand");
+ }
+ auto AddressType = Function.registerInfo().getType(*Destination);
+ if (!AddressType)
+ return std::unexpected(AddressType.error());
+
+ const std::int64_t LLVMFrameIndex = FunctionInfo->varArgsFrameIndex();
+ const bool IsFixedObject = LLVMFrameIndex < 0;
+ const std::uint64_t ObjectOrdinal = IsFixedObject
+ ? static_cast<std::uint64_t>(-LLVMFrameIndex - 1)
+ : static_cast<std::uint64_t>(LLVMFrameIndex);
+ std::optional<FrameIndex> VarArgsFrameIndex;
+ std::uint64_t CurrentOrdinal = 0;
+ for (const StackObject &Object : Function.frameInfo().objects()) {
+ if (Object.isTombstone() || Object.isFixed() != IsFixedObject)
+ continue;
+ if (CurrentOrdinal == ObjectOrdinal) {
+ VarArgsFrameIndex = Object.index();
+ break;
+ }
+ ++CurrentOrdinal;
+ }
+ if (!VarArgsFrameIndex) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V G_VASTART references a missing varargs frame object");
+ }
+
+ auto VarArgsAddress = Function.registerInfo().createVirtualRegister(*AddressType);
+ if (!VarArgsAddress)
+ return std::unexpected(VarArgsAddress.error());
+ if (auto Point = MIRBuilder.setInsertPointBefore(Instruction); !Point)
+ return std::unexpected(Point.error());
+
+ auto FrameIndexBuilder = MIRBuilder.buildInstr(opcode(GenericOpcode::G_FRAME_INDEX));
+ if (!FrameIndexBuilder)
+ return std::unexpected(FrameIndexBuilder.error());
+ if (auto Added = FrameIndexBuilder->addDef(*VarArgsAddress); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = FrameIndexBuilder->addFrameIndex(*VarArgsFrameIndex); !Added)
+ return std::unexpected(Added.error());
+ if (auto Built = FrameIndexBuilder->build(); !Built)
+ return std::unexpected(Built.error());
+
+ auto StoreBuilder = MIRBuilder.buildInstr(opcode(GenericOpcode::G_STORE));
+ if (!StoreBuilder)
+ return std::unexpected(StoreBuilder.error());
+ if (auto Added = StoreBuilder->addUse(*VarArgsAddress); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = StoreBuilder->addUse(*Destination); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = StoreBuilder->addMemOperand(*Instruction.memOperands().front()); !Added)
+ return std::unexpected(Added.error());
+ if (auto Built = StoreBuilder->build(); !Built)
+ return std::unexpected(Built.error());
+
+ return Instruction.eraseFromParent();
+}
+
+Expected<void> RISCVLegalizerInfo::legalizeCustom(LegalizerHelper &Helper,
+ MachineInstr &Instruction,
+ const LegalizationDecision &Decision) const {
+ if (Decision.Action != LegalizationAction::Custom)
+ return makeUnexpected(ErrorCode::InvalidArgument,
+ "RISC-V custom legalizer received a non-custom action");
+
+ MachineFunction &Function = Helper.MIRBuilder.function();
+ const TargetDescription &Target = Helper.MIRBuilder.target();
+ if (Instruction.opcode() == opcode(GenericOpcode::G_BRJT))
+ return legalizeBRJT(Instruction, Helper.MIRBuilder);
+ if (Instruction.opcode() == opcode(GenericOpcode::G_VASTART))
+ return legalizeVAStart(Instruction, Helper.MIRBuilder);
+ if (Instruction.opcode() == opcode(GenericOpcode::G_ADD) ||
+ Instruction.opcode() == opcode(GenericOpcode::G_SUB)) {
+ auto XLenType = LowLevelType::makeCategoryNeutralScalar(64);
+ if (!XLenType)
+ return std::unexpected(XLenType.error());
+
+ Helper.Observer.changingInstr(Instruction);
+ if (auto Widened = Helper.widenScalarSrc(Instruction, *XLenType, 1, GenericOpcode::G_ANYEXT);
+ !Widened)
+ return std::unexpected(Widened.error());
+ if (auto Widened = Helper.widenScalarSrc(Instruction, *XLenType, 2, GenericOpcode::G_ANYEXT);
+ !Widened)
+ return std::unexpected(Widened.error());
+
+ auto ALUDestination = Function.registerInfo().createVirtualRegister(*XLenType);
+ if (!ALUDestination)
+ return std::unexpected(ALUDestination.error());
+ auto SignExtendedDestination = Function.registerInfo().createVirtualRegister(*XLenType);
+ if (!SignExtendedDestination)
+ return std::unexpected(SignExtendedDestination.error());
+
+ MachineOperand *DestinationOperand = Instruction.operandAt(0);
+ const Register *Destination =
+ DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+ if (Destination == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V word arithmetic requires a register destination");
+ const Register OriginalDestination = *Destination;
+ if (auto Point = Helper.MIRBuilder.setInsertPointAfter(Instruction); !Point)
+ return std::unexpected(Point.error());
+ if (auto Built =
+ Helper.MIRBuilder.buildSExtInReg(*SignExtendedDestination, *ALUDestination, 32);
+ !Built)
+ return std::unexpected(Built.error());
+ if (auto Built = Helper.MIRBuilder.buildTrunc(OriginalDestination, *SignExtendedDestination);
+ !Built)
+ return std::unexpected(Built.error());
+ if (auto Changed = DestinationOperand->setRegister(*ALUDestination); !Changed)
+ return std::unexpected(Changed.error());
+ Helper.Observer.changedInstr(Instruction);
+ return {};
+ }
+ if (Instruction.opcode() == opcode(GenericOpcode::G_SEXT_INREG)) {
+ const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+ const MachineOperand *SizeOperand = Instruction.operandAt(2);
+ const Register *Destination =
+ DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+ const std::int64_t *Size =
+ SizeOperand == nullptr ? nullptr : SizeOperand->getIf<std::int64_t>();
+ if (Destination == nullptr || Size == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V G_SEXT_INREG has invalid operands");
+ auto DestinationType = Function.registerInfo().getType(*Destination);
+ if (!DestinationType)
+ return std::unexpected(DestinationType.error());
+ if (DestinationType->minimumBitWidth() == 64 && *Size == 32)
+ return {};
+ if (Subtarget.features().has(RISCVExtension::Zbb) && (*Size == 8 || *Size == 16))
+ return {};
+ return Helper.lower(Instruction, LegalizationDecision{
+ .Action = LegalizationAction::Lower,
+ .TypeIndex = 0,
+ .NewType = {},
+ });
+ }
+ if (Instruction.opcode() == opcode(GenericOpcode::G_ASHR) ||
+ Instruction.opcode() == opcode(GenericOpcode::G_LSHR) ||
+ Instruction.opcode() == opcode(GenericOpcode::G_SHL)) {
+ auto XLenType = LowLevelType::makeCategoryNeutralScalar(64);
+ if (!XLenType)
+ return std::unexpected(XLenType.error());
+ const MachineOperand *ShiftOperand = Instruction.operandAt(2);
+ const Register *ShiftAmount =
+ ShiftOperand == nullptr ? nullptr : ShiftOperand->getIf<Register>();
+ if (ShiftAmount == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V shift legalization requires a register shift amount");
+
+ const GenericOpcode Generic = static_cast<GenericOpcode>(Instruction.opcode().value());
+ if (getIConstantVRegValWithLookThrough(*ShiftAmount, Function.registerInfo())) {
+ GenericOpcode ExtensionOpcode = GenericOpcode::G_ANYEXT;
+ if (Generic == GenericOpcode::G_ASHR)
+ ExtensionOpcode = GenericOpcode::G_SEXT;
+ else if (Generic == GenericOpcode::G_LSHR)
+ ExtensionOpcode = GenericOpcode::G_ZEXT;
+
+ Helper.Observer.changingInstr(Instruction);
+ if (auto Widened = Helper.widenScalarSrc(Instruction, *XLenType, 1, ExtensionOpcode);
+ !Widened)
+ return std::unexpected(Widened.error());
+ if (auto Widened = Helper.widenScalarSrc(Instruction, *XLenType, 2, GenericOpcode::G_ZEXT);
+ !Widened)
+ return std::unexpected(Widened.error());
+ if (auto Widened = Helper.widenScalarDst(Instruction, *XLenType); !Widened)
+ return std::unexpected(Widened.error());
+ Helper.Observer.changedInstr(Instruction);
+ return {};
+ }
+
+ Helper.Observer.changingInstr(Instruction);
+ if (auto Widened = Helper.widenScalarSrc(Instruction, *XLenType, 1, GenericOpcode::G_ANYEXT);
+ !Widened)
+ return std::unexpected(Widened.error());
+ if (auto Widened = Helper.widenScalarSrc(Instruction, *XLenType, 2, GenericOpcode::G_ANYEXT);
+ !Widened)
+ return std::unexpected(Widened.error());
+ if (auto Widened = Helper.widenScalarDst(Instruction, *XLenType); !Widened)
+ return std::unexpected(Widened.error());
+ auto WordOpcode = getRISCVWOpcode(Generic);
+ if (!WordOpcode)
+ return std::unexpected(WordOpcode.error());
+ if (auto Changed = Instruction.setOpcode(*WordOpcode); !Changed)
+ return std::unexpected(Changed.error());
+ Helper.Observer.changedInstr(Instruction);
+ return {};
+ }
+ if (Instruction.opcode() == opcode(GenericOpcode::G_SDIV) ||
+ Instruction.opcode() == opcode(GenericOpcode::G_UDIV) ||
+ Instruction.opcode() == opcode(GenericOpcode::G_UREM) ||
+ Instruction.opcode() == opcode(GenericOpcode::G_ROTL) ||
+ Instruction.opcode() == opcode(GenericOpcode::G_ROTR)) {
+ auto XLenType = LowLevelType::makeCategoryNeutralScalar(64);
+ if (!XLenType)
+ return std::unexpected(XLenType.error());
+
+ Helper.Observer.changingInstr(Instruction);
+ if (auto Widened = Helper.widenScalarSrc(Instruction, *XLenType, 1, GenericOpcode::G_ANYEXT);
+ !Widened)
+ return std::unexpected(Widened.error());
+ if (auto Widened = Helper.widenScalarSrc(Instruction, *XLenType, 2, GenericOpcode::G_ANYEXT);
+ !Widened)
+ return std::unexpected(Widened.error());
+ if (auto Widened = Helper.widenScalarDst(Instruction, *XLenType); !Widened)
+ return std::unexpected(Widened.error());
+ auto WordOpcode = getRISCVWOpcode(static_cast<GenericOpcode>(Instruction.opcode().value()));
+ if (!WordOpcode)
+ return std::unexpected(WordOpcode.error());
+ if (auto Changed = Instruction.setOpcode(*WordOpcode); !Changed)
+ return std::unexpected(Changed.error());
+ Helper.Observer.changedInstr(Instruction);
+ return {};
+ }
+ if (Instruction.opcode() == opcode(GenericOpcode::G_CTLZ) ||
+ Instruction.opcode() == opcode(GenericOpcode::G_CTTZ) ||
+ Instruction.opcode() == opcode(GenericOpcode::G_CTLS)) {
+ auto XLenType = LowLevelType::makeCategoryNeutralScalar(64);
+ if (!XLenType)
+ return std::unexpected(XLenType.error());
+
+ Helper.Observer.changingInstr(Instruction);
+ if (auto Widened = Helper.widenScalarSrc(Instruction, *XLenType, 1, GenericOpcode::G_ANYEXT);
+ !Widened) {
+ return std::unexpected(Widened.error());
+ }
+ if (auto Widened = Helper.widenScalarDst(Instruction, *XLenType); !Widened)
+ return std::unexpected(Widened.error());
+ auto WordOpcode = getRISCVWOpcode(static_cast<GenericOpcode>(Instruction.opcode().value()));
+ if (!WordOpcode)
+ return std::unexpected(WordOpcode.error());
+ if (auto Changed = Instruction.setOpcode(*WordOpcode); !Changed)
+ return std::unexpected(Changed.error());
+ Helper.Observer.changedInstr(Instruction);
+ return {};
+ }
+ if (Instruction.opcode() == opcode(GenericOpcode::G_FPTOSI) ||
+ Instruction.opcode() == opcode(GenericOpcode::G_FPTOUI)) {
+ auto XLenType = LowLevelType::makeCategoryNeutralScalar(64);
+ if (!XLenType)
+ return std::unexpected(XLenType.error());
+
+ Helper.Observer.changingInstr(Instruction);
+ if (auto Widened = Helper.widenScalarDst(Instruction, *XLenType); !Widened)
+ return std::unexpected(Widened.error());
+ auto WordOpcode = getRISCVWOpcode(static_cast<GenericOpcode>(Instruction.opcode().value()));
+ if (!WordOpcode)
+ return std::unexpected(WordOpcode.error());
+ if (auto Changed = Instruction.setOpcode(*WordOpcode); !Changed)
+ return std::unexpected(Changed.error());
+ if (auto RoundingMode = Instruction.addOperand(MachineOperand::immediate(1)); !RoundingMode)
+ return std::unexpected(RoundingMode.error());
+ Helper.Observer.changedInstr(Instruction);
+ return {};
+ }
+ if (Instruction.opcode() == opcode(GenericOpcode::G_IS_FPCLASS)) {
+ const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+ const MachineOperand *SourceOperand = Instruction.operandAt(1);
+ const MachineOperand *MaskOperand = Instruction.operandAt(2);
+ const Register *Destination =
+ DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+ const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>();
+ if (Destination == nullptr || Source == nullptr || MaskOperand == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V G_IS_FPCLASS has invalid operands");
+ auto Mask = signedImmediate(*MaskOperand);
+ if (!Mask)
+ return std::unexpected(Mask.error());
+ if (*Mask < 0 || *Mask > 0x3ff)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V G_IS_FPCLASS class mask must fit ten bits");
+
+ auto XLenType = LowLevelType::makeCategoryNeutralScalar(64);
+ if (!XLenType)
+ return std::unexpected(XLenType.error());
+ const std::uint64_t ClassMask = static_cast<std::uint64_t>(*Mask);
+ const std::uint64_t RISCVClassMask = (ClassMask >> 2U) | ((ClassMask & 0x3U) << 8U);
+
+ auto FClass = Function.registerInfo().createVirtualRegister(*XLenType);
+ auto MaskConstant = Function.registerInfo().createVirtualRegister(*XLenType);
+ auto Zero = Function.registerInfo().createVirtualRegister(*XLenType);
+ auto And = Function.registerInfo().createVirtualRegister(*XLenType);
+ if (!FClass)
+ return std::unexpected(FClass.error());
+ if (!MaskConstant)
+ return std::unexpected(MaskConstant.error());
+ if (!Zero)
+ return std::unexpected(Zero.error());
+ if (!And)
+ return std::unexpected(And.error());
+ if (auto Point = Helper.MIRBuilder.setInsertPointBefore(Instruction); !Point)
+ return std::unexpected(Point.error());
+ if (auto Built = Helper.MIRBuilder.buildConstant(*MaskConstant,
+ static_cast<std::int64_t>(RISCVClassMask));
+ !Built) {
+ return std::unexpected(Built.error());
+ }
+ if (auto Built = Helper.MIRBuilder.buildConstant(*Zero, 0); !Built)
+ return std::unexpected(Built.error());
+ if (auto Built = Helper.MIRBuilder.buildInstr(instructionOpcode(InstructionOpcode::G_FCLASS),
+ {*FClass}, {*Source});
+ !Built) {
+ return std::unexpected(Built.error());
+ }
+ if (auto Built = Helper.MIRBuilder.buildAnd(*And, *FClass, *MaskConstant); !Built)
+ return std::unexpected(Built.error());
+ if (auto Built =
+ Helper.MIRBuilder.buildICmp(IntegerPredicate::NotEqual, *Destination, *And, *Zero);
+ !Built) {
+ return std::unexpected(Built.error());
+ }
+ return Instruction.eraseFromParent();
+ }
+ if (Instruction.opcode() == opcode(GenericOpcode::G_ANYEXT) ||
+ Instruction.opcode() == opcode(GenericOpcode::G_SEXT) ||
+ Instruction.opcode() == opcode(GenericOpcode::G_ZEXT)) {
+ return foldConstantExtension(Instruction, Function,
+ static_cast<GenericOpcode>(Instruction.opcode().value()));
+ }
+ if (Instruction.opcode() == opcode(GenericOpcode::G_CONSTANT)) {
+ const MachineOperand *Immediate = Instruction.operandAt(1);
+ const std::optional<LowLevelType> Type = resultType(Function, Instruction);
+ if (Immediate != nullptr && Type && Type->isCategoryNeutralScalar() &&
+ Type->scalarBitWidth() == 64) {
+ auto Value = signedImmediate(*Immediate);
+ if (Value && shouldUseRV64IntegerConstantPool(*Value, Subtarget.features())) {
+ Helper.Observer.changingInstr(Instruction);
+ auto Lowered = lowerConstantToPool(Instruction, Function, Target);
+ if (!Lowered)
+ return std::unexpected(Lowered.error());
+ Helper.Observer.changedInstr(Instruction);
+ return {};
+ }
+ return {};
+ }
+ return makeUnexpected(ErrorCode::Unsupported,
+ "RISC-V constant custom legalization is unavailable");
+ }
+ return makeUnexpected(ErrorCode::Unsupported,
+ "RISC-V legalizer has no lowering rule for this instruction");
+}
+
+Expected<bool> RISCVLegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
+ MachineInstr &Instruction) const {
+ const MachineOperand *IntrinsicMachineOperand = Instruction.operandAt(0);
+ const auto *Intrinsic = IntrinsicMachineOperand == nullptr
+ ? nullptr
+ : IntrinsicMachineOperand->getIf<rcc::IntrinsicOperand>();
+ if (Intrinsic == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V intrinsic legalization requires an intrinsic operand");
+ if (Intrinsic->ID != intrinsicID(GenericIntrinsic::VACopy))
+ return false;
+ if (Instruction.operandCount() != 3)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "llvm.va_copy requires destination and source operands");
+
+ const MachineOperand *DestinationOperand = Instruction.operandAt(1);
+ const MachineOperand *SourceOperand = Instruction.operandAt(2);
+ const Register *Destination =
+ DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+ const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>();
+ if (Destination == nullptr || Source == nullptr || !Destination->isVirtual() ||
+ !Source->isVirtual()) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "llvm.va_copy requires virtual-register pointer operands");
+ }
+
+ MachineFunction &Function = Helper.MIRBuilder.function();
+ auto DestinationType = Function.registerInfo().getType(*Destination);
+ if (!DestinationType)
+ return std::unexpected(DestinationType.error());
+ auto SourceType = Function.registerInfo().getType(*Source);
+ if (!SourceType)
+ return std::unexpected(SourceType.error());
+ if (!DestinationType->isPointer() || !SourceType->isPointer() ||
+ *DestinationType != *SourceType || DestinationType->scalarBitWidth() == 0) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "llvm.va_copy operands must have the same scalar pointer type");
+ }
+
+ const auto AddressSpace = DestinationType->addressSpace();
+ if (!AddressSpace)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "llvm.va_copy pointer type has no address space");
+ auto Layout = DataLayout::parse(TargetDataLayout);
+ if (!Layout)
+ return std::unexpected(Layout.error());
+ const GenericConstantType PointerType = GenericConstantType::pointer(*AddressSpace);
+ auto PointerWidth = Layout->getPointerSizeInBits(*AddressSpace);
+ if (!PointerWidth)
+ return std::unexpected(PointerWidth.error());
+ if (*PointerWidth != DestinationType->scalarBitWidth())
+ return makeUnexpected(ErrorCode::VerificationError,
+ "llvm.va_copy pointer type disagrees with the target data layout");
+ auto ABIAlignment = Layout->getABITypeAlign(PointerType);
+ if (!ABIAlignment)
+ return std::unexpected(ABIAlignment.error());
+ auto Size = MemorySize::fixed((static_cast<std::uint64_t>(*PointerWidth) + 7U) / 8U);
+ auto Alignment = rcc::Alignment::create(*ABIAlignment);
+ if (!Size || !Alignment)
+ return makeUnexpected(ErrorCode::InternalError,
+ "llvm.va_copy pointer type has an invalid memory layout");
+
+ MachineMemOperandDesc LoadDescription;
+ LoadDescription.Accesses = {MemoryAccessFlag::Load};
+ LoadDescription.Size = *Size;
+ LoadDescription.MemoryType = *DestinationType;
+ LoadDescription.Align = *Alignment;
+ auto LoadMemory = Function.createMemOperand(std::move(LoadDescription));
+ if (!LoadMemory)
+ return std::unexpected(LoadMemory.error());
+
+ MachineMemOperandDesc StoreDescription;
+ StoreDescription.Accesses = {MemoryAccessFlag::Store};
+ StoreDescription.Size = *Size;
+ StoreDescription.MemoryType = *DestinationType;
+ StoreDescription.Align = *Alignment;
+ auto StoreMemory = Function.createMemOperand(std::move(StoreDescription));
+ if (!StoreMemory)
+ return std::unexpected(StoreMemory.error());
+
+ auto Temporary = Function.registerInfo().createVirtualRegister(*DestinationType);
+ if (!Temporary)
+ return std::unexpected(Temporary.error());
+
+ if (auto Point = Helper.MIRBuilder.setInsertPointBefore(Instruction); !Point)
+ return std::unexpected(Point.error());
+ auto LoadBuilder = Helper.MIRBuilder.buildInstr(opcode(GenericOpcode::G_LOAD));
+ if (!LoadBuilder)
+ return std::unexpected(LoadBuilder.error());
+ if (auto Added = LoadBuilder->addDef(*Temporary); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = LoadBuilder->addUse(*Source); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = LoadBuilder->addMemOperand(**LoadMemory); !Added)
+ return std::unexpected(Added.error());
+ if (auto Built = LoadBuilder->build(); !Built)
+ return std::unexpected(Built.error());
+
+ auto StoreBuilder = Helper.MIRBuilder.buildInstr(opcode(GenericOpcode::G_STORE));
+ if (!StoreBuilder)
+ return std::unexpected(StoreBuilder.error());
+ if (auto Added = StoreBuilder->addUse(*Temporary); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = StoreBuilder->addUse(*Destination); !Added)
+ return std::unexpected(Added.error());
+ if (auto Added = StoreBuilder->addMemOperand(**StoreMemory); !Added)
+ return std::unexpected(Added.error());
+ if (auto Built = StoreBuilder->build(); !Built)
+ return std::unexpected(Built.error());
+
+ if (auto Removed = Instruction.eraseFromParent(); !Removed)
+ return std::unexpected(Removed.error());
+ return true;
+}
+
+bool RISCVLegalizerInfo::isSExtCheaperThanZExt(LowLevelType SourceType,
+ LowLevelType DestinationType) const {
+ return SourceType.isCategoryNeutralScalar() && DestinationType.isCategoryNeutralScalar() &&
+ SourceType.scalarBitWidth() == 32 && DestinationType.scalarBitWidth() == 64;
+}
+
+} // namespace rcc::riscv
diff --git a/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.h b/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.h
new file mode 100644
index 0000000..36d7486
--- /dev/null
+++ b/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.h
@@ -0,0 +1,37 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef RCC_LIB_TARGET_RISCV_GISEL_RISCVLEGALIZERINFO_H
+#define RCC_LIB_TARGET_RISCV_GISEL_RISCVLEGALIZERINFO_H
+
+#include "rcc/CodeGen/GlobalISel/LegalizerInfo.h"
+
+namespace rcc {
+class MachineIRBuilder;
+} // namespace rcc
+
+namespace rcc::riscv {
+
+class RISCVSubtarget;
+
+class RISCVLegalizerInfo final : public LegalizerInfo {
+public:
+ explicit RISCVLegalizerInfo(const RISCVSubtarget &Subtarget);
+ [[nodiscard]] Expected<void> legalizeCustom(LegalizerHelper &Helper, MachineInstr &Instruction,
+ const LegalizationDecision &Decision) const override;
+ [[nodiscard]] Expected<bool> legalizeIntrinsic(LegalizerHelper &Helper,
+ MachineInstr &Instruction) const override;
+ [[nodiscard]] bool isSExtCheaperThanZExt(LowLevelType SourceType,
+ LowLevelType DestinationType) const override;
+
+private:
+ [[nodiscard]] Expected<void> legalizeBRJT(MachineInstr &Instruction,
+ MachineIRBuilder &MIRBuilder) const;
+ [[nodiscard]] Expected<void> legalizeVAStart(MachineInstr &Instruction,
+ MachineIRBuilder &MIRBuilder) const;
+
+ const RISCVSubtarget &Subtarget;
+};
+
+} // namespace rcc::riscv
+
+#endif // RCC_LIB_TARGET_RISCV_GISEL_RISCVLEGALIZERINFO_H
diff --git a/backend/lib/Target/RISCV/GISel/RISCVRegisterBankInfo.cpp b/backend/lib/Target/RISCV/GISel/RISCVRegisterBankInfo.cpp
new file mode 100644
index 0000000..c299ccb
--- /dev/null
+++ b/backend/lib/Target/RISCV/GISel/RISCVRegisterBankInfo.cpp
@@ -0,0 +1,388 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "RISCVRegisterBankInfo.h"
+
+#include "rcc/CodeGen/GenericOpcodes.h"
+#include "rcc/CodeGen/MachineFunction.h"
+#include "rcc/CodeGen/MachineInstr.h"
+#include "rcc/CodeGen/MachineRegisterInfo.h"
+#include "rcc/CodeGen/TargetDescription.h"
+#include "rcc/Target/RISCV/RISCVRegisters.h"
+
+#include <cstdint>
+#include <optional>
+#include <ranges>
+#include <string_view>
+#include <unordered_set>
+#include <utility>
+#include <vector>
+
+namespace rcc::riscv {
+namespace {
+
+[[nodiscard]] bool isScalarGPRType(LowLevelType Type) noexcept {
+ return (Type.isPointer() && Type.addressSpace() == 0 && Type.scalarBitWidth() == 64) ||
+ (Type.isCategoryNeutralScalar() && Type.scalarBitWidth() >= 1 &&
+ Type.scalarBitWidth() <= 64);
+}
+
+[[nodiscard]] constexpr RegisterBankID gprBank() noexcept {
+ return RegisterBankID::fromValue(static_cast<std::uint32_t>(RegisterBank::GPR));
+}
+
+[[nodiscard]] constexpr RegisterBankID fprBank() noexcept {
+ return RegisterBankID::fromValue(static_cast<std::uint32_t>(RegisterBank::FPR));
+}
+
+[[nodiscard]] std::optional<std::size_t> operandIndex(const MachineInstr &Instruction,
+ const MachineOperand &Operand) noexcept {
+ for (std::size_t Index = 0; Index < Instruction.operandCount(); ++Index) {
+ if (Instruction.operandAt(Index) == &Operand)
+ return Index;
+ }
+ return std::nullopt;
+}
+
+[[nodiscard]] std::optional<RegisterBankID>
+physicalRegisterBank(Register Reg, const TargetDescription &Target) noexcept {
+ const std::optional<PhysicalRegister> Physical = Reg.physicalRegister();
+ if (!Physical)
+ return std::nullopt;
+ for (const RegisterBankDesc &Bank : Target.registers().registerBanks()) {
+ if (Target.registers().bankContainsRegister(Bank.ID, *Physical))
+ return Bank.ID;
+ }
+ return std::nullopt;
+}
+
+[[nodiscard]] bool isFloatingGenericOpcode(GenericOpcode OpcodeValue) noexcept {
+ switch (OpcodeValue) {
+ case GenericOpcode::G_FRAME_INDEX:
+ case GenericOpcode::G_FREEZE:
+ case GenericOpcode::G_FENCE:
+ case GenericOpcode::G_FSHL:
+ case GenericOpcode::G_FSHR:
+ return false;
+ default:
+ return getGenericOpcodeName(OpcodeValue).starts_with("G_F");
+ }
+}
+
+[[nodiscard]] std::optional<RegisterBankID>
+requiredBankForOperand(const MachineInstr &Instruction, const MachineOperand &Operand,
+ const TargetDescription &Target) {
+ const std::optional<std::size_t> Index = operandIndex(Instruction, Operand);
+ if (!Index)
+ return std::nullopt;
+
+ if (const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction.opcode());
+ Descriptor != nullptr && *Index < Descriptor->Operands.size() &&
+ Descriptor->Operands[*Index].RegBank) {
+ return Descriptor->Operands[*Index].RegBank;
+ }
+
+ if (Instruction.opcode() == opcode(GenericOpcode::COPY)) {
+ if (!Operand.isDef())
+ return std::nullopt;
+ for (const MachineOperand &Other : Instruction.operands()) {
+ if (&Other == &Operand)
+ continue;
+ const Register *OtherReg = Other.getIf<Register>();
+ if (OtherReg != nullptr && Other.isUse() && OtherReg->isPhysical())
+ return physicalRegisterBank(*OtherReg, Target);
+ }
+ return std::nullopt;
+ }
+
+ if (!isGenericOpcode(Instruction.opcode()))
+ return std::nullopt;
+ const GenericOpcode Generic = static_cast<GenericOpcode>(Instruction.opcode().value());
+ switch (Generic) {
+ case GenericOpcode::G_FPEXTLOAD:
+ case GenericOpcode::G_FPTRUNCSTORE:
+ return *Index == 0 ? fprBank() : gprBank();
+ case GenericOpcode::G_FPTOSI:
+ case GenericOpcode::G_FPTOUI:
+ case GenericOpcode::G_FPTOSI_SAT:
+ case GenericOpcode::G_FPTOUI_SAT:
+ case GenericOpcode::G_LROUND:
+ case GenericOpcode::G_LLROUND:
+ return *Index == 0 ? gprBank() : fprBank();
+ case GenericOpcode::G_SITOFP:
+ case GenericOpcode::G_UITOFP:
+ return *Index == 0 ? fprBank() : gprBank();
+ case GenericOpcode::G_FCMP:
+ case GenericOpcode::G_IS_FPCLASS:
+ return *Index == 0 ? gprBank() : fprBank();
+ case GenericOpcode::G_FPOWI:
+ case GenericOpcode::G_FLDEXP:
+ return *Index < 2 ? fprBank() : gprBank();
+ case GenericOpcode::G_FFREXP:
+ return *Index == 1 ? gprBank() : fprBank();
+ case GenericOpcode::G_LOAD:
+ case GenericOpcode::G_SEXTLOAD:
+ case GenericOpcode::G_ZEXTLOAD:
+ return *Index == 1 ? std::optional<RegisterBankID>(gprBank()) : std::nullopt;
+ case GenericOpcode::G_STORE:
+ return *Index == 1 ? std::optional<RegisterBankID>(gprBank()) : std::nullopt;
+ case GenericOpcode::G_SELECT:
+ return *Index == 1 ? std::optional<RegisterBankID>(gprBank()) : std::nullopt;
+ case GenericOpcode::G_IMPLICIT_DEF:
+ case GenericOpcode::G_PHI:
+ return std::nullopt;
+ default:
+ return isFloatingGenericOpcode(Generic) ? fprBank() : gprBank();
+ }
+}
+
+void appendConnectedRegisters(const MachineInstr &Instruction, const MachineOperand &Operand,
+ std::vector<Register> &Pending) {
+ const std::optional<std::size_t> Index = operandIndex(Instruction, Operand);
+ if (!Index)
+ return;
+ const bool IsPhi = Instruction.opcode() == opcode(GenericOpcode::G_PHI);
+ if (!IsPhi)
+ return;
+ for (std::size_t OtherIndex = 0; OtherIndex < Instruction.operandCount(); ++OtherIndex) {
+ if (OtherIndex == *Index)
+ continue;
+ const MachineOperand *Other = Instruction.operandAt(OtherIndex);
+ const Register *OtherReg = Other == nullptr ? nullptr : Other->getIf<Register>();
+ if (OtherReg != nullptr && OtherReg->isVirtual())
+ Pending.push_back(*OtherReg);
+ }
+}
+
+[[nodiscard]] Expected<std::optional<RegisterBankID>>
+requiredBankForRegister(const MachineFunction &Function, Register Reg,
+ const TargetDescription &Target) {
+ std::vector<Register> Pending{Reg};
+ std::unordered_set<Register> Visited;
+ std::optional<RegisterBankID> Required;
+ while (!Pending.empty()) {
+ const Register Current = Pending.back();
+ Pending.pop_back();
+ if (!Visited.insert(Current).second)
+ continue;
+ for (const MachineOperand &Operand : Function.registerInfo().regOperands(Current)) {
+ const MachineInstr *Instruction = Operand.parent();
+ if (Instruction == nullptr)
+ continue;
+ const std::optional<RegisterBankID> OperandBank =
+ requiredBankForOperand(*Instruction, Operand, Target);
+ if (OperandBank) {
+ if (Required && *Required != *OperandBank) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V virtual register has conflicting register-bank uses");
+ }
+ Required = *OperandBank;
+ }
+ appendConnectedRegisters(*Instruction, Operand, Pending);
+ }
+ }
+ return Required;
+}
+
+[[nodiscard]] bool hasFloatingCopyUse(const MachineFunction &Function, Register Reg,
+ const TargetDescription &Target) {
+ for (const MachineOperand &Operand : Function.registerInfo().regOperands(Reg)) {
+ const MachineInstr *Instruction = Operand.parent();
+ if (!Operand.isUse() || Instruction == nullptr ||
+ Instruction->opcode() != opcode(GenericOpcode::COPY))
+ continue;
+ for (const MachineOperand &Other : Instruction->operands()) {
+ const Register *OtherReg = Other.getIf<Register>();
+ if (!Other.isDef() || OtherReg == nullptr || !OtherReg->isPhysical())
+ continue;
+ if (physicalRegisterBank(*OtherReg, Target) == fprBank())
+ return true;
+ }
+ }
+ return false;
+}
+
+[[nodiscard]] Expected<RegisterClassID> requiredRegisterClass(const MachineFunction &Function,
+ Register Reg,
+ const TargetDescription &Target) {
+ if (!Reg.isVirtual())
+ return makeUnexpected(ErrorCode::InvalidArgument,
+ "RISC-V register-class selection requires a virtual register");
+ auto Info = Function.registerInfo().virtualRegisterInfo(Reg);
+ if (!Info)
+ return std::unexpected(Info.error());
+ if (!Info->get().Type) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V selection requires a type for every virtual register");
+ }
+ const VirtualRegisterInfo &RegisterInfo = Info->get();
+ const LowLevelType Type = RegisterInfo.Type.value_or(LowLevelType{});
+ RegisterClassID RequiredClass;
+ if (RegisterInfo.Bank && *RegisterInfo.Bank == fprBank()) {
+ if (Type.scalarBitWidth() != 32 && Type.scalarBitWidth() != 64) {
+ return makeUnexpected(ErrorCode::Unsupported,
+ "RISC-V RV64GC has no floating register class for type '" +
+ Type.toString() + "'");
+ }
+ RequiredClass =
+ Type.scalarBitWidth() == 32
+ ? RegisterClassID::fromValue(static_cast<std::uint32_t>(RegisterClass::FPR32))
+ : RegisterClassID::fromValue(static_cast<std::uint32_t>(RegisterClass::FPR64));
+ } else if (isScalarGPRType(Type)) {
+ RequiredClass = RegisterClassID::fromValue(static_cast<std::uint32_t>(RegisterClass::GPR));
+ } else {
+ return makeUnexpected(ErrorCode::Unsupported,
+ "RISC-V RV64GC has no register class for type '" + Type.toString() + "'");
+ }
+
+ if (Target.registers().registerClass(RequiredClass) == nullptr) {
+ return makeUnexpected(ErrorCode::InternalError,
+ "RISC-V target description is missing a required register class");
+ }
+ if (RegisterInfo.Bank && !Target.registers().bankCoversClass(*RegisterInfo.Bank, RequiredClass)) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V virtual register bank cannot cover its required class");
+ }
+ if (RegisterInfo.RegClass &&
+ !Target.registers().isSubclass(*RegisterInfo.RegClass, RequiredClass)) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V virtual register has a conflicting register class");
+ }
+ return RequiredClass;
+}
+
+} // namespace
+
+Expected<RegisterBankSelection>
+RISCVRegisterBankInfo::selectBank(const MachineFunction &Function, Register Reg,
+ const VirtualRegisterInfo &Info, const MachineInstr *,
+ const TargetDescription &Target) const {
+ if (!Info.Type) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V register-bank selection requires a virtual-register type");
+ }
+
+ auto Required = requiredBankForRegister(Function, Reg, Target);
+ if (!Required)
+ return std::unexpected(Required.error());
+ const bool IsConstrained = Required->has_value() || Info.Bank.has_value();
+ const RegisterBankID Selected = Required->value_or(Info.Bank.value_or(gprBank()));
+ if (Info.Bank && Required->has_value() && *Info.Bank != Selected) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V virtual register has a conflicting register bank");
+ }
+ return RegisterBankSelection{.Bank = Selected, .IsConstrained = IsConstrained};
+}
+
+Expected<RegisterBankInstructionMapping>
+RISCVRegisterBankInfo::getInstructionMapping(const MachineInstr &Instruction,
+ const MachineFunction &Function,
+ const TargetDescription &Target) const {
+ if (Instruction.opcode() != opcode(GenericOpcode::G_SELECT))
+ return RegisterBankInstructionMapping{};
+ if (Instruction.operandCount() != 4) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V G_SELECT register-bank mapping requires four operands");
+ }
+
+ RegisterBankInstructionMapping Mapping(Instruction.operandCount());
+ Mapping[1] = gprBank();
+ unsigned FloatingConstraints = 0;
+
+ const Register *Result = Instruction.operandAt(0)->getIf<Register>();
+ if (Result == nullptr || !Result->isVirtual()) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V G_SELECT result must be a virtual register");
+ }
+ auto ResultRequirement = requiredBankForRegister(Function, *Result, Target);
+ if (!ResultRequirement)
+ return std::unexpected(ResultRequirement.error());
+ if (*ResultRequirement == fprBank() || hasFloatingCopyUse(Function, *Result, Target))
+ ++FloatingConstraints;
+
+ for (const std::size_t Index : {std::size_t{2}, std::size_t{3}}) {
+ const Register *Source = Instruction.operandAt(Index)->getIf<Register>();
+ if (Source == nullptr || !Source->isVirtual()) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V G_SELECT sources must be virtual registers");
+ }
+ auto SourceInfo = Function.registerInfo().virtualRegisterInfo(*Source);
+ if (!SourceInfo)
+ return std::unexpected(SourceInfo.error());
+ if (SourceInfo->get().Bank == fprBank())
+ ++FloatingConstraints;
+ }
+
+ const RegisterBankID ValueBank = FloatingConstraints >= 2 ? fprBank() : gprBank();
+ Mapping[0] = ValueBank;
+ Mapping[2] = ValueBank;
+ Mapping[3] = ValueBank;
+ return Mapping;
+}
+
+Expected<void> RISCVRegisterBankInfo::constrainSelectedInstruction(
+ MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target) const {
+ std::vector<std::pair<Register, RegisterClassID>> PlannedClasses;
+ for (const MachineOperand &Operand : Instruction.operands()) {
+ const Register *Reg = Operand.getIf<Register>();
+ if (Reg == nullptr || !Reg->isVirtual())
+ continue;
+ auto Required = requiredRegisterClass(Function, *Reg, Target);
+ if (!Required)
+ return std::unexpected(Required.error());
+ if (std::ranges::find_if(PlannedClasses, [Reg](const auto &Entry) {
+ return Entry.first == *Reg;
+ }) == PlannedClasses.end())
+ PlannedClasses.emplace_back(*Reg, *Required);
+ }
+ for (const auto &[Reg, Required] : PlannedClasses) {
+ auto Info = Function.registerInfo().virtualRegisterInfo(Reg);
+ if (!Info)
+ return std::unexpected(Info.error());
+ if (Info->get().RegClass)
+ continue;
+ auto SetClass = Function.registerInfo().setRegisterClass(Reg, Required);
+ if (!SetClass)
+ return std::unexpected(SetClass.error());
+ }
+ return {};
+}
+
+Expected<void> RISCVRegisterBankInfo::constrainRegister(MachineFunction &Function, Register Reg,
+ const TargetDescription &Target) const {
+ auto Info = Function.registerInfo().virtualRegisterInfo(Reg);
+ if (!Info)
+ return std::unexpected(Info.error());
+ auto Bank = selectBank(Function, Reg, Info->get(), nullptr, Target);
+ if (!Bank)
+ return std::unexpected(Bank.error());
+ auto RequiredClass = requiredRegisterClass(Function, Reg, Target);
+ if (!RequiredClass)
+ return std::unexpected(RequiredClass.error());
+ if (!Info->get().Bank) {
+ auto SetBank = Function.registerInfo().setRegisterBank(Reg, Bank->Bank);
+ if (!SetBank)
+ return std::unexpected(SetBank.error());
+ }
+ if (!Info->get().RegClass) {
+ auto SetClass = Function.registerInfo().setRegisterClass(Reg, *RequiredClass);
+ if (!SetClass)
+ return std::unexpected(SetClass.error());
+ }
+ return {};
+}
+
+Expected<void> RISCVRegisterBankInfo::validateInstruction(const MachineInstr &Instruction,
+ const MachineFunction &Function,
+ const TargetDescription &Target) const {
+ for (const MachineOperand &Operand : Instruction.operands()) {
+ const Register *Reg = Operand.getIf<Register>();
+ if (Reg == nullptr || !Reg->isVirtual())
+ continue;
+ auto Required = requiredRegisterClass(Function, *Reg, Target);
+ if (!Required)
+ return std::unexpected(Required.error());
+ }
+ return {};
+}
+
+} // namespace rcc::riscv
diff --git a/backend/lib/Target/RISCV/GISel/RISCVRegisterBankInfo.h b/backend/lib/Target/RISCV/GISel/RISCVRegisterBankInfo.h
new file mode 100644
index 0000000..69eb539
--- /dev/null
+++ b/backend/lib/Target/RISCV/GISel/RISCVRegisterBankInfo.h
@@ -0,0 +1,30 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef RCC_LIB_TARGET_RISCV_GISEL_RISCVREGISTERBANKINFO_H
+#define RCC_LIB_TARGET_RISCV_GISEL_RISCVREGISTERBANKINFO_H
+
+#include "rcc/CodeGen/RegisterBankInfo.h"
+
+namespace rcc::riscv {
+
+class RISCVRegisterBankInfo final : public RegisterBankInfo {
+public:
+ [[nodiscard]] Expected<RegisterBankSelection>
+ selectBank(const MachineFunction &Function, Register Reg, const VirtualRegisterInfo &Info,
+ const MachineInstr *Instruction, const TargetDescription &Target) const override;
+ [[nodiscard]] Expected<RegisterBankInstructionMapping>
+ getInstructionMapping(const MachineInstr &Instruction, const MachineFunction &Function,
+ const TargetDescription &Target) const override;
+ [[nodiscard]] Expected<void> constrainSelectedInstruction(MachineInstr &Instruction,
+ MachineFunction &Function,
+ const TargetDescription &Target) const;
+ [[nodiscard]] Expected<void> constrainRegister(MachineFunction &Function, Register Reg,
+ const TargetDescription &Target) const;
+ [[nodiscard]] Expected<void> validateInstruction(const MachineInstr &Instruction,
+ const MachineFunction &Function,
+ const TargetDescription &Target) const override;
+};
+
+} // namespace rcc::riscv
+
+#endif // RCC_LIB_TARGET_RISCV_GISEL_RISCVREGISTERBANKINFO_H
generated by cgit v1.2.3 (git 2.46.0) at 2026-10-30 15:37:42 +0000