aboutsummaryrefslogtreecommitdiff
path: root/backend/lib/Target/RISCV/RISCVCodeGen.cpp
diff options
context:
space:
mode:
authorRichard-Qin-X <richard.qin.001@gmail.com>2026-08-08 02:58:50 +0800
committerRichard-Qin-X <richard.qin.001@gmail.com>2026-08-08 02:58:50 +0800
commit10b7e69448f866bcd6893b21d7f78d766252eb11 (patch)
treefe937dbf2084a0c176a23ed8a08a47ea20a9c1a3 /backend/lib/Target/RISCV/RISCVCodeGen.cpp
parentf99ef4322a7a2c09380fff61f88ab9440f05a92c (diff)
downloadrcc-project-10b7e69448f866bcd6893b21d7f78d766252eb11.tar.gz
rcc-project-10b7e69448f866bcd6893b21d7f78d766252eb11.tar.xz
[CodeGen] Implement RV64GC legalization and instruction selection
Diffstat (limited to 'backend/lib/Target/RISCV/RISCVCodeGen.cpp')
-rw-r--r--backend/lib/Target/RISCV/RISCVCodeGen.cpp3239
1 files changed, 3239 insertions, 0 deletions
diff --git a/backend/lib/Target/RISCV/RISCVCodeGen.cpp b/backend/lib/Target/RISCV/RISCVCodeGen.cpp
new file mode 100644
index 0000000..631ac33
--- /dev/null
+++ b/backend/lib/Target/RISCV/RISCVCodeGen.cpp
@@ -0,0 +1,3239 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "rcc/Target/RISCV/RISCVCodeGen.h"
+
+#include "rcc/CodeGen/GenericOpcodes.h"
+#include "rcc/CodeGen/LowLevelType.h"
+#include "rcc/CodeGen/MachineFunction.h"
+#include "rcc/CodeGen/MachineIRBuilder.h"
+#include "rcc/CodeGen/MachineInstr.h"
+#include "rcc/CodeGen/MachineMemOperand.h"
+#include "rcc/CodeGen/MachineOperand.h"
+#include "rcc/CodeGen/MachineRegisterInfo.h"
+#include "rcc/CodeGen/TargetDescription.h"
+#include "rcc/Target/RISCV/RISCVTarget.h"
+
+#include <array>
+#include <bit>
+#include <cstdint>
+#include <limits>
+#include <optional>
+#include <ranges>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace rcc::riscv {
+namespace {
+
+[[nodiscard]] Opcode targetOpcode(InstructionOpcode Value) noexcept {
+ return Opcode::fromValue(static_cast<std::uint32_t>(Value));
+}
+
+[[nodiscard]] std::optional<LLT> registerType(const MachineFunction &Function,
+ const MachineOperand &Operand) {
+ const Register *Reg = Operand.getIf<Register>();
+ if (Reg == nullptr || !Reg->isVirtual())
+ return std::nullopt;
+ auto Info = Function.registerInfo().virtualRegisterInfo(*Reg);
+ if (!Info)
+ return std::nullopt;
+ return Info->get().Type;
+}
+
+[[nodiscard]] std::optional<LLT> resultType(const MachineFunction &Function,
+ const MachineInstr &Instruction) {
+ const MachineOperand *Result = Instruction.operandAt(0);
+ return Result == nullptr ? std::nullopt : registerType(Function, *Result);
+}
+
+[[nodiscard]] std::optional<LLT> operandType(const MachineFunction &Function,
+ const MachineInstr &Instruction, std::size_t Index) {
+ const MachineOperand *Operand = Instruction.operandAt(Index);
+ return Operand == nullptr ? std::nullopt : registerType(Function, *Operand);
+}
+
+[[nodiscard]] bool isRVFloatType(LLT Type) noexcept {
+ if (!Type.isFloat())
+ return false;
+ const std::optional<FloatSemantics> Semantics = Type.floatSemantics();
+ return Semantics &&
+ (*Semantics == FloatSemantics::IEEESingle || *Semantics == FloatSemantics::IEEEDouble);
+}
+
+[[nodiscard]] bool isSupportedIntegerType(LLT Type) noexcept {
+ return (Type.isInteger() && Type.scalarBitWidth() != 0 && Type.scalarBitWidth() <= 64) ||
+ (Type.isPointer() && Type.scalarBitWidth() == 64);
+}
+
+[[nodiscard]] bool isSupportedIntegerScalar(LLT Type) noexcept {
+ return Type.isInteger() && Type.scalarBitWidth() != 0 && Type.scalarBitWidth() <= 64;
+}
+
+[[nodiscard]] bool isNativeIntegerScalar(LLT Type) noexcept {
+ return Type.isInteger() && (Type.scalarBitWidth() == 32 || Type.scalarBitWidth() == 64);
+}
+
+[[nodiscard]] bool isScalarGPRType(LLT Type) noexcept {
+ return (Type.isInteger() && Type.scalarBitWidth() != 0 && Type.scalarBitWidth() <= 64) ||
+ (Type.isPointer() && Type.scalarBitWidth() == 64);
+}
+
+[[nodiscard]] bool isI1Type(LLT Type) noexcept {
+ return Type.isInteger() && Type.scalarBitWidth() == 1;
+}
+
+[[nodiscard]] bool isScalarIntegerExtension(LLT Destination, LLT Source) noexcept {
+ return Destination.isInteger() && Source.isInteger() && Destination.scalarBitWidth() != 0 &&
+ Source.scalarBitWidth() != 0 && Source.scalarBitWidth() < Destination.scalarBitWidth() &&
+ Destination.scalarBitWidth() <= 64;
+}
+
+[[nodiscard]] bool fitsSigned(std::int64_t Value, std::uint32_t Bits) noexcept {
+ const std::int64_t Minimum = -(std::int64_t{1} << (Bits - 1U));
+ const std::int64_t Maximum = (std::int64_t{1} << (Bits - 1U)) - 1;
+ return Value >= Minimum && Value <= Maximum;
+}
+
+[[nodiscard]] bool isSupportedNonAtomicMemoryAccess(const MachineInstr &Instruction) noexcept {
+ if (Instruction.memOperands().size() != 1)
+ return false;
+ const MachineMemOperand &Memory = *Instruction.memOperands().front();
+ const MachineMemOperandDesc &Description = Memory.description();
+ return !Memory.isAtomic() && Description.AddressSpace == 0 && Description.Size.isKnown() &&
+ !Description.Size.isScalable() && Description.Size.minimumBytes() != 0;
+}
+
+[[nodiscard]] bool isSupportedAtomicMemoryAccess(const MachineInstr &Instruction,
+ AtomicOperation Operation,
+ std::uint64_t Bytes) noexcept {
+ if (Instruction.memOperands().size() != 1 || (Bytes != 4 && Bytes != 8))
+ return false;
+ const MachineMemOperand &Memory = *Instruction.memOperands().front();
+ const MachineMemOperandDesc &Description = Memory.description();
+ return Description.AtomicKind == Operation && Description.AddressSpace == 0 &&
+ Description.Size.isKnown() && !Description.Size.isScalable() &&
+ Description.Size.minimumBytes() == Bytes && Description.Align.bytes() >= Bytes &&
+ Description.SyncScope.value() == 0;
+}
+
+[[nodiscard]] std::optional<std::uint64_t>
+memoryAccessBytes(const MachineInstr &Instruction) noexcept {
+ if (Instruction.memOperands().size() != 1)
+ return std::nullopt;
+ const MachineMemOperandDesc &Description = Instruction.memOperands().front()->description();
+ if (!Description.Size.isKnown() || Description.Size.isScalable())
+ return std::nullopt;
+ return Description.Size.minimumBytes();
+}
+
+[[nodiscard]] bool memoryMatchesType(const MachineInstr &Instruction, LLT Type) noexcept {
+ const auto Bytes = memoryAccessBytes(Instruction);
+ if (!Bytes || (!isSupportedIntegerType(Type) && !isRVFloatType(Type)))
+ return false;
+ if (isRVFloatType(Type))
+ return Type.scalarBitWidth() == 32 ? *Bytes == 4 : *Bytes == 8;
+ const std::uint64_t ValueBytes = (Type.scalarBitWidth() + 7U) / 8U;
+ return (*Bytes == 1 || *Bytes == 2 || *Bytes == 4 || *Bytes == 8) && *Bytes <= ValueBytes;
+}
+
+[[nodiscard]] Expected<RegisterClassID> requiredRegisterClass(const MachineFunction &Function,
+ Register Reg,
+ const TargetDescription &Target) {
+ if (!Reg.isVirtual())
+ return makeUnexpected(ErrorCode::InvalidArgument,
+ "RISC-V register-class selection requires a virtual register");
+ auto Info = Function.registerInfo().virtualRegisterInfo(Reg);
+ if (!Info)
+ return std::unexpected(Info.error());
+ if (!Info->get().Type) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V selection requires a type for every virtual register");
+ }
+ const VirtualRegisterInfo &RegisterInfo = Info->get();
+ const LLT Type = RegisterInfo.Type.value_or(LLT{});
+ RegisterClassID RequiredClass;
+ if (isScalarGPRType(Type)) {
+ RequiredClass = RegisterClassID::fromValue(static_cast<std::uint32_t>(RegisterClass::GPR));
+ } else if (Type.isFloat() && Type.scalarBitWidth() == 32) {
+ RequiredClass = RegisterClassID::fromValue(static_cast<std::uint32_t>(RegisterClass::FPR32));
+ } else if (Type.isFloat() && Type.scalarBitWidth() == 64) {
+ RequiredClass = RegisterClassID::fromValue(static_cast<std::uint32_t>(RegisterClass::FPR64));
+ } else {
+ return makeUnexpected(ErrorCode::Unsupported,
+ "RISC-V RV64GC has no register class for type '" + Type.toString() + "'");
+ }
+
+ if (Target.registers().registerClass(RequiredClass) == nullptr) {
+ return makeUnexpected(ErrorCode::InternalError,
+ "RISC-V target description is missing a required register class");
+ }
+ if (RegisterInfo.Bank && !Target.registers().bankCoversClass(*RegisterInfo.Bank, RequiredClass)) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V virtual register bank cannot cover its required class");
+ }
+ if (RegisterInfo.RegClass &&
+ !Target.registers().isSubclass(*RegisterInfo.RegClass, RequiredClass)) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V virtual register has a conflicting register class");
+ }
+ return RequiredClass;
+}
+
+[[nodiscard]] Expected<void> appendImmediate(MachineInstr &Instruction, std::int64_t Value) {
+ return Instruction.addOperand(MachineOperand::immediate(Value));
+}
+
+[[nodiscard]] Expected<void> insertPhysicalRegister(MachineInstr &Instruction, std::size_t Index,
+ PhysicalRegister Reg,
+ RegisterOperandFlag Flag) {
+ auto Operand = MachineOperand::registerOperand(Register::fromPhysical(Reg), {Flag});
+ if (!Operand)
+ return std::unexpected(Operand.error());
+ return Instruction.insertOperand(Index, std::move(*Operand));
+}
+
+[[nodiscard]] Expected<void> insertVirtualUse(MachineInstr &Instruction, std::size_t Index,
+ Register Reg) {
+ auto Operand = MachineOperand::registerOperand(Reg, {RegisterOperandFlag::Use});
+ if (!Operand)
+ return std::unexpected(Operand.error());
+ return Instruction.insertOperand(Index, std::move(*Operand));
+}
+
+[[nodiscard]] Expected<Register> createTemporary(MachineFunction &Function, LLT Type,
+ const TargetDescription &Target) {
+ auto Temporary = Function.registerInfo().createVirtualRegister(Type);
+ if (!Temporary)
+ return std::unexpected(Temporary.error());
+ auto RequiredClass = requiredRegisterClass(Function, *Temporary, Target);
+ if (!RequiredClass)
+ return std::unexpected(RequiredClass.error());
+ const RegisterBank BankKind = Type.isFloat() ? RegisterBank::FPR : RegisterBank::GPR;
+ auto Bank = Function.registerInfo().setRegisterBank(
+ *Temporary, RegisterBankID::fromValue(static_cast<std::uint32_t>(BankKind)));
+ if (!Bank)
+ return std::unexpected(Bank.error());
+ auto Class = Function.registerInfo().setRegisterClass(*Temporary, *RequiredClass);
+ if (!Class)
+ return std::unexpected(Class.error());
+ return *Temporary;
+}
+
+[[nodiscard]] Expected<std::int64_t> signedImmediate(const MachineOperand &Operand) {
+ if (const std::int64_t *Value = Operand.getIf<std::int64_t>())
+ return *Value;
+ const auto *Integer = Operand.getIf<IntegerImmediateOperand>();
+ if (Integer == nullptr || Integer->Value.bitWidth() == 0 || Integer->Value.bitWidth() > 64)
+ return makeUnexpected(ErrorCode::Unsupported,
+ "RISC-V constant selection requires an immediate no wider than 64 bits");
+ std::uint64_t Raw = Integer->Value.words().empty() ? 0 : Integer->Value.words().front();
+ const std::uint32_t Width = Integer->Value.bitWidth();
+ if (Width < 64 && (Raw & (std::uint64_t{1} << (Width - 1U))) != 0)
+ Raw |= ~std::uint64_t{0} << Width;
+ return std::bit_cast<std::int64_t>(Raw);
+}
+
+class RISCVLegalizerInfo final : public TargetLegalizerInfo {
+public:
+ [[nodiscard]] Expected<LegalizationAction> classify(const MachineInstr &Instruction,
+ const MachineFunction &Function,
+ const TargetDescription &) const override {
+ if (!isGenericOpcode(Instruction.opcode()))
+ return LegalizationAction::Legal;
+
+ const GenericOpcode Generic = static_cast<GenericOpcode>(Instruction.opcode().value());
+ const std::optional<LLT> Type = resultType(Function, Instruction);
+ switch (Generic) {
+ case GenericOpcode::G_BR:
+ return LegalizationAction::Legal;
+ case GenericOpcode::G_BRCOND: {
+ const std::optional<LLT> Condition = operandType(Function, Instruction, 0);
+ return Condition && isI1Type(*Condition) ? LegalizationAction::Legal
+ : LegalizationAction::Unsupported;
+ }
+ case GenericOpcode::G_PHI: {
+ if (!Type || (!isSupportedIntegerType(*Type) && !isRVFloatType(*Type)))
+ return LegalizationAction::Unsupported;
+ for (std::size_t Index = 1; Index < Instruction.operandCount(); Index += 2) {
+ const std::optional<LLT> Incoming = operandType(Function, Instruction, Index);
+ if (!Incoming || *Incoming != *Type)
+ return LegalizationAction::Unsupported;
+ }
+ return LegalizationAction::Legal;
+ }
+ case GenericOpcode::G_IMPLICIT_DEF:
+ return Type && (isSupportedIntegerType(*Type) || isRVFloatType(*Type))
+ ? LegalizationAction::Legal
+ : LegalizationAction::Unsupported;
+ case GenericOpcode::G_STORE:
+ if (Instruction.operandCount() == 0)
+ return LegalizationAction::Unsupported;
+ if (const std::optional<LLT> ValueType = registerType(Function, *Instruction.operandAt(0));
+ ValueType) {
+ const std::optional<LLT> AddressType = operandType(Function, Instruction, 1);
+ if (!AddressType || !AddressType->isPointer() || AddressType->addressSpace() != 0 ||
+ AddressType->scalarBitWidth() != 64) {
+ return LegalizationAction::Unsupported;
+ }
+ if (Instruction.memOperands().size() == 1 &&
+ Instruction.memOperands().front()->isAtomic()) {
+ const std::uint64_t Bytes = ValueType->scalarBitWidth() / 8U;
+ return isSupportedIntegerType(*ValueType) &&
+ isSupportedAtomicMemoryAccess(Instruction, AtomicOperation::Store, Bytes)
+ ? LegalizationAction::Legal
+ : LegalizationAction::Unsupported;
+ }
+ if ((isSupportedIntegerType(*ValueType) || isRVFloatType(*ValueType)) &&
+ isSupportedNonAtomicMemoryAccess(Instruction) &&
+ memoryMatchesType(Instruction, *ValueType)) {
+ return LegalizationAction::Legal;
+ }
+ }
+ return LegalizationAction::Unsupported;
+ case GenericOpcode::G_SEXTLOAD:
+ case GenericOpcode::G_ZEXTLOAD: {
+ const auto Bytes = memoryAccessBytes(Instruction);
+ const std::optional<LLT> AddressType = operandType(Function, Instruction, 1);
+ return Type && Bytes && AddressType && isSupportedIntegerType(*Type) &&
+ AddressType->isPointer() && AddressType->addressSpace() == 0 &&
+ AddressType->scalarBitWidth() == 64 &&
+ *Bytes < ((Type->scalarBitWidth() + 7U) / 8U) &&
+ (*Bytes == 1 || *Bytes == 2 || *Bytes == 4) &&
+ isSupportedNonAtomicMemoryAccess(Instruction)
+ ? LegalizationAction::Legal
+ : LegalizationAction::Unsupported;
+ }
+ case GenericOpcode::G_FPEXTLOAD: {
+ const auto Bytes = memoryAccessBytes(Instruction);
+ const std::optional<LLT> AddressType = operandType(Function, Instruction, 1);
+ return Type && Bytes && AddressType && Type->isFloat() && Type->scalarBitWidth() == 64 &&
+ AddressType->isPointer() && AddressType->addressSpace() == 0 &&
+ AddressType->scalarBitWidth() == 64 && *Bytes == 4 &&
+ isSupportedNonAtomicMemoryAccess(Instruction)
+ ? LegalizationAction::Legal
+ : LegalizationAction::Unsupported;
+ }
+ case GenericOpcode::G_FPTRUNCSTORE: {
+ const std::optional<LLT> ValueType = operandType(Function, Instruction, 0);
+ const std::optional<LLT> AddressType = operandType(Function, Instruction, 1);
+ const auto Bytes = memoryAccessBytes(Instruction);
+ return ValueType && AddressType && Bytes && ValueType->isFloat() &&
+ ValueType->scalarBitWidth() == 64 && AddressType->isPointer() &&
+ AddressType->addressSpace() == 0 && AddressType->scalarBitWidth() == 64 &&
+ *Bytes == 4 && isSupportedNonAtomicMemoryAccess(Instruction)
+ ? LegalizationAction::Legal
+ : LegalizationAction::Unsupported;
+ }
+ case GenericOpcode::G_LOAD:
+ if (Type) {
+ const std::optional<LLT> AddressType = operandType(Function, Instruction, 1);
+ if (!AddressType || !AddressType->isPointer() || AddressType->addressSpace() != 0 ||
+ AddressType->scalarBitWidth() != 64) {
+ return LegalizationAction::Unsupported;
+ }
+ if (Instruction.memOperands().size() == 1 &&
+ Instruction.memOperands().front()->isAtomic()) {
+ const std::uint64_t Bytes = Type->scalarBitWidth() / 8U;
+ return isSupportedIntegerType(*Type) &&
+ isSupportedAtomicMemoryAccess(Instruction, AtomicOperation::Load, Bytes)
+ ? LegalizationAction::Legal
+ : LegalizationAction::Unsupported;
+ }
+ if ((isSupportedIntegerType(*Type) || isRVFloatType(*Type)) &&
+ isSupportedNonAtomicMemoryAccess(Instruction) &&
+ memoryMatchesType(Instruction, *Type)) {
+ return LegalizationAction::Legal;
+ }
+ }
+ return LegalizationAction::Unsupported;
+ case GenericOpcode::G_FADD:
+ case GenericOpcode::G_FSUB:
+ case GenericOpcode::G_FMUL:
+ case GenericOpcode::G_FDIV: {
+ const std::optional<LLT> Left = operandType(Function, Instruction, 1);
+ const std::optional<LLT> Right = operandType(Function, Instruction, 2);
+ return Type && Left && Right && isRVFloatType(*Type) && *Left == *Type && *Right == *Type
+ ? LegalizationAction::Legal
+ : LegalizationAction::Unsupported;
+ }
+ case GenericOpcode::G_FMA: {
+ const std::optional<LLT> First = operandType(Function, Instruction, 1);
+ const std::optional<LLT> Second = operandType(Function, Instruction, 2);
+ const std::optional<LLT> Third = operandType(Function, Instruction, 3);
+ return Type && First && Second && Third && isRVFloatType(*Type) && *First == *Type &&
+ *Second == *Type && *Third == *Type
+ ? LegalizationAction::Legal
+ : LegalizationAction::Unsupported;
+ }
+ case GenericOpcode::G_FNEG:
+ case GenericOpcode::G_FABS:
+ case GenericOpcode::G_FSQRT: {
+ const std::optional<LLT> Source = operandType(Function, Instruction, 1);
+ return Type && Source && isRVFloatType(*Type) && *Source == *Type
+ ? LegalizationAction::Legal
+ : LegalizationAction::Unsupported;
+ }
+ case GenericOpcode::G_FCOPYSIGN: {
+ const std::optional<LLT> Left = operandType(Function, Instruction, 1);
+ const std::optional<LLT> Right = operandType(Function, Instruction, 2);
+ return Type && Left && Right && isRVFloatType(*Type) && *Left == *Type && *Right == *Type
+ ? LegalizationAction::Legal
+ : LegalizationAction::Unsupported;
+ }
+ case GenericOpcode::G_FCONSTANT: {
+ const MachineOperand *Immediate = Instruction.operandAt(1);
+ const auto *Floating =
+ Immediate == nullptr ? nullptr : Immediate->getIf<FloatingImmediateOperand>();
+ const std::optional<FloatSemantics> Semantics = Type ? Type->floatSemantics() : std::nullopt;
+ return Type && isRVFloatType(*Type) && Floating != nullptr && Semantics &&
+ Floating->Bits.semantics() == *Semantics
+ ? LegalizationAction::Legal
+ : LegalizationAction::Unsupported;
+ }
+ case GenericOpcode::G_ASSERT_SEXT:
+ case GenericOpcode::G_ASSERT_ZEXT:
+ case GenericOpcode::G_ASSERT_ALIGN: {
+ const std::optional<LLT> Source = operandType(Function, Instruction, 1);
+ return Type && Source && *Type == *Source && isScalarGPRType(*Type) &&
+ Instruction.operandCount() == 3
+ ? LegalizationAction::Legal
+ : LegalizationAction::Unsupported;
+ }
+ case GenericOpcode::G_FREEZE:
+ case GenericOpcode::G_CONSTANT_FOLD_BARRIER: {
+ const std::optional<LLT> Source = operandType(Function, Instruction, 1);
+ return Type && Source && *Type == *Source &&
+ (isSupportedIntegerType(*Type) || isRVFloatType(*Type))
+ ? LegalizationAction::Legal
+ : LegalizationAction::Unsupported;
+ }
+ case GenericOpcode::G_PTRTOINT: {
+ const std::optional<LLT> Source = operandType(Function, Instruction, 1);
+ return Type && Source && Type->isInteger() && Type->scalarBitWidth() == 64 &&
+ Source->isPointer() && Source->scalarBitWidth() == 64
+ ? LegalizationAction::Legal
+ : LegalizationAction::Unsupported;
+ }
+ case GenericOpcode::G_INTTOPTR: {
+ const std::optional<LLT> Source = operandType(Function, Instruction, 1);
+ return Type && Source && Type->isPointer() && Type->scalarBitWidth() == 64 &&
+ Source->isInteger() && Source->scalarBitWidth() == 64
+ ? LegalizationAction::Legal
+ : LegalizationAction::Unsupported;
+ }
+ case GenericOpcode::G_BITCAST: {
+ const std::optional<LLT> Source = operandType(Function, Instruction, 1);
+ const bool IntegerFloatPair =
+ Type && Source && Type->scalarBitWidth() == Source->scalarBitWidth() &&
+ ((Type->isFloat() && Source->isInteger()) || (Type->isInteger() && Source->isFloat())) &&
+ (Type->scalarBitWidth() == 32 || Type->scalarBitWidth() == 64);
+ return IntegerFloatPair ? LegalizationAction::Legal : LegalizationAction::Unsupported;
+ }
+ case GenericOpcode::G_ANYEXT:
+ case GenericOpcode::G_SEXT:
+ case GenericOpcode::G_ZEXT: {
+ const std::optional<LLT> Source = operandType(Function, Instruction, 1);
+ return Type && Source && isScalarIntegerExtension(*Type, *Source)
+ ? LegalizationAction::Legal
+ : LegalizationAction::Unsupported;
+ }
+ case GenericOpcode::G_TRUNC: {
+ const std::optional<LLT> Source = operandType(Function, Instruction, 1);
+ return Type && Source && isSupportedIntegerScalar(*Type) &&
+ isSupportedIntegerScalar(*Source) && Type->scalarBitWidth() != 0 &&
+ Source->scalarBitWidth() >= Type->scalarBitWidth()
+ ? LegalizationAction::Legal
+ : LegalizationAction::Unsupported;
+ }
+ case GenericOpcode::G_SEXT_INREG: {
+ const std::optional<LLT> Source = operandType(Function, Instruction, 1);
+ const MachineOperand *WidthOperand = Instruction.operandAt(2);
+ const auto Width =
+ WidthOperand == nullptr
+ ? Expected<std::int64_t>(std::unexpected(
+ Error(ErrorCode::InvalidArgument, "missing in-register extension width")))
+ : signedImmediate(*WidthOperand);
+ return Type && Source && *Type == *Source && isScalarGPRType(*Type) && Width && *Width > 0 &&
+ static_cast<std::uint64_t>(*Width) <= Type->scalarBitWidth()
+ ? LegalizationAction::Legal
+ : LegalizationAction::Unsupported;
+ }
+ case GenericOpcode::G_ICMP: {
+ const MachineOperand *Predicate = Instruction.operandAt(1);
+ const auto *IntegerPredicate =
+ Predicate == nullptr ? nullptr : Predicate->getIf<PredicateOperand>();
+ const std::optional<LLT> Left = operandType(Function, Instruction, 2);
+ const std::optional<LLT> Right = operandType(Function, Instruction, 3);
+ return Type && isI1Type(*Type) && IntegerPredicate != nullptr &&
+ IntegerPredicate->integerPredicate() != nullptr && Left && Right &&
+ *Left == *Right && isSupportedIntegerType(*Left)
+ ? LegalizationAction::Legal
+ : LegalizationAction::Unsupported;
+ }
+ case GenericOpcode::G_FCMP: {
+ const MachineOperand *Predicate = Instruction.operandAt(1);
+ const auto *FloatingPredicate =
+ Predicate == nullptr ? nullptr : Predicate->getIf<PredicateOperand>();
+ const std::optional<LLT> Left = operandType(Function, Instruction, 2);
+ const std::optional<LLT> Right = operandType(Function, Instruction, 3);
+ return Type && isI1Type(*Type) && FloatingPredicate != nullptr &&
+ FloatingPredicate->floatingPredicate() != nullptr && Left && Right &&
+ *Left == *Right && isRVFloatType(*Left)
+ ? LegalizationAction::Legal
+ : LegalizationAction::Unsupported;
+ }
+ case GenericOpcode::G_FPEXT:
+ case GenericOpcode::G_FPTRUNC: {
+ const std::optional<LLT> Source = operandType(Function, Instruction, 1);
+ const bool IsExtension = Generic == GenericOpcode::G_FPEXT;
+ return Type && Source && isRVFloatType(*Type) && isRVFloatType(*Source) &&
+ ((IsExtension && Source->scalarBitWidth() == 32 &&
+ Type->scalarBitWidth() == 64) ||
+ (!IsExtension && Source->scalarBitWidth() == 64 &&
+ Type->scalarBitWidth() == 32))
+ ? LegalizationAction::Legal
+ : LegalizationAction::Unsupported;
+ }
+ case GenericOpcode::G_FPTOSI:
+ case GenericOpcode::G_FPTOUI: {
+ const std::optional<LLT> Source = operandType(Function, Instruction, 1);
+ return Type && Source && isSupportedIntegerType(*Type) && isRVFloatType(*Source)
+ ? LegalizationAction::Legal
+ : LegalizationAction::Unsupported;
+ }
+ case GenericOpcode::G_SITOFP:
+ case GenericOpcode::G_UITOFP: {
+ const std::optional<LLT> Source = operandType(Function, Instruction, 1);
+ return Type && Source && isRVFloatType(*Type) && isSupportedIntegerType(*Source)
+ ? LegalizationAction::Legal
+ : LegalizationAction::Unsupported;
+ }
+ case GenericOpcode::G_FMINNUM:
+ case GenericOpcode::G_FMAXNUM: {
+ const std::optional<LLT> Left = operandType(Function, Instruction, 1);
+ const std::optional<LLT> Right = operandType(Function, Instruction, 2);
+ return Type && Left && Right && isRVFloatType(*Type) && *Left == *Type && *Right == *Type
+ ? LegalizationAction::Legal
+ : LegalizationAction::Unsupported;
+ }
+ case GenericOpcode::G_SELECT: {
+ const std::optional<LLT> Condition = operandType(Function, Instruction, 1);
+ const std::optional<LLT> TrueValue = operandType(Function, Instruction, 2);
+ const std::optional<LLT> FalseValue = operandType(Function, Instruction, 3);
+ return Type && Condition && TrueValue && FalseValue && isI1Type(*Condition) &&
+ *TrueValue == *Type && *FalseValue == *Type &&
+ (isSupportedIntegerType(*Type) || isRVFloatType(*Type))
+ ? LegalizationAction::Legal
+ : LegalizationAction::Unsupported;
+ }
+ case GenericOpcode::G_PTRMASK: {
+ const std::optional<LLT> Pointer = operandType(Function, Instruction, 1);
+ const std::optional<LLT> Mask = operandType(Function, Instruction, 2);
+ return Type && Pointer && Mask && Type->isPointer() && Type->addressSpace() == 0 &&
+ Type->scalarBitWidth() == 64 && *Pointer == *Type &&
+ isSupportedIntegerScalar(*Mask) && Mask->scalarBitWidth() == 64
+ ? LegalizationAction::Legal
+ : LegalizationAction::Unsupported;
+ }
+ case GenericOpcode::G_ABS: {
+ const std::optional<LLT> Source = operandType(Function, Instruction, 1);
+ return Type && Source && *Source == *Type && isSupportedIntegerScalar(*Type)
+ ? LegalizationAction::Legal
+ : LegalizationAction::Unsupported;
+ }
+ case GenericOpcode::G_SDIVREM:
+ case GenericOpcode::G_UDIVREM: {
+ const std::optional<LLT> Remainder = operandType(Function, Instruction, 1);
+ const std::optional<LLT> Left = operandType(Function, Instruction, 2);
+ const std::optional<LLT> Right = operandType(Function, Instruction, 3);
+ return Type && Remainder && Left && Right && isSupportedIntegerScalar(*Type) &&
+ *Remainder == *Type && *Left == *Type && *Right == *Type
+ ? LegalizationAction::Legal
+ : LegalizationAction::Unsupported;
+ }
+ case GenericOpcode::G_SMULH:
+ case GenericOpcode::G_UMULH: {
+ const std::optional<LLT> Left = operandType(Function, Instruction, 1);
+ const std::optional<LLT> Right = operandType(Function, Instruction, 2);
+ return Type && Left && Right && isNativeIntegerScalar(*Type) && *Left == *Type &&
+ *Right == *Type
+ ? LegalizationAction::Legal
+ : LegalizationAction::Unsupported;
+ }
+ case GenericOpcode::G_SMIN:
+ case GenericOpcode::G_SMAX:
+ case GenericOpcode::G_UMIN:
+ case GenericOpcode::G_UMAX: {
+ const std::optional<LLT> Left = operandType(Function, Instruction, 1);
+ const std::optional<LLT> Right = operandType(Function, Instruction, 2);
+ return Type && Left && Right && isSupportedIntegerScalar(*Type) && *Left == *Type &&
+ *Right == *Type
+ ? LegalizationAction::Legal
+ : LegalizationAction::Unsupported;
+ }
+ case GenericOpcode::G_UADDO:
+ case GenericOpcode::G_USUBO:
+ case GenericOpcode::G_SADDO:
+ case GenericOpcode::G_SSUBO:
+ case GenericOpcode::G_UMULO:
+ case GenericOpcode::G_SMULO: {
+ const std::optional<LLT> Overflow = operandType(Function, Instruction, 1);
+ const std::optional<LLT> Left = operandType(Function, Instruction, 2);
+ const std::optional<LLT> Right = operandType(Function, Instruction, 3);
+ return Type && Overflow && Left && Right && isNativeIntegerScalar(*Type) &&
+ isI1Type(*Overflow) && *Left == *Type && *Right == *Type
+ ? LegalizationAction::Legal
+ : LegalizationAction::Unsupported;
+ }
+ case GenericOpcode::G_UADDE:
+ case GenericOpcode::G_USUBE:
+ case GenericOpcode::G_SADDE:
+ case GenericOpcode::G_SSUBE: {
+ const std::optional<LLT> CarryOut = operandType(Function, Instruction, 1);
+ const std::optional<LLT> Left = operandType(Function, Instruction, 2);
+ const std::optional<LLT> Right = operandType(Function, Instruction, 3);
+ const std::optional<LLT> CarryIn = operandType(Function, Instruction, 4);
+ return Type && CarryOut && Left && Right && CarryIn && isNativeIntegerScalar(*Type) &&
+ isI1Type(*CarryOut) && isI1Type(*CarryIn) && *Left == *Type && *Right == *Type
+ ? LegalizationAction::Legal
+ : LegalizationAction::Unsupported;
+ }
+ case GenericOpcode::G_ATOMICRMW_XCHG:
+ case GenericOpcode::G_ATOMICRMW_ADD:
+ case GenericOpcode::G_ATOMICRMW_SUB:
+ case GenericOpcode::G_ATOMICRMW_AND:
+ case GenericOpcode::G_ATOMICRMW_OR:
+ case GenericOpcode::G_ATOMICRMW_XOR:
+ case GenericOpcode::G_ATOMICRMW_MAX:
+ case GenericOpcode::G_ATOMICRMW_MIN:
+ case GenericOpcode::G_ATOMICRMW_UMAX:
+ case GenericOpcode::G_ATOMICRMW_UMIN: {
+ const std::optional<LLT> Address = operandType(Function, Instruction, 1);
+ const std::optional<LLT> Value = operandType(Function, Instruction, 2);
+ const std::uint64_t Bytes = Type ? Type->scalarBitWidth() / 8U : 0;
+ return Type && Address && Value && isNativeIntegerScalar(*Type) && *Value == *Type &&
+ Address->isPointer() && Address->addressSpace() == 0 &&
+ Address->scalarBitWidth() == 64 &&
+ isSupportedAtomicMemoryAccess(Instruction, AtomicOperation::ReadModifyWrite,
+ Bytes)
+ ? LegalizationAction::Legal
+ : LegalizationAction::Unsupported;
+ }
+ case GenericOpcode::G_FENCE: {
+ const MachineOperand *OrderingOperand = Instruction.operandAt(0);
+ const MachineOperand *ScopeOperand = Instruction.operandAt(1);
+ const auto Ordering = OrderingOperand == nullptr
+ ? Expected<std::int64_t>(std::unexpected(
+ Error(ErrorCode::InvalidArgument, "missing fence ordering")))
+ : signedImmediate(*OrderingOperand);
+ const auto Scope = ScopeOperand == nullptr
+ ? Expected<std::int64_t>(std::unexpected(
+ Error(ErrorCode::InvalidArgument, "missing fence scope")))
+ : signedImmediate(*ScopeOperand);
+ return Ordering && Scope && *Ordering >= 4 && *Ordering <= 7 && *Scope == 0
+ ? LegalizationAction::Legal
+ : LegalizationAction::Unsupported;
+ }
+ case GenericOpcode::G_BRINDIRECT:
+ return operandType(Function, Instruction, 0)
+ .transform([](LLT Value) {
+ return Value.isPointer() && Value.addressSpace() == 0 &&
+ Value.scalarBitWidth() == 64;
+ })
+ .value_or(false)
+ ? LegalizationAction::Legal
+ : LegalizationAction::Unsupported;
+ case GenericOpcode::G_TRAP:
+ case GenericOpcode::G_DEBUGTRAP:
+ return Instruction.operandCount() == 0 ? LegalizationAction::Legal
+ : LegalizationAction::Unsupported;
+ case GenericOpcode::G_UBSANTRAP:
+ return Instruction.operandCount() == 1 ? LegalizationAction::Legal
+ : LegalizationAction::Unsupported;
+ case GenericOpcode::G_CONSTANT: {
+ const MachineOperand *Immediate = Instruction.operandAt(1);
+ return Type && isSupportedIntegerType(*Type) && Immediate != nullptr &&
+ (Immediate->kind() == MachineOperand::Kind::Immediate ||
+ Immediate->kind() == MachineOperand::Kind::IntegerImmediate)
+ ? LegalizationAction::Legal
+ : LegalizationAction::Unsupported;
+ }
+ case GenericOpcode::G_ADD:
+ case GenericOpcode::G_SUB:
+ case GenericOpcode::G_MUL:
+ case GenericOpcode::G_SDIV:
+ case GenericOpcode::G_UDIV:
+ case GenericOpcode::G_SREM:
+ case GenericOpcode::G_UREM:
+ case GenericOpcode::G_AND:
+ case GenericOpcode::G_OR:
+ case GenericOpcode::G_XOR:
+ case GenericOpcode::G_SHL:
+ case GenericOpcode::G_LSHR:
+ case GenericOpcode::G_ASHR: {
+ const std::optional<LLT> Left = operandType(Function, Instruction, 1);
+ const std::optional<LLT> Right = operandType(Function, Instruction, 2);
+ return Type && Left && Right && isSupportedIntegerScalar(*Type) && *Left == *Type &&
+ *Right == *Type
+ ? LegalizationAction::Legal
+ : LegalizationAction::Unsupported;
+ }
+ case GenericOpcode::G_PTR_ADD: {
+ const std::optional<LLT> Pointer = operandType(Function, Instruction, 1);
+ const std::optional<LLT> Offset = operandType(Function, Instruction, 2);
+ return Type && Pointer && Offset && Type->isPointer() && Type->addressSpace() == 0 &&
+ Type->scalarBitWidth() == 64 && *Pointer == *Type && Offset->isInteger() &&
+ Offset->scalarBitWidth() == 64
+ ? LegalizationAction::Legal
+ : LegalizationAction::Unsupported;
+ }
+ default:
+ break;
+ }
+
+ // Keep the unsupported policy mechanically synchronized with the complete
+ // target-independent opcode inventory. Supported cases return above.
+ switch (Generic) {
+#define RCC_CORE_OPCODE(Name)
+#define RCC_GENERIC_OPCODE(Name) case GenericOpcode::Name:
+#include "rcc/CodeGen/GenericOpcode.def"
+#undef RCC_GENERIC_OPCODE
+#undef RCC_CORE_OPCODE
+ return LegalizationAction::Unsupported;
+ case GenericOpcode::Invalid:
+ case GenericOpcode::CORE_PSEUDO_END:
+ case GenericOpcode::G_GENERIC_END:
+ default:
+ break;
+ }
+ return makeUnexpected(ErrorCode::InternalError,
+ "RISC-V legalizer received an invalid generic opcode value");
+ }
+
+ [[nodiscard]] Expected<void> lower(MachineInstr &, MachineFunction &, const TargetDescription &,
+ LegalizationAction) const override {
+ return makeUnexpected(ErrorCode::Unsupported,
+ "RISC-V legalizer has no lowering rule for this instruction");
+ }
+};
+
+class RISCVRegisterBankInfo final : public TargetRegisterBankInfo {
+public:
+ [[nodiscard]] Expected<RegisterBankID> selectBank(const MachineFunction &, Register,
+ const VirtualRegisterInfo &Info,
+ const MachineInstr *,
+ const TargetDescription &) const override {
+ if (!Info.Type) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V register-bank selection requires a virtual-register type");
+ }
+ if (isScalarGPRType(*Info.Type)) {
+ return RegisterBankID::fromValue(static_cast<std::uint32_t>(RegisterBank::GPR));
+ }
+ if (isRVFloatType(*Info.Type))
+ return RegisterBankID::fromValue(static_cast<std::uint32_t>(RegisterBank::FPR));
+ return makeUnexpected(ErrorCode::Unsupported,
+ "RISC-V RV64GC has no register bank for virtual register type '" +
+ Info.Type->toString() + "'");
+ }
+
+ [[nodiscard]] Expected<void>
+ constrainInstruction(MachineInstr &Instruction, MachineFunction &Function,
+ const TargetDescription &Target) const override {
+ std::vector<std::pair<Register, RegisterClassID>> PlannedClasses;
+ for (const MachineOperand &Operand : Instruction.operands()) {
+ const Register *Reg = Operand.getIf<Register>();
+ if (Reg == nullptr || !Reg->isVirtual())
+ continue;
+ auto Required = requiredRegisterClass(Function, *Reg, Target);
+ if (!Required)
+ return std::unexpected(Required.error());
+ if (std::ranges::find_if(PlannedClasses, [Reg](const auto &Entry) {
+ return Entry.first == *Reg;
+ }) == PlannedClasses.end())
+ PlannedClasses.emplace_back(*Reg, *Required);
+ }
+ for (const auto &[Reg, Required] : PlannedClasses) {
+ auto Info = Function.registerInfo().virtualRegisterInfo(Reg);
+ if (!Info)
+ return std::unexpected(Info.error());
+ if (Info->get().RegClass)
+ continue;
+ auto SetClass = Function.registerInfo().setRegisterClass(Reg, Required);
+ if (!SetClass)
+ return std::unexpected(SetClass.error());
+ }
+ return {};
+ }
+
+ [[nodiscard]] Expected<void> validateInstruction(const MachineInstr &Instruction,
+ const MachineFunction &Function,
+ const TargetDescription &Target) const override {
+ for (const MachineOperand &Operand : Instruction.operands()) {
+ const Register *Reg = Operand.getIf<Register>();
+ if (Reg == nullptr || !Reg->isVirtual())
+ continue;
+ auto Required = requiredRegisterClass(Function, *Reg, Target);
+ if (!Required)
+ return std::unexpected(Required.error());
+ }
+ return {};
+ }
+};
+
+class RISCVInstructionSelector final : public TargetInstructionSelector {
+public:
+ [[nodiscard]] Expected<void> canSelect(const MachineInstr &Instruction,
+ const MachineFunction &Function,
+ const TargetDescription &Target) const override {
+ if (!isGenericOpcode(Instruction.opcode()))
+ return {};
+ const GenericOpcode Generic = static_cast<GenericOpcode>(Instruction.opcode().value());
+ const std::optional<LLT> Type = resultType(Function, Instruction);
+ const auto Integer = [Type] { return Type && isSupportedIntegerType(*Type); };
+ const auto Float = [Type] { return Type && isRVFloatType(*Type); };
+ auto Classification = legalizerInfo().classify(Instruction, Function, Target);
+ if (!Classification)
+ return std::unexpected(Classification.error());
+ if (*Classification != LegalizationAction::Legal) {
+ std::string Message = "RISC-V selector cannot select opcode '" +
+ std::string(getGenericOpcodeName(Generic)) + "'";
+ if (Type)
+ Message += " with result type '" + Type->toString() + "'";
+ Message += ": the legalization rule explicitly marks this combination unsupported";
+ return makeUnexpected(ErrorCode::Unsupported, std::move(Message));
+ }
+ switch (Generic) {
+ case GenericOpcode::G_ADD:
+ case GenericOpcode::G_SUB:
+ case GenericOpcode::G_MUL:
+ case GenericOpcode::G_SDIV:
+ case GenericOpcode::G_UDIV:
+ case GenericOpcode::G_SREM:
+ case GenericOpcode::G_UREM:
+ case GenericOpcode::G_AND:
+ case GenericOpcode::G_OR:
+ case GenericOpcode::G_XOR:
+ case GenericOpcode::G_SHL:
+ case GenericOpcode::G_LSHR:
+ case GenericOpcode::G_ASHR:
+ case GenericOpcode::G_PTR_ADD:
+ case GenericOpcode::G_PTRMASK:
+ case GenericOpcode::G_CONSTANT:
+ if (!Integer())
+ return makeUnexpected(ErrorCode::Unsupported,
+ "RISC-V selector requires an i1-i64 integer or 64-bit pointer value");
+ if (Generic == GenericOpcode::G_CONSTANT) {
+ const MachineOperand *Immediate = Instruction.operandAt(1);
+ if (Immediate == nullptr || (Immediate->kind() != MachineOperand::Kind::Immediate &&
+ Immediate->kind() != MachineOperand::Kind::IntegerImmediate))
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V integer constant requires an immediate operand");
+ }
+ return {};
+ case GenericOpcode::G_LOAD:
+ case GenericOpcode::G_SEXTLOAD:
+ case GenericOpcode::G_ZEXTLOAD:
+ case GenericOpcode::G_FPEXTLOAD:
+ case GenericOpcode::G_FPTRUNCSTORE:
+ case GenericOpcode::G_STORE:
+ case GenericOpcode::G_BR:
+ case GenericOpcode::G_BRCOND:
+ return {};
+ case GenericOpcode::G_FADD:
+ case GenericOpcode::G_FSUB:
+ case GenericOpcode::G_FMUL:
+ case GenericOpcode::G_FDIV:
+ case GenericOpcode::G_FMA:
+ case GenericOpcode::G_FSQRT:
+ case GenericOpcode::G_FNEG:
+ case GenericOpcode::G_FABS:
+ case GenericOpcode::G_FCOPYSIGN:
+ if (!Float())
+ return makeUnexpected(ErrorCode::Unsupported,
+ "RISC-V selector requires f32 or f64 floating values");
+ return {};
+ case GenericOpcode::G_FCONSTANT: {
+ const MachineOperand *Immediate = Instruction.operandAt(1);
+ const auto *Floating =
+ Immediate == nullptr ? nullptr : Immediate->getIf<FloatingImmediateOperand>();
+ const std::optional<FloatSemantics> Semantics = Type ? Type->floatSemantics() : std::nullopt;
+ if (!Float() || Floating == nullptr || !Semantics || Floating->Bits.semantics() != *Semantics)
+ return makeUnexpected(ErrorCode::Unsupported,
+ "RISC-V selector requires a matching floating constant");
+ return {};
+ }
+ case GenericOpcode::G_ANYEXT:
+ case GenericOpcode::G_ASSERT_SEXT:
+ case GenericOpcode::G_ASSERT_ZEXT:
+ case GenericOpcode::G_ASSERT_ALIGN:
+ case GenericOpcode::G_FREEZE:
+ case GenericOpcode::G_CONSTANT_FOLD_BARRIER:
+ case GenericOpcode::G_PTRTOINT:
+ case GenericOpcode::G_INTTOPTR:
+ case GenericOpcode::G_BITCAST:
+ case GenericOpcode::G_TRUNC:
+ case GenericOpcode::G_SEXT:
+ case GenericOpcode::G_SEXT_INREG:
+ case GenericOpcode::G_ZEXT:
+ case GenericOpcode::G_ICMP:
+ case GenericOpcode::G_FCMP:
+ case GenericOpcode::G_FPEXT:
+ case GenericOpcode::G_FPTRUNC:
+ case GenericOpcode::G_FPTOSI:
+ case GenericOpcode::G_FPTOUI:
+ case GenericOpcode::G_SITOFP:
+ case GenericOpcode::G_UITOFP:
+ case GenericOpcode::G_FMINNUM:
+ case GenericOpcode::G_FMAXNUM:
+ case GenericOpcode::G_SDIVREM:
+ case GenericOpcode::G_UDIVREM:
+ case GenericOpcode::G_SMULH:
+ case GenericOpcode::G_UMULH:
+ case GenericOpcode::G_SMIN:
+ case GenericOpcode::G_SMAX:
+ case GenericOpcode::G_UMIN:
+ case GenericOpcode::G_UMAX:
+ case GenericOpcode::G_SELECT:
+ case GenericOpcode::G_ABS:
+ case GenericOpcode::G_UADDO:
+ case GenericOpcode::G_USUBO:
+ case GenericOpcode::G_SADDO:
+ case GenericOpcode::G_SSUBO:
+ case GenericOpcode::G_UMULO:
+ case GenericOpcode::G_SMULO:
+ case GenericOpcode::G_UADDE:
+ case GenericOpcode::G_USUBE:
+ case GenericOpcode::G_SADDE:
+ case GenericOpcode::G_SSUBE:
+ case GenericOpcode::G_ATOMICRMW_XCHG:
+ case GenericOpcode::G_ATOMICRMW_ADD:
+ case GenericOpcode::G_ATOMICRMW_SUB:
+ case GenericOpcode::G_ATOMICRMW_AND:
+ case GenericOpcode::G_ATOMICRMW_OR:
+ case GenericOpcode::G_ATOMICRMW_XOR:
+ case GenericOpcode::G_ATOMICRMW_MAX:
+ case GenericOpcode::G_ATOMICRMW_MIN:
+ case GenericOpcode::G_ATOMICRMW_UMAX:
+ case GenericOpcode::G_ATOMICRMW_UMIN:
+ case GenericOpcode::G_FENCE:
+ case GenericOpcode::G_BRINDIRECT:
+ case GenericOpcode::G_TRAP:
+ case GenericOpcode::G_DEBUGTRAP:
+ case GenericOpcode::G_UBSANTRAP:
+ case GenericOpcode::G_PHI:
+ case GenericOpcode::G_IMPLICIT_DEF:
+ return {};
+ default:
+ return makeUnexpected(ErrorCode::Unsupported,
+ "RISC-V instruction selector has no rule for generic opcode");
+ }
+ }
+
+ [[nodiscard]] Expected<void> select(MachineInstr &Instruction, MachineFunction &Function,
+ const TargetDescription &Target) const override {
+ auto Match = canSelect(Instruction, Function, Target);
+ if (!Match)
+ return std::unexpected(Match.error());
+ const GenericOpcode Generic = static_cast<GenericOpcode>(Instruction.opcode().value());
+ const std::optional<LLT> Type = resultType(Function, Instruction);
+
+ const auto SelectDirect = [&Instruction](InstructionOpcode Selected) {
+ return Instruction.setOpcode(targetOpcode(Selected));
+ };
+ switch (Generic) {
+ case GenericOpcode::G_ADD:
+ return selectIntegerBinary(Instruction, Type, InstructionOpcode::ADD,
+ InstructionOpcode::ADDW);
+ case GenericOpcode::G_SUB:
+ return selectIntegerBinary(Instruction, Type, InstructionOpcode::SUB,
+ InstructionOpcode::SUBW);
+ case GenericOpcode::G_MUL:
+ return selectIntegerBinary(Instruction, Type, InstructionOpcode::MUL,
+ InstructionOpcode::MULW);
+ case GenericOpcode::G_SDIV:
+ return selectNormalizedIntegerBinary(Instruction, Function, Target, Type,
+ InstructionOpcode::DIV, InstructionOpcode::DIVW, true,
+ true);
+ case GenericOpcode::G_UDIV:
+ return selectNormalizedIntegerBinary(Instruction, Function, Target, Type,
+ InstructionOpcode::DIVU, InstructionOpcode::DIVUW, false,
+ true);
+ case GenericOpcode::G_SREM:
+ return selectNormalizedIntegerBinary(Instruction, Function, Target, Type,
+ InstructionOpcode::REM, InstructionOpcode::REMW, true,
+ true);
+ case GenericOpcode::G_UREM:
+ return selectNormalizedIntegerBinary(Instruction, Function, Target, Type,
+ InstructionOpcode::REMU, InstructionOpcode::REMUW, false,
+ true);
+ case GenericOpcode::G_SDIVREM:
+ return selectDivRem(Instruction, Function, Target, Type, true);
+ case GenericOpcode::G_UDIVREM:
+ return selectDivRem(Instruction, Function, Target, Type, false);
+ case GenericOpcode::G_AND:
+ return SelectDirect(InstructionOpcode::AND);
+ case GenericOpcode::G_OR:
+ return SelectDirect(InstructionOpcode::OR);
+ case GenericOpcode::G_XOR:
+ return SelectDirect(InstructionOpcode::XOR);
+ case GenericOpcode::G_SHL:
+ return selectIntegerBinary(Instruction, Type, InstructionOpcode::SLL,
+ InstructionOpcode::SLLW);
+ case GenericOpcode::G_LSHR:
+ return selectNormalizedIntegerBinary(Instruction, Function, Target, Type,
+ InstructionOpcode::SRL, InstructionOpcode::SRLW, false,
+ false);
+ case GenericOpcode::G_ASHR:
+ return selectNormalizedIntegerBinary(Instruction, Function, Target, Type,
+ InstructionOpcode::SRA, InstructionOpcode::SRAW, true,
+ false);
+ case GenericOpcode::G_PTR_ADD:
+ return SelectDirect(InstructionOpcode::ADD);
+ case GenericOpcode::G_PTRMASK:
+ return SelectDirect(InstructionOpcode::AND);
+ case GenericOpcode::G_LOAD:
+ return selectLoad(Instruction, Type);
+ case GenericOpcode::G_SEXTLOAD:
+ return selectExtLoad(Instruction, Type, true);
+ case GenericOpcode::G_ZEXTLOAD:
+ return selectExtLoad(Instruction, Type, false);
+ case GenericOpcode::G_FPEXTLOAD:
+ return selectFloatExtLoad(Instruction, Function, Target);
+ case GenericOpcode::G_STORE:
+ return selectStore(Instruction, Function);
+ case GenericOpcode::G_FPTRUNCSTORE:
+ return selectFloatTruncStore(Instruction, Function, Target);
+ case GenericOpcode::G_BR:
+ return selectBranch(Instruction);
+ case GenericOpcode::G_BRCOND:
+ return selectConditionalBranch(Instruction);
+ case GenericOpcode::G_CONSTANT:
+ return selectIntegerConstant(Instruction, Function, Target, Type);
+ case GenericOpcode::G_FCONSTANT:
+ return selectFloatingConstant(Instruction, Function, Target, Type);
+ case GenericOpcode::G_ASSERT_SEXT:
+ case GenericOpcode::G_ASSERT_ZEXT:
+ case GenericOpcode::G_ASSERT_ALIGN:
+ if (Instruction.operandCount() == 3) {
+ auto EraseImmediate = Instruction.eraseOperand(2);
+ if (!EraseImmediate)
+ return std::unexpected(EraseImmediate.error());
+ }
+ return Instruction.setOpcode(opcode(GenericOpcode::COPY));
+ case GenericOpcode::G_FREEZE:
+ case GenericOpcode::G_CONSTANT_FOLD_BARRIER:
+ case GenericOpcode::G_PTRTOINT:
+ case GenericOpcode::G_INTTOPTR:
+ return Instruction.setOpcode(opcode(GenericOpcode::COPY));
+ case GenericOpcode::G_BITCAST:
+ return selectBitcast(Instruction, Function);
+ case GenericOpcode::G_FADD:
+ return selectRoundingBinary(Instruction, Type, InstructionOpcode::FADD_S,
+ InstructionOpcode::FADD_D);
+ case GenericOpcode::G_FSUB:
+ return selectRoundingBinary(Instruction, Type, InstructionOpcode::FSUB_S,
+ InstructionOpcode::FSUB_D);
+ case GenericOpcode::G_FMUL:
+ return selectRoundingBinary(Instruction, Type, InstructionOpcode::FMUL_S,
+ InstructionOpcode::FMUL_D);
+ case GenericOpcode::G_FDIV:
+ return selectRoundingBinary(Instruction, Type, InstructionOpcode::FDIV_S,
+ InstructionOpcode::FDIV_D);
+ case GenericOpcode::G_FMA:
+ return selectRoundingTernary(Instruction, Type, InstructionOpcode::FMADD_S,
+ InstructionOpcode::FMADD_D);
+ case GenericOpcode::G_FSQRT:
+ return selectRoundingUnary(Instruction, Type, InstructionOpcode::FSQRT_S,
+ InstructionOpcode::FSQRT_D);
+ case GenericOpcode::G_FNEG:
+ return selectSignInjection(Instruction, Type, InstructionOpcode::FSGNJN_S,
+ InstructionOpcode::FSGNJN_D);
+ case GenericOpcode::G_FABS:
+ return selectSignInjection(Instruction, Type, InstructionOpcode::FSGNJX_S,
+ InstructionOpcode::FSGNJX_D);
+ case GenericOpcode::G_FCOPYSIGN:
+ return selectFloatingMinMax(Instruction, Type, InstructionOpcode::FSGNJ_S,
+ InstructionOpcode::FSGNJ_D);
+ case GenericOpcode::G_ANYEXT:
+ case GenericOpcode::G_SEXT:
+ case GenericOpcode::G_ZEXT:
+ return selectIntegerExtension(Instruction, Function, Target, Generic, Type);
+ case GenericOpcode::G_TRUNC:
+ return Instruction.setOpcode(opcode(GenericOpcode::COPY));
+ case GenericOpcode::G_SEXT_INREG:
+ return selectSignExtendInRegister(Instruction, Function, Target, Type);
+ case GenericOpcode::G_ICMP:
+ return selectIntegerCompare(Instruction, Function, Target);
+ case GenericOpcode::G_FCMP:
+ return selectFloatingCompare(Instruction, Function, Target);
+ case GenericOpcode::G_FPEXT:
+ return selectRoundingUnary(Instruction, Type, InstructionOpcode::FCVT_D_S,
+ InstructionOpcode::FCVT_D_S);
+ case GenericOpcode::G_FPTRUNC:
+ return selectRoundingUnary(Instruction, Type, InstructionOpcode::FCVT_S_D,
+ InstructionOpcode::FCVT_S_D);
+ case GenericOpcode::G_FPTOSI:
+ case GenericOpcode::G_FPTOUI:
+ return selectFloatToInteger(Instruction, Function, Type, Generic == GenericOpcode::G_FPTOUI);
+ case GenericOpcode::G_SITOFP:
+ case GenericOpcode::G_UITOFP:
+ return selectIntegerToFloat(Instruction, Function, Target, Type,
+ Generic == GenericOpcode::G_UITOFP);
+ case GenericOpcode::G_FMINNUM:
+ return selectFloatingMinMax(Instruction, Type, InstructionOpcode::FMIN_S,
+ InstructionOpcode::FMIN_D);
+ case GenericOpcode::G_FMAXNUM:
+ return selectFloatingMinMax(Instruction, Type, InstructionOpcode::FMAX_S,
+ InstructionOpcode::FMAX_D);
+ case GenericOpcode::G_SMULH:
+ return selectMultiplyHigh(Instruction, Function, Target, Type, true);
+ case GenericOpcode::G_UMULH:
+ return selectMultiplyHigh(Instruction, Function, Target, Type, false);
+ case GenericOpcode::G_SMIN:
+ case GenericOpcode::G_SMAX:
+ case GenericOpcode::G_UMIN:
+ case GenericOpcode::G_UMAX:
+ return selectIntegerMinMax(Instruction, Function, Target, Type, Generic);
+ case GenericOpcode::G_SELECT:
+ return selectValue(Instruction, Function, Target, Type);
+ case GenericOpcode::G_ABS:
+ return selectAbsoluteValue(Instruction, Function, Target, Type);
+ case GenericOpcode::G_UADDO:
+ case GenericOpcode::G_USUBO:
+ case GenericOpcode::G_SADDO:
+ case GenericOpcode::G_SSUBO:
+ case GenericOpcode::G_UMULO:
+ case GenericOpcode::G_SMULO:
+ return selectOverflow(Instruction, Function, Target, Type, Generic);
+ case GenericOpcode::G_UADDE:
+ case GenericOpcode::G_USUBE:
+ case GenericOpcode::G_SADDE:
+ case GenericOpcode::G_SSUBE:
+ return selectExtendedOverflow(Instruction, Function, Target, Type, Generic);
+ case GenericOpcode::G_ATOMICRMW_XCHG:
+ case GenericOpcode::G_ATOMICRMW_ADD:
+ case GenericOpcode::G_ATOMICRMW_SUB:
+ case GenericOpcode::G_ATOMICRMW_AND:
+ case GenericOpcode::G_ATOMICRMW_OR:
+ case GenericOpcode::G_ATOMICRMW_XOR:
+ case GenericOpcode::G_ATOMICRMW_MAX:
+ case GenericOpcode::G_ATOMICRMW_MIN:
+ case GenericOpcode::G_ATOMICRMW_UMAX:
+ case GenericOpcode::G_ATOMICRMW_UMIN:
+ return selectAtomicRMW(Instruction, Function, Target, Type, Generic);
+ case GenericOpcode::G_FENCE:
+ return selectFence(Instruction);
+ case GenericOpcode::G_BRINDIRECT:
+ return selectIndirectBranch(Instruction);
+ case GenericOpcode::G_TRAP:
+ case GenericOpcode::G_DEBUGTRAP:
+ case GenericOpcode::G_UBSANTRAP:
+ return selectTrap(Instruction);
+ case GenericOpcode::G_PHI:
+ return Instruction.setOpcode(opcode(GenericOpcode::PHI));
+ case GenericOpcode::G_IMPLICIT_DEF:
+ return Instruction.setOpcode(opcode(GenericOpcode::IMPLICIT_DEF));
+ case GenericOpcode::COPY:
+ case GenericOpcode::IMPLICIT_DEF:
+ case GenericOpcode::PHI:
+ case GenericOpcode::REG_SEQUENCE:
+ case GenericOpcode::INSERT_SUBREG:
+ case GenericOpcode::EXTRACT_SUBREG:
+ case GenericOpcode::SUBREG_TO_REG:
+ case GenericOpcode::KILL:
+ case GenericOpcode::BUNDLE:
+ case GenericOpcode::INLINEASM:
+ case GenericOpcode::CORE_PSEUDO_END:
+ break;
+ default:
+ return makeUnexpected(ErrorCode::Unsupported,
+ "RISC-V instruction selector has no rule for generic opcode");
+ }
+ return makeUnexpected(ErrorCode::Unsupported,
+ "RISC-V instruction selector has no rule for generic opcode");
+ }
+
+private:
+ [[nodiscard]] static Expected<void> replaceImmediate(MachineInstr &Instruction, std::size_t Index,
+ std::int64_t Value) {
+ if (Instruction.operandAt(Index) == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V instruction is missing an immediate operand");
+ auto Erased = Instruction.eraseOperand(Index);
+ if (!Erased)
+ return std::unexpected(Erased.error());
+ return Instruction.insertOperand(Index, MachineOperand::immediate(Value));
+ }
+
+ [[nodiscard]] static Expected<MachineInstr *>
+ buildBinaryAfter(MachineIRBuilder &Builder, MachineInstr &Anchor, InstructionOpcode Opcode,
+ Register Def, Register Left, Register Right) {
+ auto Point = Builder.setInsertPointAfter(Anchor);
+ if (!Point)
+ return std::unexpected(Point.error());
+ auto Pending = Builder.buildInstr(targetOpcode(Opcode));
+ if (!Pending)
+ return std::unexpected(Pending.error());
+ auto DefResult = Pending->addDef(Def);
+ if (!DefResult)
+ return std::unexpected(DefResult.error());
+ auto LeftResult = Pending->addUse(Left);
+ if (!LeftResult)
+ return std::unexpected(LeftResult.error());
+ auto RightResult = Pending->addUse(Right);
+ if (!RightResult)
+ return std::unexpected(RightResult.error());
+ auto Built = Pending->build();
+ if (!Built)
+ return std::unexpected(Built.error());
+ return *Built;
+ }
+
+ [[nodiscard]] static Expected<MachineInstr *>
+ buildImmediateAfter(MachineIRBuilder &Builder, MachineInstr &Anchor, InstructionOpcode Opcode,
+ Register Def, Register Source, std::int64_t Value) {
+ auto Point = Builder.setInsertPointAfter(Anchor);
+ if (!Point)
+ return std::unexpected(Point.error());
+ auto Pending = Builder.buildInstr(targetOpcode(Opcode));
+ if (!Pending)
+ return std::unexpected(Pending.error());
+ auto DefResult = Pending->addDef(Def);
+ if (!DefResult)
+ return std::unexpected(DefResult.error());
+ auto SourceResult = Pending->addUse(Source);
+ if (!SourceResult)
+ return std::unexpected(SourceResult.error());
+ auto ImmediateResult = Pending->addImmediate(Value);
+ if (!ImmediateResult)
+ return std::unexpected(ImmediateResult.error());
+ auto Built = Pending->build();
+ if (!Built)
+ return std::unexpected(Built.error());
+ return *Built;
+ }
+
+ [[nodiscard]] static Expected<MachineInstr *>
+ buildImmediateBefore(MachineIRBuilder &Builder, MachineInstr &Anchor, InstructionOpcode Opcode,
+ Register Def, Register Source, std::int64_t Value) {
+ auto Point = Builder.setInsertPointBefore(Anchor);
+ if (!Point)
+ return std::unexpected(Point.error());
+ auto Pending = Builder.buildInstr(targetOpcode(Opcode));
+ if (!Pending)
+ return std::unexpected(Pending.error());
+ auto DefResult = Pending->addDef(Def);
+ if (!DefResult)
+ return std::unexpected(DefResult.error());
+ auto SourceResult = Pending->addUse(Source);
+ if (!SourceResult)
+ return std::unexpected(SourceResult.error());
+ auto ImmediateResult = Pending->addImmediate(Value);
+ if (!ImmediateResult)
+ return std::unexpected(ImmediateResult.error());
+ auto Built = Pending->build();
+ if (!Built)
+ return std::unexpected(Built.error());
+ return *Built;
+ }
+
+ [[nodiscard]] static Expected<Register>
+ normalizeIntegerBefore(MachineInstr &Anchor, MachineFunction &Function,
+ const TargetDescription &Target, Register Source, LLT Type, bool Signed) {
+ if (!isSupportedIntegerScalar(Type))
+ return makeUnexpected(ErrorCode::Unsupported,
+ "RISC-V integer normalization requires a scalar i1-i64 type");
+ const std::uint32_t Bits = Type.scalarBitWidth();
+ if (Bits == 64)
+ return Source;
+
+ MachineIRBuilder Builder(Function, Target);
+ if (Signed && Bits == 32) {
+ auto Result = createTemporary(Function, Type, Target);
+ if (!Result)
+ return std::unexpected(Result.error());
+ auto Extended =
+ buildImmediateBefore(Builder, Anchor, InstructionOpcode::ADDIW, *Result, Source, 0);
+ if (!Extended)
+ return std::unexpected(Extended.error());
+ return *Result;
+ }
+
+ const std::uint32_t ContainerBits = Bits < 32 ? 32 : 64;
+ const std::int64_t Shift = static_cast<std::int64_t>(ContainerBits - Bits);
+ auto Shifted = createTemporary(Function, Type, Target);
+ auto Result = createTemporary(Function, Type, Target);
+ if (!Shifted || !Result)
+ return std::unexpected(!Shifted ? Shifted.error() : Result.error());
+ const InstructionOpcode ShiftLeft =
+ ContainerBits == 32 ? InstructionOpcode::SLLIW : InstructionOpcode::SLLI;
+ const InstructionOpcode ShiftRight =
+ Signed ? (ContainerBits == 32 ? InstructionOpcode::SRAIW : InstructionOpcode::SRAI)
+ : (ContainerBits == 32 ? InstructionOpcode::SRLIW : InstructionOpcode::SRLI);
+ auto Left = buildImmediateBefore(Builder, Anchor, ShiftLeft, *Shifted, Source, Shift);
+ if (!Left)
+ return std::unexpected(Left.error());
+ auto Right = buildImmediateBefore(Builder, Anchor, ShiftRight, *Result, *Shifted, Shift);
+ if (!Right)
+ return std::unexpected(Right.error());
+ return *Result;
+ }
+
+ [[nodiscard]] static Expected<MachineInstr *> buildUnaryAfter(MachineIRBuilder &Builder,
+ MachineInstr &Anchor,
+ InstructionOpcode Opcode,
+ Register Def, Register Source) {
+ auto Point = Builder.setInsertPointAfter(Anchor);
+ if (!Point)
+ return std::unexpected(Point.error());
+ auto Pending = Builder.buildInstr(targetOpcode(Opcode));
+ if (!Pending)
+ return std::unexpected(Pending.error());
+ auto DefResult = Pending->addDef(Def);
+ if (!DefResult)
+ return std::unexpected(DefResult.error());
+ auto SourceResult = Pending->addUse(Source);
+ if (!SourceResult)
+ return std::unexpected(SourceResult.error());
+ auto Built = Pending->build();
+ if (!Built)
+ return std::unexpected(Built.error());
+ return *Built;
+ }
+
+ [[nodiscard]] static Expected<MachineInstr *>
+ buildGPRSelectAfter(MachineIRBuilder &Builder, MachineInstr &Anchor, MachineFunction &Function,
+ const TargetDescription &Target, LLT ValueType, Register Destination,
+ Register Condition, Register TrueValue, Register FalseValue) {
+ auto IntegerType = LLT::makeInteger(ValueType.scalarBitWidth());
+ if (!IntegerType)
+ return std::unexpected(IntegerType.error());
+ auto Mask = createTemporary(Function, *IntegerType, Target);
+ auto Difference = createTemporary(Function, *IntegerType, Target);
+ auto MaskedDifference = createTemporary(Function, *IntegerType, Target);
+ if (!Mask || !Difference || !MaskedDifference) {
+ return std::unexpected(
+ (!Mask ? Mask.error() : (!Difference ? Difference.error() : MaskedDifference.error())));
+ }
+ auto Zero = physicalRegister(PhysicalRegisterID::X0);
+ if (!Zero)
+ return std::unexpected(Zero.error());
+ const InstructionOpcode Subtract =
+ ValueType.scalarBitWidth() == 32 ? InstructionOpcode::SUBW : InstructionOpcode::SUB;
+ auto MaskInstruction = buildBinaryAfter(Builder, Anchor, Subtract, *Mask,
+ Register::fromPhysical(*Zero), Condition);
+ if (!MaskInstruction)
+ return std::unexpected(MaskInstruction.error());
+ auto DifferenceInstruction = buildBinaryAfter(
+ Builder, **MaskInstruction, InstructionOpcode::XOR, *Difference, TrueValue, FalseValue);
+ if (!DifferenceInstruction)
+ return std::unexpected(DifferenceInstruction.error());
+ auto MaskInstructionResult =
+ buildBinaryAfter(Builder, **DifferenceInstruction, InstructionOpcode::AND,
+ *MaskedDifference, *Difference, *Mask);
+ if (!MaskInstructionResult)
+ return std::unexpected(MaskInstructionResult.error());
+ return buildBinaryAfter(Builder, **MaskInstructionResult, InstructionOpcode::XOR, Destination,
+ FalseValue, *MaskedDifference);
+ }
+
+ [[nodiscard]] static Expected<void> selectDivRem(MachineInstr &Instruction,
+ MachineFunction &Function,
+ const TargetDescription &Target,
+ std::optional<LLT> Type, bool Signed) {
+ const MachineOperand *RemainderOperand = Instruction.operandAt(1);
+ const MachineOperand *LeftOperand = Instruction.operandAt(2);
+ const MachineOperand *RightOperand = Instruction.operandAt(3);
+ const Register *Remainder =
+ RemainderOperand == nullptr ? nullptr : RemainderOperand->getIf<Register>();
+ const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf<Register>();
+ const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf<Register>();
+ if (!Type || Remainder == nullptr || Left == nullptr || Right == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V division-with-remainder has invalid operands");
+ const Register RemainderRegister = *Remainder;
+ auto LeftRegister = normalizeIntegerBefore(Instruction, Function, Target, *Left, *Type, Signed);
+ auto RightRegister =
+ normalizeIntegerBefore(Instruction, Function, Target, *Right, *Type, Signed);
+ if (!LeftRegister || !RightRegister)
+ return std::unexpected(!LeftRegister ? LeftRegister.error() : RightRegister.error());
+ auto RemoveRemainder = Instruction.eraseOperand(1);
+ if (!RemoveRemainder)
+ return std::unexpected(RemoveRemainder.error());
+ auto SetLeft = Instruction.operandAt(1)->setRegister(*LeftRegister);
+ if (!SetLeft)
+ return std::unexpected(SetLeft.error());
+ auto SetRight = Instruction.operandAt(2)->setRegister(*RightRegister);
+ if (!SetRight)
+ return std::unexpected(SetRight.error());
+ const bool IsWord = Type->scalarBitWidth() <= 32;
+ const InstructionOpcode Division =
+ IsWord ? (Signed ? InstructionOpcode::DIVW : InstructionOpcode::DIVUW)
+ : (Signed ? InstructionOpcode::DIV : InstructionOpcode::DIVU);
+ const InstructionOpcode RemainderOpcode =
+ IsWord ? (Signed ? InstructionOpcode::REMW : InstructionOpcode::REMUW)
+ : (Signed ? InstructionOpcode::REM : InstructionOpcode::REMU);
+ auto SetDivision = Instruction.setOpcode(targetOpcode(Division));
+ if (!SetDivision)
+ return std::unexpected(SetDivision.error());
+ MachineIRBuilder Builder(Function, Target);
+ auto BuiltRemainder = buildBinaryAfter(Builder, Instruction, RemainderOpcode, RemainderRegister,
+ *LeftRegister, *RightRegister);
+ if (!BuiltRemainder)
+ return std::unexpected(BuiltRemainder.error());
+ return {};
+ }
+
+ [[nodiscard]] static Expected<void> selectMultiplyHigh(MachineInstr &Instruction,
+ MachineFunction &Function,
+ const TargetDescription &Target,
+ std::optional<LLT> Type, bool Signed) {
+ if (!Type)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V high multiplication has no result type");
+ if (Type->scalarBitWidth() == 64) {
+ return Instruction.setOpcode(
+ targetOpcode(Signed ? InstructionOpcode::MULH : InstructionOpcode::MULHU));
+ }
+
+ const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+ const MachineOperand *LeftOperand = Instruction.operandAt(1);
+ const MachineOperand *RightOperand = Instruction.operandAt(2);
+ const Register *Destination =
+ DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+ const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf<Register>();
+ const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf<Register>();
+ if (Destination == nullptr || Left == nullptr || Right == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V high multiplication has invalid register operands");
+ const Register DestinationRegister = *Destination;
+ const Register RightRegister = *Right;
+ auto I64 = LLT::makeInteger(64);
+ if (!I64)
+ return std::unexpected(I64.error());
+ auto ExtendedLeft = createTemporary(Function, *I64, Target);
+ auto ExtendedRight = createTemporary(Function, *I64, Target);
+ auto Product = createTemporary(Function, *I64, Target);
+ if (!ExtendedLeft || !ExtendedRight || !Product) {
+ return std::unexpected(!ExtendedLeft
+ ? ExtendedLeft.error()
+ : (!ExtendedRight ? ExtendedRight.error() : Product.error()));
+ }
+
+ auto RemoveRight = Instruction.eraseOperand(2);
+ if (!RemoveRight)
+ return std::unexpected(RemoveRight.error());
+ MachineIRBuilder Builder(Function, Target);
+ MachineInstr *Anchor = &Instruction;
+ if (Signed) {
+ auto SetDestination = Instruction.operandAt(0)->setRegister(*ExtendedLeft);
+ if (!SetDestination)
+ return std::unexpected(SetDestination.error());
+ auto Immediate = appendImmediate(Instruction, 0);
+ if (!Immediate)
+ return std::unexpected(Immediate.error());
+ auto SetOpcode = Instruction.setOpcode(targetOpcode(InstructionOpcode::ADDIW));
+ if (!SetOpcode)
+ return std::unexpected(SetOpcode.error());
+ auto ExtendRight = buildImmediateAfter(Builder, *Anchor, InstructionOpcode::ADDIW,
+ *ExtendedRight, RightRegister, 0);
+ if (!ExtendRight)
+ return std::unexpected(ExtendRight.error());
+ Anchor = *ExtendRight;
+ } else {
+ auto ShiftedLeft = createTemporary(Function, *I64, Target);
+ auto ShiftedRight = createTemporary(Function, *I64, Target);
+ if (!ShiftedLeft || !ShiftedRight)
+ return std::unexpected(!ShiftedLeft ? ShiftedLeft.error() : ShiftedRight.error());
+ auto SetDestination = Instruction.operandAt(0)->setRegister(*ShiftedLeft);
+ if (!SetDestination)
+ return std::unexpected(SetDestination.error());
+ auto Immediate = appendImmediate(Instruction, 32);
+ if (!Immediate)
+ return std::unexpected(Immediate.error());
+ auto SetOpcode = Instruction.setOpcode(targetOpcode(InstructionOpcode::SLLI));
+ if (!SetOpcode)
+ return std::unexpected(SetOpcode.error());
+ auto ExtendLeft = buildImmediateAfter(Builder, *Anchor, InstructionOpcode::SRLI,
+ *ExtendedLeft, *ShiftedLeft, 32);
+ if (!ExtendLeft)
+ return std::unexpected(ExtendLeft.error());
+ auto ShiftRight = buildImmediateAfter(Builder, **ExtendLeft, InstructionOpcode::SLLI,
+ *ShiftedRight, RightRegister, 32);
+ if (!ShiftRight)
+ return std::unexpected(ShiftRight.error());
+ auto ExtendRight = buildImmediateAfter(Builder, **ShiftRight, InstructionOpcode::SRLI,
+ *ExtendedRight, *ShiftedRight, 32);
+ if (!ExtendRight)
+ return std::unexpected(ExtendRight.error());
+ Anchor = *ExtendRight;
+ }
+ auto Multiply = buildBinaryAfter(Builder, *Anchor, InstructionOpcode::MUL, *Product,
+ *ExtendedLeft, *ExtendedRight);
+ if (!Multiply)
+ return std::unexpected(Multiply.error());
+ auto Extract = buildImmediateAfter(Builder, **Multiply,
+ Signed ? InstructionOpcode::SRAI : InstructionOpcode::SRLI,
+ DestinationRegister, *Product, 32);
+ if (!Extract)
+ return std::unexpected(Extract.error());
+ return {};
+ }
+
+ [[nodiscard]] static Expected<void> selectIntegerMinMax(MachineInstr &Instruction,
+ MachineFunction &Function,
+ const TargetDescription &Target,
+ std::optional<LLT> Type,
+ GenericOpcode Generic) {
+ const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+ const MachineOperand *LeftOperand = Instruction.operandAt(1);
+ const MachineOperand *RightOperand = Instruction.operandAt(2);
+ const Register *Destination =
+ DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+ const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf<Register>();
+ const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf<Register>();
+ if (!Type || Destination == nullptr || Left == nullptr || Right == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V integer min/max has invalid operands");
+ auto I1 = LLT::makeInteger(1);
+ if (!I1)
+ return std::unexpected(I1.error());
+ auto Condition = createTemporary(Function, *I1, Target);
+ if (!Condition)
+ return std::unexpected(Condition.error());
+ const Register DestinationRegister = *Destination;
+ const Register LeftRegister = *Left;
+ const Register RightRegister = *Right;
+ const bool Unsigned = Generic == GenericOpcode::G_UMIN || Generic == GenericOpcode::G_UMAX;
+ auto NormalizedLeft =
+ normalizeIntegerBefore(Instruction, Function, Target, LeftRegister, *Type, !Unsigned);
+ auto NormalizedRight =
+ normalizeIntegerBefore(Instruction, Function, Target, RightRegister, *Type, !Unsigned);
+ if (!NormalizedLeft || !NormalizedRight)
+ return std::unexpected(!NormalizedLeft ? NormalizedLeft.error() : NormalizedRight.error());
+ auto SetCondition = Instruction.operandAt(0)->setRegister(*Condition);
+ if (!SetCondition)
+ return std::unexpected(SetCondition.error());
+ auto SetLeft = Instruction.operandAt(1)->setRegister(*NormalizedLeft);
+ if (!SetLeft)
+ return std::unexpected(SetLeft.error());
+ auto SetRight = Instruction.operandAt(2)->setRegister(*NormalizedRight);
+ if (!SetRight)
+ return std::unexpected(SetRight.error());
+ auto SetCompare = Instruction.setOpcode(
+ targetOpcode(Unsigned ? InstructionOpcode::SLTU : InstructionOpcode::SLT));
+ if (!SetCompare)
+ return std::unexpected(SetCompare.error());
+ const bool IsMaximum = Generic == GenericOpcode::G_SMAX || Generic == GenericOpcode::G_UMAX;
+ const Register TrueValue = IsMaximum ? RightRegister : LeftRegister;
+ const Register FalseValue = IsMaximum ? LeftRegister : RightRegister;
+ MachineIRBuilder Builder(Function, Target);
+ auto Selected = buildGPRSelectAfter(Builder, Instruction, Function, Target, *Type,
+ DestinationRegister, *Condition, TrueValue, FalseValue);
+ if (!Selected)
+ return std::unexpected(Selected.error());
+ return {};
+ }
+
+ [[nodiscard]] static Expected<void> selectValue(MachineInstr &Instruction,
+ MachineFunction &Function,
+ const TargetDescription &Target,
+ std::optional<LLT> Type) {
+ const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+ const MachineOperand *ConditionOperand = Instruction.operandAt(1);
+ const MachineOperand *TrueOperand = Instruction.operandAt(2);
+ const MachineOperand *FalseOperand = Instruction.operandAt(3);
+ const Register *Destination =
+ DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+ const Register *Condition =
+ ConditionOperand == nullptr ? nullptr : ConditionOperand->getIf<Register>();
+ const Register *TrueValue = TrueOperand == nullptr ? nullptr : TrueOperand->getIf<Register>();
+ const Register *FalseValue =
+ FalseOperand == nullptr ? nullptr : FalseOperand->getIf<Register>();
+ if (!Type || Destination == nullptr || Condition == nullptr || TrueValue == nullptr ||
+ FalseValue == nullptr) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V value selection has invalid operands");
+ }
+ const Register DestinationRegister = *Destination;
+ const Register TrueRegister = *TrueValue;
+ const Register FalseRegister = *FalseValue;
+ auto I1 = LLT::makeInteger(1);
+ if (!I1)
+ return std::unexpected(I1.error());
+ auto NormalizedCondition = createTemporary(Function, *I1, Target);
+ if (!NormalizedCondition)
+ return std::unexpected(NormalizedCondition.error());
+ auto SetCondition = Instruction.operandAt(0)->setRegister(*NormalizedCondition);
+ if (!SetCondition)
+ return std::unexpected(SetCondition.error());
+ auto EraseFalse = Instruction.eraseOperand(3);
+ if (!EraseFalse)
+ return std::unexpected(EraseFalse.error());
+ auto EraseTrue = Instruction.eraseOperand(2);
+ if (!EraseTrue)
+ return std::unexpected(EraseTrue.error());
+ auto Mask = appendImmediate(Instruction, 1);
+ if (!Mask)
+ return std::unexpected(Mask.error());
+ auto SetNormalize = Instruction.setOpcode(targetOpcode(InstructionOpcode::ANDI));
+ if (!SetNormalize)
+ return std::unexpected(SetNormalize.error());
+
+ MachineIRBuilder Builder(Function, Target);
+ if (!Type->isFloat()) {
+ auto Selected =
+ buildGPRSelectAfter(Builder, Instruction, Function, Target, *Type, DestinationRegister,
+ *NormalizedCondition, TrueRegister, FalseRegister);
+ if (!Selected)
+ return std::unexpected(Selected.error());
+ return {};
+ }
+
+ auto IntegerType = LLT::makeInteger(Type->scalarBitWidth());
+ if (!IntegerType)
+ return std::unexpected(IntegerType.error());
+ auto IntegerTrue = createTemporary(Function, *IntegerType, Target);
+ auto IntegerFalse = createTemporary(Function, *IntegerType, Target);
+ auto IntegerResult = createTemporary(Function, *IntegerType, Target);
+ if (!IntegerTrue || !IntegerFalse || !IntegerResult) {
+ return std::unexpected(!IntegerTrue
+ ? IntegerTrue.error()
+ : (!IntegerFalse ? IntegerFalse.error() : IntegerResult.error()));
+ }
+ const InstructionOpcode ToInteger =
+ Type->scalarBitWidth() == 32 ? InstructionOpcode::FMV_X_W : InstructionOpcode::FMV_X_D;
+ const InstructionOpcode ToFloat =
+ Type->scalarBitWidth() == 32 ? InstructionOpcode::FMV_W_X : InstructionOpcode::FMV_D_X;
+ auto MoveTrue = buildUnaryAfter(Builder, Instruction, ToInteger, *IntegerTrue, TrueRegister);
+ if (!MoveTrue)
+ return std::unexpected(MoveTrue.error());
+ auto MoveFalse = buildUnaryAfter(Builder, **MoveTrue, ToInteger, *IntegerFalse, FalseRegister);
+ if (!MoveFalse)
+ return std::unexpected(MoveFalse.error());
+ auto Selected =
+ buildGPRSelectAfter(Builder, **MoveFalse, Function, Target, *IntegerType, *IntegerResult,
+ *NormalizedCondition, *IntegerTrue, *IntegerFalse);
+ if (!Selected)
+ return std::unexpected(Selected.error());
+ auto MoveResult =
+ buildUnaryAfter(Builder, **Selected, ToFloat, DestinationRegister, *IntegerResult);
+ if (!MoveResult)
+ return std::unexpected(MoveResult.error());
+ return {};
+ }
+
+ [[nodiscard]] static Expected<void> selectAbsoluteValue(MachineInstr &Instruction,
+ MachineFunction &Function,
+ const TargetDescription &Target,
+ std::optional<LLT> Type) {
+ const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+ const MachineOperand *SourceOperand = Instruction.operandAt(1);
+ const Register *Destination =
+ DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+ const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>();
+ if (!Type || Destination == nullptr || Source == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V absolute value has invalid operands");
+ const Register DestinationRegister = *Destination;
+ const Register SourceRegister = *Source;
+ auto NormalizedSource =
+ normalizeIntegerBefore(Instruction, Function, Target, SourceRegister, *Type, true);
+ if (!NormalizedSource)
+ return std::unexpected(NormalizedSource.error());
+ auto SignMask = createTemporary(Function, *Type, Target);
+ auto Flipped = createTemporary(Function, *Type, Target);
+ if (!SignMask || !Flipped)
+ return std::unexpected(!SignMask ? SignMask.error() : Flipped.error());
+ auto SetMask = Instruction.operandAt(0)->setRegister(*SignMask);
+ if (!SetMask)
+ return std::unexpected(SetMask.error());
+ auto SetSource = Instruction.operandAt(1)->setRegister(*NormalizedSource);
+ if (!SetSource)
+ return std::unexpected(SetSource.error());
+ const std::uint32_t ContainerBits = Type->scalarBitWidth() <= 32 ? 32 : 64;
+ auto Shift = appendImmediate(Instruction, ContainerBits - 1U);
+ if (!Shift)
+ return std::unexpected(Shift.error());
+ const bool IsWord = Type->scalarBitWidth() <= 32;
+ auto SetShift = Instruction.setOpcode(
+ targetOpcode(IsWord ? InstructionOpcode::SRAIW : InstructionOpcode::SRAI));
+ if (!SetShift)
+ return std::unexpected(SetShift.error());
+ MachineIRBuilder Builder(Function, Target);
+ auto Flip = buildBinaryAfter(Builder, Instruction, InstructionOpcode::XOR, *Flipped,
+ *NormalizedSource, *SignMask);
+ if (!Flip)
+ return std::unexpected(Flip.error());
+ auto Result =
+ buildBinaryAfter(Builder, **Flip, IsWord ? InstructionOpcode::SUBW : InstructionOpcode::SUB,
+ DestinationRegister, *Flipped, *SignMask);
+ if (!Result)
+ return std::unexpected(Result.error());
+ return {};
+ }
+
+ [[nodiscard]] static Expected<std::pair<Register, MachineInstr *>>
+ buildSignedOverflowAfter(MachineIRBuilder &Builder, MachineInstr &Anchor,
+ MachineFunction &Function, const TargetDescription &Target, LLT Type,
+ Register Result, Register Left, Register Right, bool IsAdd) {
+ auto FirstDifference = createTemporary(Function, Type, Target);
+ auto SecondDifference = createTemporary(Function, Type, Target);
+ auto SignBits = createTemporary(Function, Type, Target);
+ auto I1 = LLT::makeInteger(1);
+ if (!I1)
+ return std::unexpected(I1.error());
+ auto Overflow = createTemporary(Function, *I1, Target);
+ if (!FirstDifference || !SecondDifference || !SignBits || !Overflow) {
+ return std::unexpected(!FirstDifference ? FirstDifference.error()
+ : !SecondDifference ? SecondDifference.error()
+ : !SignBits ? SignBits.error()
+ : Overflow.error());
+ }
+ const Register FirstLeft = IsAdd ? Result : Left;
+ const Register FirstRight = IsAdd ? Left : Right;
+ const Register SecondLeft = Result;
+ const Register SecondRight = IsAdd ? Right : Left;
+ auto First = buildBinaryAfter(Builder, Anchor, InstructionOpcode::XOR, *FirstDifference,
+ FirstLeft, FirstRight);
+ if (!First)
+ return std::unexpected(First.error());
+ auto Second = buildBinaryAfter(Builder, **First, InstructionOpcode::XOR, *SecondDifference,
+ SecondLeft, SecondRight);
+ if (!Second)
+ return std::unexpected(Second.error());
+ auto Both = buildBinaryAfter(Builder, **Second, InstructionOpcode::AND, *SignBits,
+ *FirstDifference, *SecondDifference);
+ if (!Both)
+ return std::unexpected(Both.error());
+ const InstructionOpcode Shift =
+ Type.scalarBitWidth() == 32 ? InstructionOpcode::SRLIW : InstructionOpcode::SRLI;
+ auto Extract = buildImmediateAfter(Builder, **Both, Shift, *Overflow, *SignBits,
+ Type.scalarBitWidth() - 1U);
+ if (!Extract)
+ return std::unexpected(Extract.error());
+ return std::pair{*Overflow, *Extract};
+ }
+
+ [[nodiscard]] static Expected<void>
+ selectOverflow(MachineInstr &Instruction, MachineFunction &Function,
+ const TargetDescription &Target, std::optional<LLT> Type, GenericOpcode Generic) {
+ const MachineOperand *ResultOperand = Instruction.operandAt(0);
+ const MachineOperand *OverflowOperand = Instruction.operandAt(1);
+ const MachineOperand *LeftOperand = Instruction.operandAt(2);
+ const MachineOperand *RightOperand = Instruction.operandAt(3);
+ const Register *Result = ResultOperand == nullptr ? nullptr : ResultOperand->getIf<Register>();
+ const Register *Overflow =
+ OverflowOperand == nullptr ? nullptr : OverflowOperand->getIf<Register>();
+ const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf<Register>();
+ const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf<Register>();
+ if (!Type || Result == nullptr || Overflow == nullptr || Left == nullptr || Right == nullptr) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V overflow operation has invalid operands");
+ }
+ const Register ResultRegister = *Result;
+ const Register OverflowRegister = *Overflow;
+ const Register LeftRegister = *Left;
+ const Register RightRegister = *Right;
+ auto RemoveOverflow = Instruction.eraseOperand(1);
+ if (!RemoveOverflow)
+ return std::unexpected(RemoveOverflow.error());
+ const bool IsWord = Type->scalarBitWidth() <= 32;
+ const bool IsAdd = Generic == GenericOpcode::G_UADDO || Generic == GenericOpcode::G_SADDO;
+ const bool IsSub = Generic == GenericOpcode::G_USUBO || Generic == GenericOpcode::G_SSUBO;
+ const bool IsUnsigned = Generic == GenericOpcode::G_UADDO ||
+ Generic == GenericOpcode::G_USUBO || Generic == GenericOpcode::G_UMULO;
+ const InstructionOpcode Arithmetic =
+ IsAdd ? (IsWord ? InstructionOpcode::ADDW : InstructionOpcode::ADD)
+ : IsSub ? (IsWord ? InstructionOpcode::SUBW : InstructionOpcode::SUB)
+ : (IsWord ? InstructionOpcode::MULW : InstructionOpcode::MUL);
+ auto SetArithmetic = Instruction.setOpcode(targetOpcode(Arithmetic));
+ if (!SetArithmetic)
+ return std::unexpected(SetArithmetic.error());
+ MachineIRBuilder Builder(Function, Target);
+ if (IsUnsigned && IsAdd) {
+ auto Carry = buildBinaryAfter(Builder, Instruction, InstructionOpcode::SLTU, OverflowRegister,
+ ResultRegister, LeftRegister);
+ if (!Carry)
+ return std::unexpected(Carry.error());
+ return {};
+ }
+ if (IsUnsigned && IsSub) {
+ auto Borrow = buildBinaryAfter(Builder, Instruction, InstructionOpcode::SLTU,
+ OverflowRegister, LeftRegister, RightRegister);
+ if (!Borrow)
+ return std::unexpected(Borrow.error());
+ return {};
+ }
+ if (IsAdd || IsSub) {
+ auto FirstDifference = createTemporary(Function, *Type, Target);
+ auto SecondDifference = createTemporary(Function, *Type, Target);
+ auto SignBits = createTemporary(Function, *Type, Target);
+ if (!FirstDifference || !SecondDifference || !SignBits) {
+ return std::unexpected(
+ !FirstDifference ? FirstDifference.error()
+ : (!SecondDifference ? SecondDifference.error() : SignBits.error()));
+ }
+ const Register FirstLeft = IsAdd ? ResultRegister : LeftRegister;
+ const Register FirstRight = IsAdd ? LeftRegister : RightRegister;
+ const Register SecondLeft = ResultRegister;
+ const Register SecondRight = IsAdd ? RightRegister : LeftRegister;
+ auto First = buildBinaryAfter(Builder, Instruction, InstructionOpcode::XOR, *FirstDifference,
+ FirstLeft, FirstRight);
+ if (!First)
+ return std::unexpected(First.error());
+ auto Second = buildBinaryAfter(Builder, **First, InstructionOpcode::XOR, *SecondDifference,
+ SecondLeft, SecondRight);
+ if (!Second)
+ return std::unexpected(Second.error());
+ auto Both = buildBinaryAfter(Builder, **Second, InstructionOpcode::AND, *SignBits,
+ *FirstDifference, *SecondDifference);
+ if (!Both)
+ return std::unexpected(Both.error());
+ auto OverflowResult = buildImmediateAfter(
+ Builder, **Both, IsWord ? InstructionOpcode::SRLIW : InstructionOpcode::SRLI,
+ OverflowRegister, *SignBits, Type->scalarBitWidth() - 1U);
+ if (!OverflowResult)
+ return std::unexpected(OverflowResult.error());
+ return {};
+ }
+
+ auto I64 = LLT::makeInteger(64);
+ if (!I64)
+ return std::unexpected(I64.error());
+ auto Zero = physicalRegister(PhysicalRegisterID::X0);
+ if (!Zero)
+ return std::unexpected(Zero.error());
+ if (!IsWord) {
+ auto High = createTemporary(Function, *I64, Target);
+ if (!High)
+ return std::unexpected(High.error());
+ auto HighProduct = buildBinaryAfter(
+ Builder, Instruction, IsUnsigned ? InstructionOpcode::MULHU : InstructionOpcode::MULH,
+ *High, LeftRegister, RightRegister);
+ if (!HighProduct)
+ return std::unexpected(HighProduct.error());
+ if (IsUnsigned) {
+ auto OverflowResult =
+ buildBinaryAfter(Builder, **HighProduct, InstructionOpcode::SLTU, OverflowRegister,
+ Register::fromPhysical(*Zero), *High);
+ if (!OverflowResult)
+ return std::unexpected(OverflowResult.error());
+ return {};
+ }
+ auto ExpectedHigh = createTemporary(Function, *I64, Target);
+ auto Difference = createTemporary(Function, *I64, Target);
+ if (!ExpectedHigh || !Difference)
+ return std::unexpected(!ExpectedHigh ? ExpectedHigh.error() : Difference.error());
+ auto Sign = buildImmediateAfter(Builder, **HighProduct, InstructionOpcode::SRAI,
+ *ExpectedHigh, ResultRegister, 63);
+ if (!Sign)
+ return std::unexpected(Sign.error());
+ auto Compare = buildBinaryAfter(Builder, **Sign, InstructionOpcode::XOR, *Difference, *High,
+ *ExpectedHigh);
+ if (!Compare)
+ return std::unexpected(Compare.error());
+ auto OverflowResult =
+ buildBinaryAfter(Builder, **Compare, InstructionOpcode::SLTU, OverflowRegister,
+ Register::fromPhysical(*Zero), *Difference);
+ if (!OverflowResult)
+ return std::unexpected(OverflowResult.error());
+ return {};
+ }
+
+ auto ExtendedLeft = createTemporary(Function, *I64, Target);
+ auto ExtendedRight = createTemporary(Function, *I64, Target);
+ auto FullProduct = createTemporary(Function, *I64, Target);
+ if (!ExtendedLeft || !ExtendedRight || !FullProduct) {
+ return std::unexpected(!ExtendedLeft
+ ? ExtendedLeft.error()
+ : (!ExtendedRight ? ExtendedRight.error() : FullProduct.error()));
+ }
+ MachineInstr *Anchor = &Instruction;
+ if (IsUnsigned) {
+ auto ShiftedLeft = createTemporary(Function, *I64, Target);
+ auto ShiftedRight = createTemporary(Function, *I64, Target);
+ if (!ShiftedLeft || !ShiftedRight)
+ return std::unexpected(!ShiftedLeft ? ShiftedLeft.error() : ShiftedRight.error());
+ auto LeftShift = buildImmediateAfter(Builder, *Anchor, InstructionOpcode::SLLI, *ShiftedLeft,
+ LeftRegister, 32);
+ if (!LeftShift)
+ return std::unexpected(LeftShift.error());
+ auto LeftExtend = buildImmediateAfter(Builder, **LeftShift, InstructionOpcode::SRLI,
+ *ExtendedLeft, *ShiftedLeft, 32);
+ if (!LeftExtend)
+ return std::unexpected(LeftExtend.error());
+ auto RightShift = buildImmediateAfter(Builder, **LeftExtend, InstructionOpcode::SLLI,
+ *ShiftedRight, RightRegister, 32);
+ if (!RightShift)
+ return std::unexpected(RightShift.error());
+ auto RightExtend = buildImmediateAfter(Builder, **RightShift, InstructionOpcode::SRLI,
+ *ExtendedRight, *ShiftedRight, 32);
+ if (!RightExtend)
+ return std::unexpected(RightExtend.error());
+ Anchor = *RightExtend;
+ } else {
+ auto LeftExtend = buildImmediateAfter(Builder, *Anchor, InstructionOpcode::ADDIW,
+ *ExtendedLeft, LeftRegister, 0);
+ if (!LeftExtend)
+ return std::unexpected(LeftExtend.error());
+ auto RightExtend = buildImmediateAfter(Builder, **LeftExtend, InstructionOpcode::ADDIW,
+ *ExtendedRight, RightRegister, 0);
+ if (!RightExtend)
+ return std::unexpected(RightExtend.error());
+ Anchor = *RightExtend;
+ }
+ auto Product = buildBinaryAfter(Builder, *Anchor, InstructionOpcode::MUL, *FullProduct,
+ *ExtendedLeft, *ExtendedRight);
+ if (!Product)
+ return std::unexpected(Product.error());
+ if (IsUnsigned) {
+ auto High = createTemporary(Function, *I64, Target);
+ if (!High)
+ return std::unexpected(High.error());
+ auto ExtractHigh =
+ buildImmediateAfter(Builder, **Product, InstructionOpcode::SRLI, *High, *FullProduct, 32);
+ if (!ExtractHigh)
+ return std::unexpected(ExtractHigh.error());
+ auto OverflowResult =
+ buildBinaryAfter(Builder, **ExtractHigh, InstructionOpcode::SLTU, OverflowRegister,
+ Register::fromPhysical(*Zero), *High);
+ if (!OverflowResult)
+ return std::unexpected(OverflowResult.error());
+ return {};
+ }
+ auto Difference = createTemporary(Function, *I64, Target);
+ if (!Difference)
+ return std::unexpected(Difference.error());
+ auto Compare = buildBinaryAfter(Builder, **Product, InstructionOpcode::XOR, *Difference,
+ *FullProduct, ResultRegister);
+ if (!Compare)
+ return std::unexpected(Compare.error());
+ auto OverflowResult =
+ buildBinaryAfter(Builder, **Compare, InstructionOpcode::SLTU, OverflowRegister,
+ Register::fromPhysical(*Zero), *Difference);
+ if (!OverflowResult)
+ return std::unexpected(OverflowResult.error());
+ return {};
+ }
+
+ [[nodiscard]] static Expected<void> selectExtendedOverflow(MachineInstr &Instruction,
+ MachineFunction &Function,
+ const TargetDescription &Target,
+ std::optional<LLT> Type,
+ GenericOpcode Generic) {
+ const MachineOperand *ResultOperand = Instruction.operandAt(0);
+ const MachineOperand *CarryOutOperand = Instruction.operandAt(1);
+ const MachineOperand *LeftOperand = Instruction.operandAt(2);
+ const MachineOperand *RightOperand = Instruction.operandAt(3);
+ const MachineOperand *CarryInOperand = Instruction.operandAt(4);
+ const Register *Result = ResultOperand == nullptr ? nullptr : ResultOperand->getIf<Register>();
+ const Register *CarryOut =
+ CarryOutOperand == nullptr ? nullptr : CarryOutOperand->getIf<Register>();
+ const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf<Register>();
+ const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf<Register>();
+ const Register *CarryIn =
+ CarryInOperand == nullptr ? nullptr : CarryInOperand->getIf<Register>();
+ if (!Type || Result == nullptr || CarryOut == nullptr || Left == nullptr || Right == nullptr ||
+ CarryIn == nullptr) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V extended overflow operation has invalid operands");
+ }
+ const Register ResultRegister = *Result;
+ const Register CarryOutRegister = *CarryOut;
+ const Register LeftRegister = *Left;
+ const Register RightRegister = *Right;
+ const Register CarryInRegister = *CarryIn;
+ const bool IsAdd = Generic == GenericOpcode::G_UADDE || Generic == GenericOpcode::G_SADDE;
+ const bool IsUnsigned = Generic == GenericOpcode::G_UADDE || Generic == GenericOpcode::G_USUBE;
+ const bool IsWord = Type->scalarBitWidth() <= 32;
+ const InstructionOpcode Arithmetic =
+ IsAdd ? (IsWord ? InstructionOpcode::ADDW : InstructionOpcode::ADD)
+ : (IsWord ? InstructionOpcode::SUBW : InstructionOpcode::SUB);
+ auto Partial = createTemporary(Function, *Type, Target);
+ auto I1 = LLT::makeInteger(1);
+ if (!Partial || !I1)
+ return std::unexpected(!Partial ? Partial.error() : I1.error());
+ auto NormalizedCarry = createTemporary(Function, *I1, Target);
+ if (!NormalizedCarry)
+ return std::unexpected(NormalizedCarry.error());
+ auto SetPartial = Instruction.operandAt(0)->setRegister(*Partial);
+ if (!SetPartial)
+ return std::unexpected(SetPartial.error());
+ auto RemoveCarryOut = Instruction.eraseOperand(1);
+ if (!RemoveCarryOut)
+ return std::unexpected(RemoveCarryOut.error());
+ auto RemoveCarryIn = Instruction.eraseOperand(3);
+ if (!RemoveCarryIn)
+ return std::unexpected(RemoveCarryIn.error());
+ auto SetArithmetic = Instruction.setOpcode(targetOpcode(Arithmetic));
+ if (!SetArithmetic)
+ return std::unexpected(SetArithmetic.error());
+ MachineIRBuilder Builder(Function, Target);
+ auto Normalize = buildImmediateAfter(Builder, Instruction, InstructionOpcode::ANDI,
+ *NormalizedCarry, CarryInRegister, 1);
+ if (!Normalize)
+ return std::unexpected(Normalize.error());
+ auto Final = buildBinaryAfter(Builder, **Normalize, Arithmetic, ResultRegister, *Partial,
+ *NormalizedCarry);
+ if (!Final)
+ return std::unexpected(Final.error());
+
+ if (IsUnsigned) {
+ auto FirstCarry = createTemporary(Function, *I1, Target);
+ auto SecondCarry = createTemporary(Function, *I1, Target);
+ if (!FirstCarry || !SecondCarry)
+ return std::unexpected(!FirstCarry ? FirstCarry.error() : SecondCarry.error());
+ auto First =
+ buildBinaryAfter(Builder, **Final, InstructionOpcode::SLTU, *FirstCarry,
+ IsAdd ? *Partial : LeftRegister, IsAdd ? LeftRegister : RightRegister);
+ if (!First)
+ return std::unexpected(First.error());
+ auto Second =
+ buildBinaryAfter(Builder, **First, InstructionOpcode::SLTU, *SecondCarry,
+ IsAdd ? ResultRegister : *Partial, IsAdd ? *Partial : *NormalizedCarry);
+ if (!Second)
+ return std::unexpected(Second.error());
+ auto Combined = buildBinaryAfter(Builder, **Second, InstructionOpcode::OR, CarryOutRegister,
+ *FirstCarry, *SecondCarry);
+ if (!Combined)
+ return std::unexpected(Combined.error());
+ return {};
+ }
+
+ auto FirstOverflow = buildSignedOverflowAfter(Builder, **Final, Function, Target, *Type,
+ *Partial, LeftRegister, RightRegister, IsAdd);
+ if (!FirstOverflow)
+ return std::unexpected(FirstOverflow.error());
+ auto SecondOverflow =
+ buildSignedOverflowAfter(Builder, *FirstOverflow->second, Function, Target, *Type,
+ ResultRegister, *Partial, *NormalizedCarry, IsAdd);
+ if (!SecondOverflow)
+ return std::unexpected(SecondOverflow.error());
+ auto Combined = buildBinaryAfter(Builder, *SecondOverflow->second, InstructionOpcode::OR,
+ CarryOutRegister, FirstOverflow->first, SecondOverflow->first);
+ if (!Combined)
+ return std::unexpected(Combined.error());
+ return {};
+ }
+
+ [[nodiscard]] static Expected<void>
+ selectAtomicRMW(MachineInstr &Instruction, MachineFunction &Function,
+ const TargetDescription &Target, std::optional<LLT> Type, GenericOpcode Generic) {
+ const MachineOperand *AddressOperand = Instruction.operandAt(1);
+ const MachineOperand *ValueOperand = Instruction.operandAt(2);
+ const Register *Address =
+ AddressOperand == nullptr ? nullptr : AddressOperand->getIf<Register>();
+ const Register *Value = ValueOperand == nullptr ? nullptr : ValueOperand->getIf<Register>();
+ if (!Type || Address == nullptr || Value == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError, "RISC-V atomic RMW has invalid operands");
+ Register SelectedValue = *Value;
+ const bool IsWord = Type->scalarBitWidth() == 32;
+ if (Generic == GenericOpcode::G_ATOMICRMW_SUB) {
+ auto Negated = createTemporary(Function, *Type, Target);
+ auto Zero = physicalRegister(PhysicalRegisterID::X0);
+ if (!Negated || !Zero)
+ return std::unexpected(!Negated ? Negated.error() : Zero.error());
+ MachineIRBuilder Builder(Function, Target);
+ auto Point = Builder.setInsertPointBefore(Instruction);
+ if (!Point)
+ return std::unexpected(Point.error());
+ auto Pending = Builder.buildInstr(
+ targetOpcode(IsWord ? InstructionOpcode::SUBW : InstructionOpcode::SUB));
+ if (!Pending)
+ return std::unexpected(Pending.error());
+ auto Def = Pending->addDef(*Negated);
+ if (!Def)
+ return std::unexpected(Def.error());
+ auto Left = Pending->addUse(Register::fromPhysical(*Zero));
+ if (!Left)
+ return std::unexpected(Left.error());
+ auto Right = Pending->addUse(*Value);
+ if (!Right)
+ return std::unexpected(Right.error());
+ auto Built = Pending->build();
+ if (!Built)
+ return std::unexpected(Built.error());
+ SelectedValue = *Negated;
+ }
+ const auto SelectOpcode = [Generic, IsWord]() -> InstructionOpcode {
+ switch (Generic) {
+ case GenericOpcode::G_ATOMICRMW_XCHG:
+ return IsWord ? InstructionOpcode::AMOSWAP_W : InstructionOpcode::AMOSWAP_D;
+ case GenericOpcode::G_ATOMICRMW_ADD:
+ case GenericOpcode::G_ATOMICRMW_SUB:
+ return IsWord ? InstructionOpcode::AMOADD_W : InstructionOpcode::AMOADD_D;
+ case GenericOpcode::G_ATOMICRMW_AND:
+ return IsWord ? InstructionOpcode::AMOAND_W : InstructionOpcode::AMOAND_D;
+ case GenericOpcode::G_ATOMICRMW_OR:
+ return IsWord ? InstructionOpcode::AMOOR_W : InstructionOpcode::AMOOR_D;
+ case GenericOpcode::G_ATOMICRMW_XOR:
+ return IsWord ? InstructionOpcode::AMOXOR_W : InstructionOpcode::AMOXOR_D;
+ case GenericOpcode::G_ATOMICRMW_MAX:
+ return IsWord ? InstructionOpcode::AMOMAX_W : InstructionOpcode::AMOMAX_D;
+ case GenericOpcode::G_ATOMICRMW_MIN:
+ return IsWord ? InstructionOpcode::AMOMIN_W : InstructionOpcode::AMOMIN_D;
+ case GenericOpcode::G_ATOMICRMW_UMAX:
+ return IsWord ? InstructionOpcode::AMOMAXU_W : InstructionOpcode::AMOMAXU_D;
+ case GenericOpcode::G_ATOMICRMW_UMIN:
+ return IsWord ? InstructionOpcode::AMOMINU_W : InstructionOpcode::AMOMINU_D;
+ default:
+ return InstructionOpcode::Invalid;
+ }
+ };
+ const InstructionOpcode Selected = SelectOpcode();
+ if (Selected == InstructionOpcode::Invalid)
+ return makeUnexpected(ErrorCode::InternalError,
+ "RISC-V atomic RMW selection reached an unhandled opcode");
+ auto SetValue = Instruction.operandAt(1)->setRegister(SelectedValue);
+ if (!SetValue)
+ return std::unexpected(SetValue.error());
+ auto SetAddress = Instruction.operandAt(2)->setRegister(*Address);
+ if (!SetAddress)
+ return std::unexpected(SetAddress.error());
+ return Instruction.setOpcode(targetOpcode(Selected));
+ }
+
+ [[nodiscard]] static Expected<void> selectIntegerExtension(MachineInstr &Instruction,
+ MachineFunction &Function,
+ const TargetDescription &Target,
+ GenericOpcode Generic,
+ std::optional<LLT> Type) {
+ const std::optional<LLT> SourceType = operandType(Function, Instruction, 1);
+ const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+ const MachineOperand *SourceOperand = Instruction.operandAt(1);
+ const Register *Destination =
+ DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+ const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>();
+ if (!Type || !SourceType || Destination == nullptr || Source == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V integer extension has invalid register operands");
+ const Register DestinationRegister = *Destination;
+ if (Generic == GenericOpcode::G_ANYEXT)
+ return Instruction.setOpcode(opcode(GenericOpcode::COPY));
+
+ const std::uint32_t DestinationBits = Type->scalarBitWidth();
+ const std::uint32_t SourceBits = SourceType->scalarBitWidth();
+ if (Generic == GenericOpcode::G_SEXT && DestinationBits == 64 && SourceBits == 32) {
+ auto ImmediateResult = appendImmediate(Instruction, 0);
+ if (!ImmediateResult)
+ return std::unexpected(ImmediateResult.error());
+ return Instruction.setOpcode(targetOpcode(InstructionOpcode::ADDIW));
+ }
+
+ const std::uint32_t ContainerBits = DestinationBits <= 32 ? 32 : 64;
+ const std::int64_t Shift = static_cast<std::int64_t>(ContainerBits - SourceBits);
+ const InstructionOpcode ShiftLeft =
+ ContainerBits == 32 ? InstructionOpcode::SLLIW : InstructionOpcode::SLLI;
+ const InstructionOpcode ShiftRight =
+ Generic == GenericOpcode::G_SEXT
+ ? (ContainerBits == 32 ? InstructionOpcode::SRAIW : InstructionOpcode::SRAI)
+ : (ContainerBits == 32 ? InstructionOpcode::SRLIW : InstructionOpcode::SRLI);
+ auto Temporary = createTemporary(Function, *Type, Target);
+ if (!Temporary)
+ return std::unexpected(Temporary.error());
+ auto SetDestination = Instruction.operandAt(0)->setRegister(*Temporary);
+ if (!SetDestination)
+ return std::unexpected(SetDestination.error());
+ auto SetOpcode = Instruction.setOpcode(targetOpcode(ShiftLeft));
+ if (!SetOpcode)
+ return std::unexpected(SetOpcode.error());
+ auto ShiftImmediate = appendImmediate(Instruction, Shift);
+ if (!ShiftImmediate)
+ return std::unexpected(ShiftImmediate.error());
+
+ MachineIRBuilder Builder(Function, Target);
+ auto Shifted = buildImmediateAfter(Builder, Instruction, ShiftRight, DestinationRegister,
+ *Temporary, Shift);
+ if (!Shifted)
+ return std::unexpected(Shifted.error());
+ return {};
+ }
+
+ [[nodiscard]] static Expected<void> selectSignExtendInRegister(MachineInstr &Instruction,
+ MachineFunction &Function,
+ const TargetDescription &Target,
+ std::optional<LLT> Type) {
+ const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+ const MachineOperand *SourceOperand = Instruction.operandAt(1);
+ const Register *Destination =
+ DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+ const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>();
+ std::optional<std::int64_t> Width;
+ if (const MachineOperand *WidthOperand = Instruction.operandAt(2); WidthOperand != nullptr) {
+ auto ParsedWidth = signedImmediate(*WidthOperand);
+ if (ParsedWidth)
+ Width = *ParsedWidth;
+ }
+ if (!Type || Destination == nullptr || Source == nullptr || !Width || *Width <= 0 ||
+ static_cast<std::uint64_t>(*Width) > Type->scalarBitWidth())
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V sign-extension-in-register has invalid operands");
+ const Register DestinationRegister = *Destination;
+ if (static_cast<std::uint64_t>(*Width) == Type->scalarBitWidth()) {
+ auto EraseWidth = Instruction.eraseOperand(2);
+ if (!EraseWidth)
+ return std::unexpected(EraseWidth.error());
+ return Instruction.setOpcode(opcode(GenericOpcode::COPY));
+ }
+
+ const std::uint32_t TypeBits = Type->scalarBitWidth();
+ if (TypeBits == 64 && *Width == 32) {
+ auto ImmediateResult = replaceImmediate(Instruction, 2, 0);
+ if (!ImmediateResult)
+ return std::unexpected(ImmediateResult.error());
+ return Instruction.setOpcode(targetOpcode(InstructionOpcode::ADDIW));
+ }
+ const std::uint32_t ContainerBits = TypeBits <= 32 ? 32 : 64;
+ const std::int64_t Shift = static_cast<std::int64_t>(ContainerBits) - *Width;
+ const InstructionOpcode ShiftLeft =
+ ContainerBits == 32 ? InstructionOpcode::SLLIW : InstructionOpcode::SLLI;
+ const InstructionOpcode ShiftRight =
+ ContainerBits == 32 ? InstructionOpcode::SRAIW : InstructionOpcode::SRAI;
+ auto Temporary = createTemporary(Function, *Type, Target);
+ if (!Temporary)
+ return std::unexpected(Temporary.error());
+ auto SetDestination = Instruction.operandAt(0)->setRegister(*Temporary);
+ if (!SetDestination)
+ return std::unexpected(SetDestination.error());
+ auto SetOpcode = Instruction.setOpcode(targetOpcode(ShiftLeft));
+ if (!SetOpcode)
+ return std::unexpected(SetOpcode.error());
+ auto ShiftImmediate = replaceImmediate(Instruction, 2, Shift);
+ if (!ShiftImmediate)
+ return std::unexpected(ShiftImmediate.error());
+
+ MachineIRBuilder Builder(Function, Target);
+ auto Shifted = buildImmediateAfter(Builder, Instruction, ShiftRight, DestinationRegister,
+ *Temporary, Shift);
+ if (!Shifted)
+ return std::unexpected(Shifted.error());
+ return {};
+ }
+
+ [[nodiscard]] static Expected<void> selectIntegerCompare(MachineInstr &Instruction,
+ MachineFunction &Function,
+ const TargetDescription &Target) {
+ const MachineOperand *PredicateMO = Instruction.operandAt(1);
+ const auto *Predicate =
+ PredicateMO == nullptr ? nullptr : PredicateMO->getIf<PredicateOperand>();
+ const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+ const MachineOperand *LeftOperand = Instruction.operandAt(2);
+ const MachineOperand *RightOperand = Instruction.operandAt(3);
+ const Register *Destination =
+ DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+ const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf<Register>();
+ const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf<Register>();
+ const std::optional<LLT> LeftType = operandType(Function, Instruction, 2);
+ if (Predicate == nullptr || Predicate->integerPredicate() == nullptr ||
+ Destination == nullptr || Left == nullptr || Right == nullptr || !LeftType)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V integer comparison has invalid operands");
+ const IntegerPredicate Condition = *Predicate->integerPredicate();
+ const Register DestinationRegister = *Destination;
+ const bool SignedOperands = Condition == IntegerPredicate::SignedGreaterThan ||
+ Condition == IntegerPredicate::SignedGreaterEqual ||
+ Condition == IntegerPredicate::SignedLessThan ||
+ Condition == IntegerPredicate::SignedLessEqual;
+ auto LeftRegister =
+ normalizeIntegerBefore(Instruction, Function, Target, *Left, *LeftType, SignedOperands);
+ auto RightRegister =
+ normalizeIntegerBefore(Instruction, Function, Target, *Right, *LeftType, SignedOperands);
+ if (!LeftRegister || !RightRegister)
+ return std::unexpected(!LeftRegister ? LeftRegister.error() : RightRegister.error());
+ auto ErasePredicate = Instruction.eraseOperand(1);
+ if (!ErasePredicate)
+ return std::unexpected(ErasePredicate.error());
+
+ const auto SetSources = [&Instruction](Register NewLeft, Register NewRight) -> Expected<void> {
+ auto SetLeft = Instruction.operandAt(1)->setRegister(NewLeft);
+ if (!SetLeft)
+ return std::unexpected(SetLeft.error());
+ return Instruction.operandAt(2)->setRegister(NewRight);
+ };
+ const auto StrictOpcode = [](IntegerPredicate Value) -> std::optional<InstructionOpcode> {
+ switch (Value) {
+ case IntegerPredicate::SignedLessThan:
+ return InstructionOpcode::SLT;
+ case IntegerPredicate::UnsignedLessThan:
+ return InstructionOpcode::SLTU;
+ default:
+ return std::nullopt;
+ }
+ };
+
+ IntegerPredicate StrictCondition = Condition;
+ bool Swap = false;
+ bool Invert = false;
+ switch (Condition) {
+ case IntegerPredicate::Equal:
+ case IntegerPredicate::NotEqual:
+ break;
+ case IntegerPredicate::SignedGreaterThan:
+ StrictCondition = IntegerPredicate::SignedLessThan;
+ Swap = true;
+ break;
+ case IntegerPredicate::SignedGreaterEqual:
+ StrictCondition = IntegerPredicate::SignedLessThan;
+ Invert = true;
+ break;
+ case IntegerPredicate::SignedLessThan:
+ StrictCondition = IntegerPredicate::SignedLessThan;
+ break;
+ case IntegerPredicate::SignedLessEqual:
+ StrictCondition = IntegerPredicate::SignedLessThan;
+ Swap = true;
+ Invert = true;
+ break;
+ case IntegerPredicate::UnsignedGreaterThan:
+ StrictCondition = IntegerPredicate::UnsignedLessThan;
+ Swap = true;
+ break;
+ case IntegerPredicate::UnsignedGreaterEqual:
+ StrictCondition = IntegerPredicate::UnsignedLessThan;
+ Invert = true;
+ break;
+ case IntegerPredicate::UnsignedLessThan:
+ StrictCondition = IntegerPredicate::UnsignedLessThan;
+ break;
+ case IntegerPredicate::UnsignedLessEqual:
+ StrictCondition = IntegerPredicate::UnsignedLessThan;
+ Swap = true;
+ Invert = true;
+ break;
+ case IntegerPredicate::Count:
+ return makeUnexpected(ErrorCode::Unsupported,
+ "RISC-V integer comparison predicate is invalid");
+ }
+
+ MachineIRBuilder Builder(Function, Target);
+ if (Condition == IntegerPredicate::Equal || Condition == IntegerPredicate::NotEqual) {
+ auto Temporary = createTemporary(Function, *LeftType, Target);
+ if (!Temporary)
+ return std::unexpected(Temporary.error());
+ auto SetDestination = Instruction.operandAt(0)->setRegister(*Temporary);
+ if (!SetDestination)
+ return std::unexpected(SetDestination.error());
+ auto Set = SetSources(*LeftRegister, *RightRegister);
+ if (!Set)
+ return std::unexpected(Set.error());
+ auto SetOpcode = Instruction.setOpcode(targetOpcode(InstructionOpcode::XOR));
+ if (!SetOpcode)
+ return std::unexpected(SetOpcode.error());
+ if (Condition == IntegerPredicate::Equal) {
+ auto Result = buildImmediateAfter(Builder, Instruction, InstructionOpcode::SLTIU,
+ DestinationRegister, *Temporary, 1);
+ if (!Result)
+ return std::unexpected(Result.error());
+ } else {
+ auto Zero = physicalRegister(PhysicalRegisterID::X0);
+ if (!Zero)
+ return std::unexpected(Zero.error());
+ auto Result =
+ buildBinaryAfter(Builder, Instruction, InstructionOpcode::SLTU, DestinationRegister,
+ Register::fromPhysical(*Zero), *Temporary);
+ if (!Result)
+ return std::unexpected(Result.error());
+ }
+ return {};
+ }
+
+ const std::optional<InstructionOpcode> OpcodeValue = StrictOpcode(StrictCondition);
+ if (!OpcodeValue)
+ return makeUnexpected(ErrorCode::Unsupported,
+ "RISC-V integer comparison predicate is invalid");
+ if (Invert) {
+ auto Temporary = createTemporary(Function, *LeftType, Target);
+ if (!Temporary)
+ return std::unexpected(Temporary.error());
+ auto SetDestination = Instruction.operandAt(0)->setRegister(*Temporary);
+ if (!SetDestination)
+ return std::unexpected(SetDestination.error());
+ const Register Base = *Temporary;
+ if (Swap) {
+ auto Set = SetSources(*RightRegister, *LeftRegister);
+ if (!Set)
+ return std::unexpected(Set.error());
+ } else {
+ auto Set = SetSources(*LeftRegister, *RightRegister);
+ if (!Set)
+ return std::unexpected(Set.error());
+ }
+ auto SetOpcode = Instruction.setOpcode(targetOpcode(*OpcodeValue));
+ if (!SetOpcode)
+ return std::unexpected(SetOpcode.error());
+ auto Result = buildImmediateAfter(Builder, Instruction, InstructionOpcode::XORI,
+ DestinationRegister, Base, 1);
+ if (!Result)
+ return std::unexpected(Result.error());
+ return {};
+ }
+ if (Swap) {
+ auto Set = SetSources(*RightRegister, *LeftRegister);
+ if (!Set)
+ return std::unexpected(Set.error());
+ } else {
+ auto Set = SetSources(*LeftRegister, *RightRegister);
+ if (!Set)
+ return std::unexpected(Set.error());
+ }
+ return Instruction.setOpcode(targetOpcode(*OpcodeValue));
+ }
+
+ [[nodiscard]] static Expected<void> selectFloatingCompare(MachineInstr &Instruction,
+ MachineFunction &Function,
+ const TargetDescription &Target) {
+ const MachineOperand *PredicateMO = Instruction.operandAt(1);
+ const auto *Predicate =
+ PredicateMO == nullptr ? nullptr : PredicateMO->getIf<PredicateOperand>();
+ const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+ const MachineOperand *LeftOperand = Instruction.operandAt(2);
+ const MachineOperand *RightOperand = Instruction.operandAt(3);
+ const Register *Destination =
+ DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+ const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf<Register>();
+ const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf<Register>();
+ const std::optional<LLT> FloatType = operandType(Function, Instruction, 2);
+ const std::optional<LLT> ResultType = resultType(Function, Instruction);
+ if (Predicate == nullptr || Predicate->floatingPredicate() == nullptr ||
+ Destination == nullptr || Left == nullptr || Right == nullptr || !FloatType || !ResultType)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V floating comparison has invalid operands");
+ const FloatingPredicate Condition = *Predicate->floatingPredicate();
+ const Register DestinationRegister = *Destination;
+ const Register LeftRegister = *Left;
+ const Register RightRegister = *Right;
+ const InstructionOpcode EqualOpcode =
+ FloatType->scalarBitWidth() == 32 ? InstructionOpcode::FEQ_S : InstructionOpcode::FEQ_D;
+ const InstructionOpcode LessOpcode =
+ FloatType->scalarBitWidth() == 32 ? InstructionOpcode::FLT_S : InstructionOpcode::FLT_D;
+ const InstructionOpcode LessEqualOpcode =
+ FloatType->scalarBitWidth() == 32 ? InstructionOpcode::FLE_S : InstructionOpcode::FLE_D;
+
+ auto ErasePredicate = Instruction.eraseOperand(1);
+ if (!ErasePredicate)
+ return std::unexpected(ErasePredicate.error());
+ const auto SetBase = [&Instruction](InstructionOpcode OpcodeValue, Register Def,
+ Register NewLeft, Register NewRight) -> Expected<void> {
+ auto SetDestination = Instruction.operandAt(0)->setRegister(Def);
+ if (!SetDestination)
+ return std::unexpected(SetDestination.error());
+ auto SetLeft = Instruction.operandAt(1)->setRegister(NewLeft);
+ if (!SetLeft)
+ return std::unexpected(SetLeft.error());
+ auto SetRight = Instruction.operandAt(2)->setRegister(NewRight);
+ if (!SetRight)
+ return std::unexpected(SetRight.error());
+ return Instruction.setOpcode(targetOpcode(OpcodeValue));
+ };
+ const auto BuildInverted = [&](InstructionOpcode BaseOpcode, Register BaseLeft,
+ Register BaseRight) -> Expected<void> {
+ auto Temporary = createTemporary(Function, *ResultType, Target);
+ if (!Temporary)
+ return std::unexpected(Temporary.error());
+ auto Base = SetBase(BaseOpcode, *Temporary, BaseLeft, BaseRight);
+ if (!Base)
+ return std::unexpected(Base.error());
+ MachineIRBuilder Builder(Function, Target);
+ auto Invert = buildImmediateAfter(Builder, Instruction, InstructionOpcode::XORI,
+ DestinationRegister, *Temporary, 1);
+ if (!Invert)
+ return std::unexpected(Invert.error());
+ return {};
+ };
+ const auto BuildSelfAnd = [&](bool Invert) -> Expected<void> {
+ auto First = createTemporary(Function, *ResultType, Target);
+ auto Second = createTemporary(Function, *ResultType, Target);
+ if (!First || !Second)
+ return std::unexpected((!First ? First.error() : Second.error()));
+ auto Base = SetBase(EqualOpcode, *First, LeftRegister, LeftRegister);
+ if (!Base)
+ return std::unexpected(Base.error());
+ MachineIRBuilder Builder(Function, Target);
+ auto SecondInstruction = buildBinaryAfter(Builder, Instruction, EqualOpcode, *Second,
+ RightRegister, RightRegister);
+ if (!SecondInstruction)
+ return std::unexpected(SecondInstruction.error());
+ Register CombinedDestination = DestinationRegister;
+ if (Invert) {
+ auto Temporary = createTemporary(Function, *ResultType, Target);
+ if (!Temporary)
+ return std::unexpected(Temporary.error());
+ CombinedDestination = *Temporary;
+ }
+ auto Combined = buildBinaryAfter(Builder, **SecondInstruction, InstructionOpcode::AND,
+ CombinedDestination, *First, *Second);
+ if (!Combined)
+ return std::unexpected(Combined.error());
+ if (!Invert)
+ return {};
+ auto Inverted = buildImmediateAfter(Builder, **Combined, InstructionOpcode::XORI,
+ DestinationRegister, CombinedDestination, 1);
+ if (!Inverted)
+ return std::unexpected(Inverted.error());
+ return {};
+ };
+ const auto BuildOrderedNotEqual = [&](bool Invert) -> Expected<void> {
+ auto First = createTemporary(Function, *ResultType, Target);
+ auto Second = createTemporary(Function, *ResultType, Target);
+ if (!First || !Second)
+ return std::unexpected((!First ? First.error() : Second.error()));
+ auto Base = SetBase(LessOpcode, *First, LeftRegister, RightRegister);
+ if (!Base)
+ return std::unexpected(Base.error());
+ MachineIRBuilder Builder(Function, Target);
+ auto SecondInstruction =
+ buildBinaryAfter(Builder, Instruction, LessOpcode, *Second, RightRegister, LeftRegister);
+ if (!SecondInstruction)
+ return std::unexpected(SecondInstruction.error());
+ Register CombinedDestination = DestinationRegister;
+ if (Invert) {
+ auto Temporary = createTemporary(Function, *ResultType, Target);
+ if (!Temporary)
+ return std::unexpected(Temporary.error());
+ CombinedDestination = *Temporary;
+ }
+ auto Combined = buildBinaryAfter(Builder, **SecondInstruction, InstructionOpcode::OR,
+ CombinedDestination, *First, *Second);
+ if (!Combined)
+ return std::unexpected(Combined.error());
+ if (!Invert)
+ return {};
+ auto Inverted = buildImmediateAfter(Builder, **Combined, InstructionOpcode::XORI,
+ DestinationRegister, CombinedDestination, 1);
+ if (!Inverted)
+ return std::unexpected(Inverted.error());
+ return {};
+ };
+
+ switch (Condition) {
+ case FloatingPredicate::AlwaysFalse:
+ case FloatingPredicate::AlwaysTrue: {
+ auto EraseRight = Instruction.eraseOperand(2);
+ if (!EraseRight)
+ return std::unexpected(EraseRight.error());
+ auto EraseLeft = Instruction.eraseOperand(1);
+ if (!EraseLeft)
+ return std::unexpected(EraseLeft.error());
+ auto Zero = physicalRegister(PhysicalRegisterID::X0);
+ if (!Zero)
+ return std::unexpected(Zero.error());
+ auto InsertZero = insertPhysicalRegister(Instruction, 1, *Zero, RegisterOperandFlag::Use);
+ if (!InsertZero)
+ return std::unexpected(InsertZero.error());
+ auto Immediate =
+ appendImmediate(Instruction, Condition == FloatingPredicate::AlwaysTrue ? 1 : 0);
+ if (!Immediate)
+ return std::unexpected(Immediate.error());
+ return Instruction.setOpcode(targetOpcode(InstructionOpcode::ADDI));
+ }
+ case FloatingPredicate::OrderedEqual:
+ return SetBase(EqualOpcode, DestinationRegister, LeftRegister, RightRegister);
+ case FloatingPredicate::OrderedGreaterThan:
+ return SetBase(LessOpcode, DestinationRegister, RightRegister, LeftRegister);
+ case FloatingPredicate::OrderedGreaterEqual:
+ return SetBase(LessEqualOpcode, DestinationRegister, RightRegister, LeftRegister);
+ case FloatingPredicate::OrderedLessThan:
+ return SetBase(LessOpcode, DestinationRegister, LeftRegister, RightRegister);
+ case FloatingPredicate::OrderedLessEqual:
+ return SetBase(LessEqualOpcode, DestinationRegister, LeftRegister, RightRegister);
+ case FloatingPredicate::OrderedNotEqual:
+ return BuildOrderedNotEqual(false);
+ case FloatingPredicate::Ordered:
+ return BuildSelfAnd(false);
+ case FloatingPredicate::Unordered:
+ return BuildSelfAnd(true);
+ case FloatingPredicate::UnorderedEqual:
+ return BuildOrderedNotEqual(true);
+ case FloatingPredicate::UnorderedGreaterThan:
+ return BuildInverted(LessEqualOpcode, LeftRegister, RightRegister);
+ case FloatingPredicate::UnorderedGreaterEqual:
+ return BuildInverted(LessOpcode, LeftRegister, RightRegister);
+ case FloatingPredicate::UnorderedLessThan:
+ return BuildInverted(LessEqualOpcode, RightRegister, LeftRegister);
+ case FloatingPredicate::UnorderedLessEqual:
+ return BuildInverted(LessOpcode, RightRegister, LeftRegister);
+ case FloatingPredicate::UnorderedNotEqual:
+ return BuildInverted(EqualOpcode, LeftRegister, RightRegister);
+ case FloatingPredicate::Count:
+ return makeUnexpected(ErrorCode::Unsupported,
+ "RISC-V floating comparison predicate is invalid");
+ }
+ return makeUnexpected(ErrorCode::Unsupported,
+ "RISC-V floating comparison predicate is invalid");
+ }
+
+ [[nodiscard]] static Expected<void> selectFloatToInteger(MachineInstr &Instruction,
+ MachineFunction &Function,
+ std::optional<LLT> Type, bool Unsigned) {
+ const std::optional<LLT> Source = operandType(Function, Instruction, 1);
+ if (!Type || !Source)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V floating-to-integer conversion has invalid types");
+ auto Rounding = appendImmediate(Instruction, 1);
+ if (!Rounding)
+ return std::unexpected(Rounding.error());
+ const bool IsWord = Type->scalarBitWidth() <= 32;
+ if (Source->scalarBitWidth() == 32) {
+ return Instruction.setOpcode(targetOpcode(
+ IsWord ? (Unsigned ? InstructionOpcode::FCVT_WU_S : InstructionOpcode::FCVT_W_S)
+ : (Unsigned ? InstructionOpcode::FCVT_LU_S : InstructionOpcode::FCVT_L_S)));
+ }
+ return Instruction.setOpcode(targetOpcode(
+ IsWord ? (Unsigned ? InstructionOpcode::FCVT_WU_D : InstructionOpcode::FCVT_W_D)
+ : (Unsigned ? InstructionOpcode::FCVT_LU_D : InstructionOpcode::FCVT_L_D)));
+ }
+
+ [[nodiscard]] static Expected<void> selectIntegerToFloat(MachineInstr &Instruction,
+ MachineFunction &Function,
+ const TargetDescription &Target,
+ std::optional<LLT> Type, bool Unsigned) {
+ const std::optional<LLT> Source = operandType(Function, Instruction, 1);
+ const MachineOperand *SourceOperand = Instruction.operandAt(1);
+ const Register *SourceRegister =
+ SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>();
+ if (!Type || !Source || SourceRegister == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V integer-to-floating conversion has invalid types");
+ auto Normalized =
+ normalizeIntegerBefore(Instruction, Function, Target, *SourceRegister, *Source, !Unsigned);
+ if (!Normalized)
+ return std::unexpected(Normalized.error());
+ auto SetSource = Instruction.operandAt(1)->setRegister(*Normalized);
+ if (!SetSource)
+ return std::unexpected(SetSource.error());
+ auto Rounding = appendImmediate(Instruction, 0);
+ if (!Rounding)
+ return std::unexpected(Rounding.error());
+ const bool IsWord = Source->scalarBitWidth() <= 32;
+ if (Type->scalarBitWidth() == 32) {
+ return Instruction.setOpcode(targetOpcode(
+ IsWord ? (Unsigned ? InstructionOpcode::FCVT_S_WU : InstructionOpcode::FCVT_S_W)
+ : (Unsigned ? InstructionOpcode::FCVT_S_LU : InstructionOpcode::FCVT_S_L)));
+ }
+ return Instruction.setOpcode(targetOpcode(
+ IsWord ? (Unsigned ? InstructionOpcode::FCVT_D_WU : InstructionOpcode::FCVT_D_W)
+ : (Unsigned ? InstructionOpcode::FCVT_D_LU : InstructionOpcode::FCVT_D_L)));
+ }
+
+ [[nodiscard]] static Expected<void> selectFloatingMinMax(MachineInstr &Instruction,
+ std::optional<LLT> Type,
+ InstructionOpcode Single,
+ InstructionOpcode Double) {
+ if (Type && Type->isFloat() && Type->scalarBitWidth() == 32)
+ return Instruction.setOpcode(targetOpcode(Single));
+ if (Type && Type->isFloat() && Type->scalarBitWidth() == 64)
+ return Instruction.setOpcode(targetOpcode(Double));
+ return makeUnexpected(ErrorCode::Unsupported,
+ "RISC-V cannot select this floating min/max type");
+ }
+
+ [[nodiscard]] static Expected<void> selectFence(MachineInstr &Instruction) {
+ const auto Ordering = signedImmediate(*Instruction.operandAt(0));
+ if (!Ordering)
+ return std::unexpected(Ordering.error());
+ std::int64_t Pred = 0;
+ std::int64_t Succ = 0;
+ switch (*Ordering) {
+ case 4:
+ Pred = 1;
+ Succ = 3;
+ break;
+ case 5:
+ Pred = 3;
+ Succ = 2;
+ break;
+ case 6:
+ case 7:
+ Pred = 3;
+ Succ = 3;
+ break;
+ default:
+ return makeUnexpected(ErrorCode::Unsupported, "RISC-V fence ordering is unsupported");
+ }
+ auto SetPred = replaceImmediate(Instruction, 0, Pred);
+ if (!SetPred)
+ return std::unexpected(SetPred.error());
+ auto SetSucc = replaceImmediate(Instruction, 1, Succ);
+ if (!SetSucc)
+ return std::unexpected(SetSucc.error());
+ return Instruction.setOpcode(targetOpcode(InstructionOpcode::FENCE));
+ }
+
+ [[nodiscard]] static Expected<void> selectIndirectBranch(MachineInstr &Instruction) {
+ auto Zero = physicalRegister(PhysicalRegisterID::X0);
+ if (!Zero)
+ return std::unexpected(Zero.error());
+ auto Inserted = insertPhysicalRegister(Instruction, 0, *Zero, RegisterOperandFlag::Define);
+ if (!Inserted)
+ return std::unexpected(Inserted.error());
+ auto Immediate = appendImmediate(Instruction, 0);
+ if (!Immediate)
+ return std::unexpected(Immediate.error());
+ return Instruction.setOpcode(targetOpcode(InstructionOpcode::JALR));
+ }
+
+ [[nodiscard]] static Expected<void> selectTrap(MachineInstr &Instruction) {
+ while (Instruction.operandCount() != 0) {
+ auto Erased = Instruction.eraseOperand(Instruction.operandCount() - 1U);
+ if (!Erased)
+ return std::unexpected(Erased.error());
+ }
+ return Instruction.setOpcode(targetOpcode(InstructionOpcode::EBREAK));
+ }
+
+ [[nodiscard]] static Expected<void> selectBitcast(MachineInstr &Instruction,
+ MachineFunction &Function) {
+ const std::optional<LLT> DestinationType = resultType(Function, Instruction);
+ const std::optional<LLT> SourceType = operandType(Function, Instruction, 1);
+ if (!DestinationType || !SourceType)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V bitcast has invalid register types");
+ if (DestinationType->isInteger() && SourceType->isFloat()) {
+ return Instruction.setOpcode(targetOpcode(SourceType->scalarBitWidth() == 32
+ ? InstructionOpcode::FMV_X_W
+ : InstructionOpcode::FMV_X_D));
+ }
+ if (DestinationType->isFloat() && SourceType->isInteger()) {
+ return Instruction.setOpcode(targetOpcode(DestinationType->scalarBitWidth() == 32
+ ? InstructionOpcode::FMV_W_X
+ : InstructionOpcode::FMV_D_X));
+ }
+ return makeUnexpected(ErrorCode::Unsupported, "RISC-V bitcast pair is unsupported");
+ }
+
+ [[nodiscard]] static Expected<void> selectIntegerBinary(MachineInstr &Instruction,
+ std::optional<LLT> Type,
+ InstructionOpcode DoubleWord,
+ InstructionOpcode Word) {
+ if (!Type || !isSupportedIntegerType(*Type))
+ return makeUnexpected(ErrorCode::Unsupported,
+ "RISC-V integer instruction requires a supported scalar integer");
+ const bool IsWord = Type->isInteger() && Type->scalarBitWidth() <= 32;
+ return Instruction.setOpcode(targetOpcode(IsWord ? Word : DoubleWord));
+ }
+
+ [[nodiscard]] static Expected<void>
+ selectNormalizedIntegerBinary(MachineInstr &Instruction, MachineFunction &Function,
+ const TargetDescription &Target, std::optional<LLT> Type,
+ InstructionOpcode DoubleWord, InstructionOpcode Word, bool Signed,
+ bool NormalizeRight) {
+ const MachineOperand *LeftOperand = Instruction.operandAt(1);
+ const MachineOperand *RightOperand = Instruction.operandAt(2);
+ const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf<Register>();
+ const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf<Register>();
+ if (!Type || !isSupportedIntegerScalar(*Type) || Left == nullptr || Right == nullptr) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V normalized integer operation has invalid operands");
+ }
+ auto NormalizedLeft =
+ normalizeIntegerBefore(Instruction, Function, Target, *Left, *Type, Signed);
+ if (!NormalizedLeft)
+ return std::unexpected(NormalizedLeft.error());
+ Register SelectedRight = *Right;
+ if (NormalizeRight) {
+ auto NormalizedRight =
+ normalizeIntegerBefore(Instruction, Function, Target, *Right, *Type, Signed);
+ if (!NormalizedRight)
+ return std::unexpected(NormalizedRight.error());
+ SelectedRight = *NormalizedRight;
+ }
+ auto SetLeft = Instruction.operandAt(1)->setRegister(*NormalizedLeft);
+ if (!SetLeft)
+ return std::unexpected(SetLeft.error());
+ auto SetRight = Instruction.operandAt(2)->setRegister(SelectedRight);
+ if (!SetRight)
+ return std::unexpected(SetRight.error());
+ const bool IsWord = Type->scalarBitWidth() <= 32;
+ return Instruction.setOpcode(targetOpcode(IsWord ? Word : DoubleWord));
+ }
+
+ [[nodiscard]] static Expected<void> selectLoad(MachineInstr &Instruction,
+ std::optional<LLT> Type) {
+ if (!Type)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "load result has no virtual-register type");
+ const auto Bytes = memoryAccessBytes(Instruction);
+ if (!Bytes)
+ return makeUnexpected(ErrorCode::Unsupported,
+ "RISC-V load size does not match its result type");
+ if (Instruction.memOperands().front()->isAtomic()) {
+ if (!isSupportedIntegerType(*Type) || (*Bytes != 4 && *Bytes != 8))
+ return makeUnexpected(ErrorCode::Unsupported,
+ "RISC-V atomic load requires a 32-bit or 64-bit GPR value");
+ return Instruction.setOpcode(
+ targetOpcode(*Bytes == 4 ? InstructionOpcode::LR_W : InstructionOpcode::LR_D));
+ }
+ auto AppendOffset = appendImmediate(Instruction, 0);
+ if (!AppendOffset)
+ return std::unexpected(AppendOffset.error());
+ if (isSupportedIntegerType(*Type)) {
+ const InstructionOpcode Selected = *Bytes == 1 ? InstructionOpcode::LB
+ : *Bytes == 2 ? InstructionOpcode::LH
+ : *Bytes == 4 ? InstructionOpcode::LW
+ : *Bytes == 8 ? InstructionOpcode::LD
+ : InstructionOpcode::Invalid;
+ if (Selected == InstructionOpcode::Invalid)
+ return makeUnexpected(ErrorCode::Unsupported, "RISC-V integer load width is unsupported");
+ return Instruction.setOpcode(targetOpcode(Selected));
+ }
+ if (Type->isFloat() && Type->scalarBitWidth() == 32)
+ return Instruction.setOpcode(targetOpcode(InstructionOpcode::FLW));
+ if (Type->isFloat() && Type->scalarBitWidth() == 64)
+ return Instruction.setOpcode(targetOpcode(InstructionOpcode::FLD));
+ return makeUnexpected(ErrorCode::Unsupported, "RISC-V cannot select this generic load type");
+ }
+
+ [[nodiscard]] static Expected<void> selectExtLoad(MachineInstr &Instruction,
+ std::optional<LLT> Type, bool Signed) {
+ if (!Type || !isSupportedIntegerType(*Type))
+ return makeUnexpected(ErrorCode::Unsupported,
+ "RISC-V extension load requires an integer result");
+ const auto Bytes = memoryAccessBytes(Instruction);
+ if (!Bytes)
+ return makeUnexpected(ErrorCode::Unsupported, "RISC-V extension load has an invalid width");
+ const InstructionOpcode Selected =
+ *Bytes == 1 ? (Signed ? InstructionOpcode::LB : InstructionOpcode::LBU)
+ : *Bytes == 2 ? (Signed ? InstructionOpcode::LH : InstructionOpcode::LHU)
+ : *Bytes == 4 ? (Signed ? InstructionOpcode::LW : InstructionOpcode::LWU)
+ : InstructionOpcode::Invalid;
+ if (Selected == InstructionOpcode::Invalid)
+ return makeUnexpected(ErrorCode::Unsupported, "RISC-V extension load width is unsupported");
+ auto Immediate = appendImmediate(Instruction, 0);
+ if (!Immediate)
+ return std::unexpected(Immediate.error());
+ return Instruction.setOpcode(targetOpcode(Selected));
+ }
+
+ [[nodiscard]] static Expected<void> selectFloatExtLoad(MachineInstr &Instruction,
+ MachineFunction &Function,
+ const TargetDescription &Target) {
+ const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+ const Register *Destination =
+ DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+ if (Destination == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V floating extension load has no destination register");
+ const Register DestinationRegister = *Destination;
+ auto F32 = LLT::makeFloat(FloatSemantics::IEEESingle);
+ if (!F32)
+ return std::unexpected(F32.error());
+ auto Temporary = createTemporary(Function, *F32, Target);
+ if (!Temporary)
+ return std::unexpected(Temporary.error());
+ auto SetDestination = Instruction.operandAt(0)->setRegister(*Temporary);
+ if (!SetDestination)
+ return std::unexpected(SetDestination.error());
+ auto Immediate = appendImmediate(Instruction, 0);
+ if (!Immediate)
+ return std::unexpected(Immediate.error());
+ auto SetOpcode = Instruction.setOpcode(targetOpcode(InstructionOpcode::FLW));
+ if (!SetOpcode)
+ return std::unexpected(SetOpcode.error());
+ MachineIRBuilder Builder(Function, Target);
+ auto Conversion = buildImmediateAfter(Builder, Instruction, InstructionOpcode::FCVT_D_S,
+ DestinationRegister, *Temporary, 0);
+ if (!Conversion)
+ return std::unexpected(Conversion.error());
+ return {};
+ }
+
+ [[nodiscard]] static Expected<void> selectFloatTruncStore(MachineInstr &Instruction,
+ MachineFunction &Function,
+ const TargetDescription &Target) {
+ const MachineOperand *ValueOperand = Instruction.operandAt(0);
+ const Register *Value = ValueOperand == nullptr ? nullptr : ValueOperand->getIf<Register>();
+ if (Value == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V floating truncation store has no value register");
+ const Register ValueRegister = *Value;
+ auto F32 = LLT::makeFloat(FloatSemantics::IEEESingle);
+ if (!F32)
+ return std::unexpected(F32.error());
+ auto Temporary = createTemporary(Function, *F32, Target);
+ if (!Temporary)
+ return std::unexpected(Temporary.error());
+ MachineIRBuilder Builder(Function, Target);
+ auto Point = Builder.setInsertPointBefore(Instruction);
+ if (!Point)
+ return std::unexpected(Point.error());
+ auto Conversion = Builder.buildInstr(targetOpcode(InstructionOpcode::FCVT_S_D));
+ if (!Conversion)
+ return std::unexpected(Conversion.error());
+ auto Def = Conversion->addDef(*Temporary);
+ if (!Def)
+ return std::unexpected(Def.error());
+ auto Use = Conversion->addUse(ValueRegister);
+ if (!Use)
+ return std::unexpected(Use.error());
+ auto Rounding = Conversion->addImmediate(0);
+ if (!Rounding)
+ return std::unexpected(Rounding.error());
+ auto Built = Conversion->build();
+ if (!Built)
+ return std::unexpected(Built.error());
+ auto SetValue = Instruction.operandAt(0)->setRegister(*Temporary);
+ if (!SetValue)
+ return std::unexpected(SetValue.error());
+ auto Immediate = appendImmediate(Instruction, 0);
+ if (!Immediate)
+ return std::unexpected(Immediate.error());
+ return Instruction.setOpcode(targetOpcode(InstructionOpcode::FSW));
+ }
+
+ [[nodiscard]] static Expected<void> selectStore(MachineInstr &Instruction,
+ MachineFunction &Function) {
+ const MachineOperand *Value = Instruction.operandAt(0);
+ const std::optional<LLT> Type =
+ Value == nullptr ? std::nullopt : registerType(Function, *Value);
+ if (!Type)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "store value has no virtual-register type");
+ const auto Bytes = memoryAccessBytes(Instruction);
+ if (!Bytes)
+ return makeUnexpected(ErrorCode::Unsupported,
+ "RISC-V store size does not match its value type");
+ if (Instruction.memOperands().front()->isAtomic()) {
+ if (!isSupportedIntegerType(*Type) || (*Bytes != 4 && *Bytes != 8))
+ return makeUnexpected(ErrorCode::Unsupported,
+ "RISC-V atomic store requires a 32-bit or 64-bit GPR value");
+ auto Zero = physicalRegister(PhysicalRegisterID::X0);
+ if (!Zero)
+ return std::unexpected(Zero.error());
+ auto Destination = insertPhysicalRegister(Instruction, 0, *Zero, RegisterOperandFlag::Define);
+ if (!Destination)
+ return std::unexpected(Destination.error());
+ return Instruction.setOpcode(
+ targetOpcode(*Bytes == 4 ? InstructionOpcode::AMOSWAP_W : InstructionOpcode::AMOSWAP_D));
+ }
+ auto AppendOffset = appendImmediate(Instruction, 0);
+ if (!AppendOffset)
+ return std::unexpected(AppendOffset.error());
+ if (isSupportedIntegerType(*Type)) {
+ const InstructionOpcode Selected = *Bytes == 1 ? InstructionOpcode::SB
+ : *Bytes == 2 ? InstructionOpcode::SH
+ : *Bytes == 4 ? InstructionOpcode::SW
+ : *Bytes == 8 ? InstructionOpcode::SD
+ : InstructionOpcode::Invalid;
+ if (Selected == InstructionOpcode::Invalid)
+ return makeUnexpected(ErrorCode::Unsupported, "RISC-V integer store width is unsupported");
+ return Instruction.setOpcode(targetOpcode(Selected));
+ }
+ if (Type->isFloat() && Type->scalarBitWidth() == 32)
+ return Instruction.setOpcode(targetOpcode(InstructionOpcode::FSW));
+ if (Type->isFloat() && Type->scalarBitWidth() == 64)
+ return Instruction.setOpcode(targetOpcode(InstructionOpcode::FSD));
+ return makeUnexpected(ErrorCode::Unsupported, "RISC-V cannot select this generic store type");
+ }
+
+ [[nodiscard]] static Expected<void> selectBranch(MachineInstr &Instruction) {
+ auto Zero = physicalRegister(PhysicalRegisterID::X0);
+ if (!Zero)
+ return std::unexpected(Zero.error());
+ auto Inserted = insertPhysicalRegister(Instruction, 0, *Zero, RegisterOperandFlag::Define);
+ if (!Inserted)
+ return std::unexpected(Inserted.error());
+ return Instruction.setOpcode(targetOpcode(InstructionOpcode::JAL));
+ }
+
+ [[nodiscard]] static Expected<void> selectConditionalBranch(MachineInstr &Instruction) {
+ auto Zero = physicalRegister(PhysicalRegisterID::X0);
+ if (!Zero)
+ return std::unexpected(Zero.error());
+ auto Inserted = insertPhysicalRegister(Instruction, 1, *Zero, RegisterOperandFlag::Use);
+ if (!Inserted)
+ return std::unexpected(Inserted.error());
+ return Instruction.setOpcode(targetOpcode(InstructionOpcode::BNE));
+ }
+
+ [[nodiscard]] static Expected<void> selectIntegerConstant(MachineInstr &Instruction,
+ MachineFunction &Function,
+ const TargetDescription &Target,
+ std::optional<LLT> Type) {
+ if (!Type || !isSupportedIntegerType(*Type)) {
+ return makeUnexpected(ErrorCode::Unsupported,
+ "RISC-V constant selection requires an i1-i64 or pointer result");
+ }
+ const MachineOperand *ImmediateOperand = Instruction.operandAt(1);
+ if (ImmediateOperand == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V generic constant has no immediate operand");
+ auto ImmediateValue = signedImmediate(*ImmediateOperand);
+ if (!ImmediateValue)
+ return std::unexpected(ImmediateValue.error());
+ const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+ const Register *DestinationValue =
+ DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+ if (DestinationValue == nullptr || !DestinationValue->isVirtual()) {
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V generic constant must define a virtual register");
+ }
+ const Register Destination = *DestinationValue;
+ const std::int64_t Value = *ImmediateValue;
+ const bool IsWord = Type->isInteger() && Type->scalarBitWidth() <= 32;
+ const InstructionOpcode AddImmediate =
+ IsWord ? InstructionOpcode::ADDIW : InstructionOpcode::ADDI;
+ const InstructionOpcode ShiftLeftImmediate =
+ IsWord ? InstructionOpcode::SLLIW : InstructionOpcode::SLLI;
+ if (fitsSigned(Value, 12)) {
+ auto Zero = physicalRegister(PhysicalRegisterID::X0);
+ if (!Zero)
+ return std::unexpected(Zero.error());
+ auto Source = insertPhysicalRegister(Instruction, 1, *Zero, RegisterOperandFlag::Use);
+ if (!Source)
+ return std::unexpected(Source.error());
+ return Instruction.setOpcode(targetOpcode(AddImmediate));
+ }
+
+ // Recursively decompose the value into signed 12-bit chunks. The
+ // quotient is computed from an unsigned bit pattern with explicit sign
+ // extension so both INT64_MIN and INT64_MAX remain well-defined.
+ std::uint64_t Remaining = static_cast<std::uint64_t>(Value);
+ std::vector<std::int64_t> Digits;
+ while (!fitsSigned(std::bit_cast<std::int64_t>(Remaining), 12)) {
+ const std::uint64_t LowBits = Remaining & 0xFFFU;
+ std::int64_t Digit = static_cast<std::int64_t>(LowBits);
+ if (Digit >= 0x800)
+ Digit -= 0x1000;
+ Digits.push_back(Digit);
+
+ const std::uint64_t Adjusted = Remaining - static_cast<std::uint64_t>(Digit);
+ std::uint64_t Quotient = Adjusted >> 12U;
+ if ((Remaining & (std::uint64_t{1} << 63U)) != 0)
+ Quotient |= ~std::uint64_t{0} << 52U;
+ Remaining = Quotient;
+ }
+ const std::int64_t Base = std::bit_cast<std::int64_t>(Remaining);
+
+ auto Temporary = createTemporary(Function, *Type, Target);
+ if (!Temporary)
+ return std::unexpected(Temporary.error());
+ auto *Parent = Instruction.parent();
+ if (Parent == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V constant selection received a detached instruction");
+ auto SetOpcode = Instruction.setOpcode(targetOpcode(AddImmediate));
+ if (!SetOpcode)
+ return std::unexpected(SetOpcode.error());
+ auto SetDestination = Instruction.operandAt(0)->setRegister(*Temporary);
+ if (!SetDestination)
+ return std::unexpected(SetDestination.error());
+ auto EraseImmediate = Instruction.eraseOperand(1);
+ if (!EraseImmediate)
+ return std::unexpected(EraseImmediate.error());
+ auto Zero = physicalRegister(PhysicalRegisterID::X0);
+ if (!Zero)
+ return std::unexpected(Zero.error());
+ auto AddSource = insertPhysicalRegister(Instruction, 1, *Zero, RegisterOperandFlag::Use);
+ if (!AddSource)
+ return std::unexpected(AddSource.error());
+ auto AddBase = appendImmediate(Instruction, Base);
+ if (!AddBase)
+ return std::unexpected(AddBase.error());
+
+ MachineIRBuilder Builder(Function, Target);
+ auto InsertPoint = Builder.setInsertPointAfter(Instruction);
+ if (!InsertPoint)
+ return std::unexpected(InsertPoint.error());
+
+ Register Current = *Temporary;
+ for (std::size_t Reverse = Digits.size(); Reverse != 0; --Reverse) {
+ const std::size_t Index = Reverse - 1U;
+ auto ShiftResult = createTemporary(Function, *Type, Target);
+ if (!ShiftResult)
+ return std::unexpected(ShiftResult.error());
+ auto Shift = Builder.buildInstr(targetOpcode(ShiftLeftImmediate));
+ if (!Shift)
+ return std::unexpected(Shift.error());
+ auto ShiftDef = Shift->addDef(*ShiftResult);
+ if (!ShiftDef)
+ return std::unexpected(ShiftDef.error());
+ auto ShiftUse = Shift->addUse(Current);
+ if (!ShiftUse)
+ return std::unexpected(ShiftUse.error());
+ auto ShiftAmount = Shift->addImmediate(12);
+ if (!ShiftAmount)
+ return std::unexpected(ShiftAmount.error());
+ auto BuiltShift = Shift->build();
+ if (!BuiltShift)
+ return std::unexpected(BuiltShift.error());
+
+ const bool IsFinal = Index == 0;
+ Register AddDestination;
+ if (IsFinal) {
+ AddDestination = Destination;
+ } else {
+ auto AddTemporary = createTemporary(Function, *Type, Target);
+ if (!AddTemporary)
+ return std::unexpected(AddTemporary.error());
+ AddDestination = *AddTemporary;
+ }
+ auto Add = Builder.setInsertPointAfter(**BuiltShift);
+ if (!Add)
+ return std::unexpected(Add.error());
+ auto AddInstruction = Builder.buildInstr(targetOpcode(AddImmediate));
+ if (!AddInstruction)
+ return std::unexpected(AddInstruction.error());
+ auto AddDef = AddInstruction->addDef(AddDestination);
+ if (!AddDef)
+ return std::unexpected(AddDef.error());
+ auto AddUse = AddInstruction->addUse(*ShiftResult);
+ if (!AddUse)
+ return std::unexpected(AddUse.error());
+ auto AddImmediateResult = AddInstruction->addImmediate(Digits[Index]);
+ if (!AddImmediateResult)
+ return std::unexpected(AddImmediateResult.error());
+ auto BuiltAdd = AddInstruction->build();
+ if (!BuiltAdd)
+ return std::unexpected(BuiltAdd.error());
+ Current = AddDestination;
+ }
+ return {};
+ }
+
+ [[nodiscard]] static Expected<void> selectFloatingConstant(MachineInstr &Instruction,
+ MachineFunction &Function,
+ const TargetDescription &Target,
+ std::optional<LLT> Type) {
+ if (!Type || !isRVFloatType(*Type))
+ return makeUnexpected(ErrorCode::Unsupported, "RISC-V floating constant requires f32 or f64");
+ const MachineOperand *Immediate = Instruction.operandAt(1);
+ const auto *Floating =
+ Immediate == nullptr ? nullptr : Immediate->getIf<FloatingImmediateOperand>();
+ const std::optional<FloatSemantics> Semantics = Type->floatSemantics();
+ if (Floating == nullptr || !Semantics || Floating->Bits.semantics() != *Semantics)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V floating constant has mismatched bit semantics");
+ const MachineOperand *DestinationOperand = Instruction.operandAt(0);
+ const Register *Destination =
+ DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>();
+ if (Destination == nullptr || !Destination->isVirtual())
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V floating constant must define a virtual register");
+ const Register DestinationRegister = *Destination;
+
+ const std::uint32_t BitWidth = Type->scalarBitWidth();
+ const std::uint64_t Bits = Floating->Bits.words().empty() ? 0 : Floating->Bits.words().front();
+ auto Integer = IntegerImmediate::fromUnsigned(64, Bits);
+ if (!Integer)
+ return std::unexpected(Integer.error());
+ auto IntegerType = LLT::makeInteger(64);
+ if (!IntegerType)
+ return std::unexpected(IntegerType.error());
+ auto Temporary = createTemporary(Function, *IntegerType, Target);
+ if (!Temporary)
+ return std::unexpected(Temporary.error());
+
+ auto SetOpcode = Instruction.setOpcode(opcode(GenericOpcode::G_CONSTANT));
+ if (!SetOpcode)
+ return std::unexpected(SetOpcode.error());
+ auto SetDestination = Instruction.operandAt(0)->setRegister(*Temporary);
+ if (!SetDestination)
+ return std::unexpected(SetDestination.error());
+ auto EraseImmediate = Instruction.eraseOperand(1);
+ if (!EraseImmediate)
+ return std::unexpected(EraseImmediate.error());
+ auto AddInteger = Instruction.addOperand(MachineOperand::integerImmediate(*Integer));
+ if (!AddInteger)
+ return std::unexpected(AddInteger.error());
+ auto IntegerSelection = selectIntegerConstant(Instruction, Function, Target, *IntegerType);
+ if (!IntegerSelection)
+ return std::unexpected(IntegerSelection.error());
+
+ MachineInstr *IntegerDefinition = nullptr;
+ for (MachineOperand &DefOperand : Function.registerInfo().defOperands(*Temporary)) {
+ IntegerDefinition = DefOperand.parent();
+ break;
+ }
+ if (IntegerDefinition == nullptr)
+ return makeUnexpected(ErrorCode::InternalError,
+ "RISC-V floating constant lost its integer materialization");
+ MachineIRBuilder Builder(Function, Target);
+ auto InsertPoint = Builder.setInsertPointAfter(*IntegerDefinition);
+ if (!InsertPoint)
+ return std::unexpected(InsertPoint.error());
+ const InstructionOpcode Move =
+ BitWidth == 32 ? InstructionOpcode::FMV_W_X : InstructionOpcode::FMV_D_X;
+ auto MoveInstruction = Builder.buildInstr(targetOpcode(Move));
+ if (!MoveInstruction)
+ return std::unexpected(MoveInstruction.error());
+ auto MoveDef = MoveInstruction->addDef(DestinationRegister);
+ if (!MoveDef)
+ return std::unexpected(MoveDef.error());
+ auto MoveUse = MoveInstruction->addUse(*Temporary);
+ if (!MoveUse)
+ return std::unexpected(MoveUse.error());
+ auto BuiltMove = MoveInstruction->build();
+ if (!BuiltMove)
+ return std::unexpected(BuiltMove.error());
+ return {};
+ }
+
+ [[nodiscard]] static Expected<void> selectRoundingBinary(MachineInstr &Instruction,
+ std::optional<LLT> Type,
+ InstructionOpcode Single,
+ InstructionOpcode Double) {
+ auto RoundingMode = appendImmediate(Instruction, 0);
+ if (!RoundingMode)
+ return std::unexpected(RoundingMode.error());
+ if (Type && Type->isFloat() && Type->scalarBitWidth() == 32)
+ return Instruction.setOpcode(targetOpcode(Single));
+ if (Type && Type->isFloat() && Type->scalarBitWidth() == 64)
+ return Instruction.setOpcode(targetOpcode(Double));
+ return makeUnexpected(ErrorCode::Unsupported, "RISC-V cannot select this floating binary type");
+ }
+
+ [[nodiscard]] static Expected<void> selectRoundingTernary(MachineInstr &Instruction,
+ std::optional<LLT> Type,
+ InstructionOpcode Single,
+ InstructionOpcode Double) {
+ return selectRoundingBinary(Instruction, Type, Single, Double);
+ }
+
+ [[nodiscard]] static Expected<void> selectRoundingUnary(MachineInstr &Instruction,
+ std::optional<LLT> Type,
+ InstructionOpcode Single,
+ InstructionOpcode Double) {
+ return selectRoundingBinary(Instruction, Type, Single, Double);
+ }
+
+ [[nodiscard]] static Expected<void> selectSignInjection(MachineInstr &Instruction,
+ std::optional<LLT> Type,
+ InstructionOpcode Single,
+ InstructionOpcode Double) {
+ const MachineOperand *Source = Instruction.operandAt(1);
+ const Register *SourceReg = Source == nullptr ? nullptr : Source->getIf<Register>();
+ if (SourceReg == nullptr)
+ return makeUnexpected(ErrorCode::VerificationError,
+ "RISC-V floating sign operation requires a register source");
+ auto Duplicate = insertVirtualUse(Instruction, 2, *SourceReg);
+ if (!Duplicate)
+ return std::unexpected(Duplicate.error());
+ if (Type && Type->isFloat() && Type->scalarBitWidth() == 32)
+ return Instruction.setOpcode(targetOpcode(Single));
+ if (Type && Type->isFloat() && Type->scalarBitWidth() == 64)
+ return Instruction.setOpcode(targetOpcode(Double));
+ return makeUnexpected(ErrorCode::Unsupported, "RISC-V cannot select this floating unary type");
+ }
+};
+
+} // namespace
+
+const TargetLegalizerInfo &legalizerInfo() noexcept {
+ static const RISCVLegalizerInfo Value;
+ return Value;
+}
+
+const TargetRegisterBankInfo &registerBankInfo() noexcept {
+ static const RISCVRegisterBankInfo Value;
+ return Value;
+}
+
+const TargetInstructionSelector &instructionSelector() noexcept {
+ static const RISCVInstructionSelector Value;
+ return Value;
+}
+
+} // namespace rcc::riscv
generated by cgit v1.2.3 (git 2.46.0) at 2026-10-29 18:59:46 +0000