// SPDX-License-Identifier: GPL-3.0-or-later #include "rcc/Target/RISCV/RISCVCodeGen.h" #include "RISCVImmediateMaterialization.h" #include "rcc/CodeGen/GenericOpcodes.h" #include "rcc/CodeGen/LowLevelType.h" #include "rcc/CodeGen/MachineBasicBlock.h" #include "rcc/CodeGen/MachineFunction.h" #include "rcc/CodeGen/MachineIRBuilder.h" #include "rcc/CodeGen/MachineInstr.h" #include "rcc/CodeGen/MachineMemOperand.h" #include "rcc/CodeGen/MachineOperand.h" #include "rcc/CodeGen/MachineRegisterInfo.h" #include "rcc/CodeGen/TargetDescription.h" #include "rcc/Target/RISCV/RISCVCallLowering.h" #include "rcc/Target/RISCV/RISCVTarget.h" #include #include #include #include #include #include #include #include #include #include namespace rcc::riscv { namespace { [[nodiscard]] Opcode targetOpcode(InstructionOpcode Value) noexcept { return Opcode::fromValue(static_cast(Value)); } [[nodiscard]] std::optional registerType(const MachineFunction &Function, const MachineOperand &Operand) { const Register *Reg = Operand.getIf(); if (Reg == nullptr || !Reg->isVirtual()) return std::nullopt; auto Info = Function.registerInfo().virtualRegisterInfo(*Reg); if (!Info) return std::nullopt; return Info->get().Type; } [[nodiscard]] std::optional resultType(const MachineFunction &Function, const MachineInstr &Instruction) { const MachineOperand *Result = Instruction.operandAt(0); return Result == nullptr ? std::nullopt : registerType(Function, *Result); } [[nodiscard]] std::optional operandType(const MachineFunction &Function, const MachineInstr &Instruction, std::size_t Index) { const MachineOperand *Operand = Instruction.operandAt(Index); return Operand == nullptr ? std::nullopt : registerType(Function, *Operand); } [[nodiscard]] bool isRVFloatType(LLT Type) noexcept { if (!Type.isFloat()) return false; const std::optional Semantics = Type.floatSemantics(); return Semantics && (*Semantics == FloatSemantics::IEEESingle || *Semantics == FloatSemantics::IEEEDouble); } [[nodiscard]] bool isSupportedIntegerType(LLT Type) noexcept { return ((Type.isInteger() || Type.isAnyScalar()) && Type.scalarBitWidth() != 0 && Type.scalarBitWidth() <= 64) || (Type.isPointer() && Type.scalarBitWidth() == 64); } [[nodiscard]] bool isSupportedIntegerScalar(LLT Type) noexcept { return (Type.isInteger() || Type.isAnyScalar()) && Type.scalarBitWidth() != 0 && Type.scalarBitWidth() <= 64; } [[nodiscard]] bool isNativeIntegerScalar(LLT Type) noexcept { return (Type.isInteger() || Type.isAnyScalar()) && (Type.scalarBitWidth() == 32 || Type.scalarBitWidth() == 64); } [[nodiscard]] bool isScalarGPRType(LLT Type) noexcept { return ((Type.isInteger() || Type.isAnyScalar()) && Type.scalarBitWidth() != 0 && Type.scalarBitWidth() <= 64) || (Type.isPointer() && Type.scalarBitWidth() == 64); } [[nodiscard]] bool isI1Type(LLT Type) noexcept { return (Type.isInteger() || Type.isAnyScalar()) && Type.scalarBitWidth() == 1; } [[nodiscard]] bool isScalarIntegerExtension(LLT Destination, LLT Source) noexcept { return (Destination.isInteger() || Destination.isAnyScalar()) && (Source.isInteger() || Source.isAnyScalar()) && Destination.scalarBitWidth() != 0 && Source.scalarBitWidth() != 0 && Source.scalarBitWidth() < Destination.scalarBitWidth() && Destination.scalarBitWidth() <= 64; } [[nodiscard]] Expected extendIntegerImmediate(const IntegerImmediate &Value, std::uint32_t DestinationWidth, bool SignExtend) { if (DestinationWidth == 0 || DestinationWidth > 64) return makeUnexpected(ErrorCode::Unsupported, "RISC-V constant extension requires a destination no wider than 64 bits"); std::uint64_t Raw = Value.words().empty() ? 0 : Value.words().front(); const std::uint32_t SourceWidth = Value.bitWidth(); if (SignExtend && Value.isNegative() && SourceWidth < 64) Raw |= ~std::uint64_t{0} << SourceWidth; if (DestinationWidth < 64) Raw &= (std::uint64_t{1} << DestinationWidth) - 1U; return IntegerImmediate::create(DestinationWidth, std::span(&Raw, 1)); } [[nodiscard]] Expected foldConstantExtension(MachineInstr &Instruction, MachineFunction &Function, GenericOpcode Generic) { const MachineOperand *SourceOperand = Instruction.operandAt(1); const MachineOperand *DestinationOperand = Instruction.operandAt(0); const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); const Register *Destination = DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); if (Source == nullptr || Destination == nullptr || !Source->isVirtual() || !Destination->isVirtual()) { return makeUnexpected(ErrorCode::VerificationError, "RISC-V constant extension requires virtual-register operands"); } const Register SourceRegister = *Source; const Register DestinationRegister = *Destination; const MachineOperand *SourceDefinition = Function.registerInfo().getUniqueVRegDef(SourceRegister); const MachineInstr *SourceDef = SourceDefinition == nullptr ? nullptr : SourceDefinition->parent(); if (SourceDef == nullptr || SourceDef->opcode() != opcode(GenericOpcode::G_CONSTANT)) return makeUnexpected(ErrorCode::Unsupported, "RISC-V constant extension requires a single constant definition"); const MachineOperand *ConstantOperand = SourceDef->operandAt(1); const auto *Integer = ConstantOperand == nullptr ? nullptr : ConstantOperand->getIf(); if (Integer == nullptr) return makeUnexpected(ErrorCode::Unsupported, "RISC-V constant extension requires an integer constant"); const auto DestinationInfo = Function.registerInfo().virtualRegisterInfo(*Destination); if (!DestinationInfo || !DestinationInfo->get().Type) return makeUnexpected(ErrorCode::VerificationError, "RISC-V constant extension destination has no type"); const std::uint32_t DestinationWidth = DestinationInfo->get().Type->scalarBitWidth(); const bool SignExtend = Generic != GenericOpcode::G_ZEXT; auto Extended = extendIntegerImmediate(Integer->Value, DestinationWidth, SignExtend); if (!Extended) return std::unexpected(Extended.error()); if (Function.registerInfo().hasOneUse(SourceRegister)) { MachineInstr *Def = const_cast(SourceDef); if (auto Replaced = Def->operandAt(0)->setRegister(DestinationRegister); !Replaced) return std::unexpected(Replaced.error()); auto Erased = Def->eraseOperand(1); if (!Erased) return std::unexpected(Erased.error()); auto Added = Def->addOperand(MachineOperand::integerImmediate(std::move(*Extended))); if (!Added) return std::unexpected(Added.error()); MachineBasicBlock *Parent = Instruction.parent(); if (Parent == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V constant extension is detached during folding"); return Parent->removeInstruction(Instruction); } auto SetOpcode = Instruction.setOpcode(opcode(GenericOpcode::G_CONSTANT)); if (!SetOpcode) return std::unexpected(SetOpcode.error()); while (Instruction.operandCount() > 1) { auto Erased = Instruction.eraseOperand(1); if (!Erased) return std::unexpected(Erased.error()); } auto Added = Instruction.addOperand(MachineOperand::integerImmediate(std::move(*Extended))); if (!Added) return std::unexpected(Added.error()); if (Function.registerInfo().useEmpty(SourceRegister)) { MachineInstr *Def = const_cast(SourceDef); MachineBasicBlock *Parent = Def->parent(); if (Parent == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V constant extension source definition is detached"); auto Removed = Parent->removeInstruction(*Def); if (!Removed) return std::unexpected(Removed.error()); // Keep the source register record as a tombstone-free historical vreg until // the legalizer worklist transaction has completed. } return {}; } struct WideConstantUnmergePlan final { MachineInstr *Extension = nullptr; MachineInstr *Unmerge = nullptr; std::vector Outputs; bool SignExtend = false; }; [[nodiscard]] MachineInstr *singleRegisterUse(const MachineFunction &Function, Register Reg) { MachineInstr *UseInstruction = nullptr; auto &RegisterInfo = const_cast(Function.registerInfo()); for (MachineOperand &Operand : RegisterInfo.useOperands(Reg)) { if (Operand.parent() == nullptr) continue; if (UseInstruction != nullptr) return nullptr; UseInstruction = Operand.parent(); } return UseInstruction; } [[nodiscard]] std::optional wideConstantUnmergePlan(const MachineInstr &Constant, const MachineFunction &Function) { const MachineOperand *ResultOperand = Constant.operandAt(0); const Register *Result = ResultOperand == nullptr ? nullptr : ResultOperand->getIf(); if (Result == nullptr || !Result->isVirtual()) return std::nullopt; MachineInstr *FirstUse = singleRegisterUse(Function, *Result); if (FirstUse == nullptr) return std::nullopt; MachineInstr *Extension = nullptr; MachineInstr *Unmerge = FirstUse; if (FirstUse->opcode() == opcode(GenericOpcode::G_ANYEXT) || FirstUse->opcode() == opcode(GenericOpcode::G_SEXT) || FirstUse->opcode() == opcode(GenericOpcode::G_ZEXT)) { Extension = FirstUse; const MachineOperand *ExtendedOperand = Extension->operandAt(0); const Register *Extended = ExtendedOperand == nullptr ? nullptr : ExtendedOperand->getIf(); if (Extended == nullptr || !Extended->isVirtual()) return std::nullopt; Unmerge = singleRegisterUse(Function, *Extended); if (Unmerge == nullptr) return std::nullopt; } if (Unmerge->opcode() != opcode(GenericOpcode::G_UNMERGE_VALUES) || Unmerge->operandCount() < 3) return std::nullopt; WideConstantUnmergePlan Plan; Plan.Extension = Extension; Plan.Unmerge = Unmerge; Plan.SignExtend = Extension != nullptr && Extension->opcode() != opcode(GenericOpcode::G_ZEXT); for (std::size_t Index = 0; Index + 1U < Unmerge->operandCount(); ++Index) { const MachineOperand *Operand = Unmerge->operandAt(Index); const Register *Reg = Operand == nullptr ? nullptr : Operand->getIf(); if (Reg == nullptr || !Reg->isVirtual() || !Operand->isDef()) return std::nullopt; const auto Info = Function.registerInfo().virtualRegisterInfo(*Reg); if (!Info || !Info->get().Type || Info->get().Type->scalarBitWidth() == 0 || Info->get().Type->scalarBitWidth() > 64 || (!Info->get().Type->isInteger() && !Info->get().Type->isAnyScalar())) return std::nullopt; Plan.Outputs.push_back(*Reg); } const MachineOperand *SourceOperand = Unmerge->operandAt(Unmerge->operandCount() - 1U); const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); const Register *ExpectedSource = Result; if (Extension != nullptr) { const MachineOperand *ExtendedOperand = Extension->operandAt(0); ExpectedSource = ExtendedOperand == nullptr ? nullptr : ExtendedOperand->getIf(); } if (Source == nullptr || ExpectedSource == nullptr || *Source != *ExpectedSource) return std::nullopt; return Plan; } [[nodiscard]] Expected wideConstantWord(const IntegerImmediate &Value, std::size_t WordIndex, std::uint32_t Width, bool SignExtend) { std::uint64_t Raw = 0; if (WordIndex < Value.words().size()) Raw = Value.words()[WordIndex]; else if (SignExtend && Value.isNegative()) Raw = std::numeric_limits::max(); const std::size_t WordBitOffset = WordIndex * 64U; if (SignExtend && Value.isNegative() && WordBitOffset < Value.bitWidth()) { const std::uint32_t Remaining = Value.bitWidth() - static_cast(WordBitOffset); if (Remaining < 64) Raw |= ~std::uint64_t{0} << Remaining; } if (Width < 64) Raw &= (std::uint64_t{1} << Width) - 1U; return IntegerImmediate::create(Width, std::span(&Raw, 1)); } [[nodiscard]] Expected foldWideConstant(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target) { const auto Plan = wideConstantUnmergePlan(Instruction, Function); if (!Plan) return makeUnexpected(ErrorCode::Unsupported, "RISC-V wide constant requires a G_UNMERGE_VALUES use"); const MachineOperand *ImmediateOperand = Instruction.operandAt(1); const auto *Integer = ImmediateOperand == nullptr ? nullptr : ImmediateOperand->getIf(); if (Integer == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide constant has no integer immediate"); std::vector Words; Words.reserve(Plan->Outputs.size()); for (std::size_t Index = 0; Index < Plan->Outputs.size(); ++Index) { const auto Info = Function.registerInfo().virtualRegisterInfo(Plan->Outputs[Index]); if (!Info || !Info->get().Type) return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide constant output has no virtual-register type"); auto Word = wideConstantWord(Integer->Value, Index, Info->get().Type->scalarBitWidth(), Plan->SignExtend); if (!Word) return std::unexpected(Word.error()); Words.push_back(std::move(*Word)); } std::vector Representatives(Plan->Outputs.size()); for (std::size_t Index = 0; Index < Representatives.size(); ++Index) { Representatives[Index] = Index; for (std::size_t Previous = 0; Previous < Index; ++Previous) { if (Words[Previous] == Words[Index]) { Representatives[Index] = Previous; break; } } } const Register FirstOutput = Plan->Outputs.front(); auto Retargeted = Instruction.operandAt(0)->setRegister(FirstOutput); if (!Retargeted) return std::unexpected(Retargeted.error()); auto SetOpcode = Instruction.setOpcode(opcode(GenericOpcode::G_CONSTANT)); if (!SetOpcode) return std::unexpected(SetOpcode.error()); while (Instruction.operandCount() > 1) { auto Erased = Instruction.eraseOperand(1); if (!Erased) return std::unexpected(Erased.error()); } auto Added = Instruction.addOperand(MachineOperand::integerImmediate(std::move(Words.front()))); if (!Added) return std::unexpected(Added.error()); MachineBasicBlock *Block = Instruction.parent(); if (Block == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide constant is detached during legalization"); if (Plan->Unmerge != &Instruction) { auto RemovedUnmerge = Plan->Unmerge->parent()->removeInstruction(*Plan->Unmerge); if (!RemovedUnmerge) return std::unexpected(RemovedUnmerge.error()); } if (Plan->Extension != nullptr) { auto RemovedExtension = Plan->Extension->parent()->removeInstruction(*Plan->Extension); if (!RemovedExtension) return std::unexpected(RemovedExtension.error()); } for (std::size_t Index = 1; Index < Plan->Outputs.size(); ++Index) { if (Representatives[Index] == Index) continue; auto Replaced = Function.registerInfo().replaceRegWith( Plan->Outputs[Index], Plan->Outputs[Representatives[Index]]); if (!Replaced) return std::unexpected(Replaced.error()); } MachineInstr *Last = &Instruction; for (std::size_t Index = 1; Index < Plan->Outputs.size(); ++Index) { if (Representatives[Index] != Index) continue; MachineIRBuilder Builder(Function, Target); auto Point = Builder.setInsertPointAfter(*Last); if (!Point) return std::unexpected(Point.error()); auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_CONSTANT)); if (!Pending) return std::unexpected(Pending.error()); auto Def = Pending->addDef(Plan->Outputs[Index]); if (!Def) return std::unexpected(Def.error()); auto Value = Pending->addIntegerImmediate(std::move(Words[Index])); if (!Value) return std::unexpected(Value.error()); auto Built = Pending->build(); if (!Built) return std::unexpected(Built.error()); Last = *Built; } return {}; } [[nodiscard]] Expected buildIntegerConstantBefore(MachineIRBuilder &Builder, MachineInstr &Anchor, std::uint32_t Width, std::uint64_t Value); [[nodiscard]] Expected buildBinaryBefore(MachineIRBuilder &Builder, MachineInstr &Anchor, GenericOpcode Generic, LLT Type, Register Left, Register Right); [[nodiscard]] Expected buildUnaryBefore(MachineIRBuilder &Builder, MachineInstr &Anchor, GenericOpcode Generic, LLT Type, Register Source); [[nodiscard]] Expected eraseInstruction(MachineInstr &Instruction); [[nodiscard]] Expected rewriteUnary(MachineInstr &Instruction, GenericOpcode Generic, Register Destination, Register Source); struct NarrowAnyExtBinaryPlan final { MachineInstr *Binary = nullptr; GenericOpcode Generic = GenericOpcode::Invalid; std::array WideInputs{}; std::vector Artifacts; std::vector Constants; std::uint32_t SourceWidth = 0; }; [[nodiscard]] std::optional narrowAnyExtBinaryPlan(const MachineInstr &Extension, const MachineFunction &Function) { const MachineOperand *DestinationOperand = Extension.operandAt(0); const MachineOperand *SourceOperand = Extension.operandAt(1); const Register *Destination = DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); if (Destination == nullptr || Source == nullptr || !Destination->isVirtual() || !Source->isVirtual() || singleRegisterUse(Function, *Source) != &Extension) return std::nullopt; const auto DestinationInfo = Function.registerInfo().virtualRegisterInfo(*Destination); const auto SourceInfo = Function.registerInfo().virtualRegisterInfo(*Source); if (!DestinationInfo || !SourceInfo || !DestinationInfo->get().Type || !SourceInfo->get().Type) return std::nullopt; const LLT DestinationType = *DestinationInfo->get().Type; const LLT SourceType = *SourceInfo->get().Type; if (!isScalarIntegerExtension(DestinationType, SourceType) || DestinationType.scalarBitWidth() != 64) return std::nullopt; const MachineOperand *SourceDefinition = Function.registerInfo().getUniqueVRegDef(*Source); MachineInstr *Binary = SourceDefinition == nullptr ? nullptr : const_cast(SourceDefinition->parent()); if (Binary == nullptr || Binary->operandCount() != 3) return std::nullopt; const GenericOpcode Generic = static_cast(Binary->opcode().value()); if (Generic != GenericOpcode::G_ADD && Generic != GenericOpcode::G_SUB && Generic != GenericOpcode::G_MUL && Generic != GenericOpcode::G_AND && Generic != GenericOpcode::G_OR && Generic != GenericOpcode::G_XOR && Generic != GenericOpcode::G_SHL) return std::nullopt; NarrowAnyExtBinaryPlan Plan; Plan.Binary = Binary; Plan.Generic = Generic; Plan.SourceWidth = SourceType.scalarBitWidth(); const auto AddArtifact = [&Plan](MachineInstr *Artifact) { if (std::ranges::find(Plan.Artifacts, Artifact) == Plan.Artifacts.end()) Plan.Artifacts.push_back(Artifact); }; const auto TruncatedWideSource = [&](MachineInstr *Truncation, MachineInstr *OnlyUse) -> std::optional { if (Truncation == nullptr || Truncation->opcode() != opcode(GenericOpcode::G_TRUNC) || Truncation->operandCount() != 2) return std::nullopt; const MachineOperand *NarrowOperand = Truncation->operandAt(0); const Register *Narrow = NarrowOperand == nullptr ? nullptr : NarrowOperand->getIf(); const MachineOperand *WideOperand = Truncation->operandAt(1); const Register *Wide = WideOperand == nullptr ? nullptr : WideOperand->getIf(); const std::optional WideType = WideOperand == nullptr ? std::nullopt : registerType(Function, *WideOperand); if (Narrow == nullptr || Wide == nullptr || !Wide->isVirtual() || !WideType || *WideType != DestinationType || singleRegisterUse(Function, *Narrow) != OnlyUse) return std::nullopt; return *Wide; }; for (std::size_t Index = 0; Index != Plan.WideInputs.size(); ++Index) { const MachineOperand *Operand = Binary->operandAt(Index + 1U); const Register *Reg = Operand == nullptr ? nullptr : Operand->getIf(); if (Reg == nullptr || !Reg->isVirtual()) return std::nullopt; const auto Info = Function.registerInfo().virtualRegisterInfo(*Reg); if (!Info || !Info->get().Type) return std::nullopt; if (*Info->get().Type == DestinationType) { Plan.WideInputs[Index] = *Reg; continue; } if (*Info->get().Type != SourceType) return std::nullopt; const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(*Reg); MachineInstr *Def = Definition == nullptr ? nullptr : const_cast(Definition->parent()); if (auto Wide = TruncatedWideSource(Def, Binary)) { Plan.WideInputs[Index] = *Wide; AddArtifact(Def); continue; } if (Def != nullptr && (Def->opcode() == opcode(GenericOpcode::COPY) || Def->opcode() == opcode(GenericOpcode::G_ANYEXT)) && Def->operandCount() == 2 && singleRegisterUse(Function, *Reg) == Binary) { const MachineOperand *NarrowOperand = Def->operandAt(1); const Register *Narrow = NarrowOperand == nullptr ? nullptr : NarrowOperand->getIf(); const MachineOperand *NarrowDefinition = Narrow == nullptr ? nullptr : Function.registerInfo().getUniqueVRegDef(*Narrow); MachineInstr *Truncation = NarrowDefinition == nullptr ? nullptr : const_cast(NarrowDefinition->parent()); if (auto Wide = TruncatedWideSource(Truncation, Def)) { Plan.WideInputs[Index] = *Wide; AddArtifact(Def); AddArtifact(Truncation); continue; } } if (Def != nullptr && Def->opcode() == opcode(GenericOpcode::G_CONSTANT) && singleRegisterUse(Function, *Reg) == Binary) { Plan.WideInputs[Index] = *Reg; Plan.Constants.push_back(Def); continue; } return std::nullopt; } return Plan; } [[nodiscard]] Expected widenConstantToRV64(MachineInstr &Instruction, MachineFunction &Function) { const MachineOperand *DestinationOperand = Instruction.operandAt(0); const Register *Destination = DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); const MachineOperand *ImmediateOperand = Instruction.operandAt(1); const auto *Immediate = ImmediateOperand == nullptr ? nullptr : ImmediateOperand->getIf(); if (Destination == nullptr || !Destination->isVirtual() || Immediate == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V narrow constant legalization has invalid operands"); auto Type = LLT::makeAnyScalar(64); if (!Type) return std::unexpected(Type.error()); auto SetType = Function.registerInfo().setType(*Destination, *Type); if (!SetType) return std::unexpected(SetType.error()); auto Extended = extendIntegerImmediate(Immediate->Value, 64, true); if (!Extended) return std::unexpected(Extended.error()); auto Erased = Instruction.eraseOperand(1); if (!Erased) return std::unexpected(Erased.error()); auto Added = Instruction.addOperand(MachineOperand::integerImmediate(std::move(*Extended))); if (!Added) return std::unexpected(Added.error()); return {}; } [[nodiscard]] Expected widenAtomicCompareExchangeSource(MachineInstr &Instruction, MachineFunction &Function, MachineIRBuilder &Builder, Register Source, LLT WideType) { const auto SourceInfo = Function.registerInfo().virtualRegisterInfo(Source); if (!SourceInfo || !SourceInfo->get().Type) return makeUnexpected(ErrorCode::VerificationError, "RISC-V compare-exchange source has no scalar type"); if (SourceInfo->get().Type->scalarBitWidth() == WideType.scalarBitWidth()) return Source; const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(Source); MachineInstr *Def = Definition == nullptr ? nullptr : const_cast(Definition->parent()); if (Def != nullptr && Def->opcode() == opcode(GenericOpcode::G_CONSTANT) && singleRegisterUse(Function, Source) == &Instruction) { auto Widened = widenConstantToRV64(*Def, Function); if (!Widened) return std::unexpected(Widened.error()); return Source; } return buildUnaryBefore(Builder, Instruction, GenericOpcode::G_ANYEXT, WideType, Source); } [[nodiscard]] Expected lowerAtomicCompareExchange(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target) { const std::optional OriginalType = resultType(Function, Instruction); const MachineOperand *DestinationOperand = Instruction.operandAt(0); const MachineOperand *CompareOperand = Instruction.operandAt(2); const MachineOperand *NewValueOperand = Instruction.operandAt(3); const Register *Destination = DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); const Register *Compare = CompareOperand == nullptr ? nullptr : CompareOperand->getIf(); const Register *NewValue = NewValueOperand == nullptr ? nullptr : NewValueOperand->getIf(); if (!OriginalType || Destination == nullptr || Compare == nullptr || NewValue == nullptr || OriginalType->scalarBitWidth() >= 64) return makeUnexpected(ErrorCode::VerificationError, "RISC-V narrow compare-exchange has invalid operands"); const Register OriginalDestination = *Destination; auto WideType = LLT::makeAnyScalar(64); if (!WideType) return std::unexpected(WideType.error()); MachineIRBuilder Builder(Function, Target); auto WideCompare = widenAtomicCompareExchangeSource(Instruction, Function, Builder, *Compare, *WideType); if (!WideCompare) return std::unexpected(WideCompare.error()); auto WideNewValue = widenAtomicCompareExchangeSource(Instruction, Function, Builder, *NewValue, *WideType); if (!WideNewValue) return std::unexpected(WideNewValue.error()); Register WideDestination; MachineInstr *FoldedExtension = singleRegisterUse(Function, OriginalDestination); if (FoldedExtension != nullptr && FoldedExtension->opcode() == opcode(GenericOpcode::G_ANYEXT) && operandType(Function, *FoldedExtension, 0) .transform([](LLT Type) { return Type.scalarBitWidth() == 64; }) .value_or(false)) { const MachineOperand *WideDestinationOperand = FoldedExtension->operandAt(0); const Register *FoldedDestination = WideDestinationOperand == nullptr ? nullptr : WideDestinationOperand->getIf(); if (FoldedDestination == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V compare-exchange extension has no destination"); WideDestination = *FoldedDestination; } else { auto Created = Function.registerInfo().createVirtualRegister(*WideType); if (!Created) return std::unexpected(Created.error()); WideDestination = *Created; FoldedExtension = nullptr; } if (auto Replaced = Instruction.operandAt(0)->setRegister(WideDestination); !Replaced) return std::unexpected(Replaced.error()); if (auto Replaced = Instruction.operandAt(2)->setRegister(*WideCompare); !Replaced) return std::unexpected(Replaced.error()); if (auto Replaced = Instruction.operandAt(3)->setRegister(*WideNewValue); !Replaced) return std::unexpected(Replaced.error()); if (FoldedExtension != nullptr) return eraseInstruction(*FoldedExtension); auto Point = Builder.setInsertPointAfter(Instruction); if (!Point) return std::unexpected(Point.error()); auto Truncation = Builder.buildInstr(opcode(GenericOpcode::G_TRUNC)); if (!Truncation) return std::unexpected(Truncation.error()); if (auto Added = Truncation->addDef(OriginalDestination); !Added) return std::unexpected(Added.error()); if (auto Added = Truncation->addUse(WideDestination); !Added) return std::unexpected(Added.error()); auto Built = Truncation->build(); if (!Built) return std::unexpected(Built.error()); return {}; } [[nodiscard]] Expected lowerAtomicCompareExchangeWithSuccess(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target) { const MachineOperand *OldValueOperand = Instruction.operandAt(0); const MachineOperand *SuccessOperand = Instruction.operandAt(1); const MachineOperand *AddressOperand = Instruction.operandAt(2); const MachineOperand *CompareOperand = Instruction.operandAt(3); const MachineOperand *NewValueOperand = Instruction.operandAt(4); const Register *OldValue = OldValueOperand == nullptr ? nullptr : OldValueOperand->getIf(); const Register *Success = SuccessOperand == nullptr ? nullptr : SuccessOperand->getIf(); const Register *Address = AddressOperand == nullptr ? nullptr : AddressOperand->getIf(); const Register *Compare = CompareOperand == nullptr ? nullptr : CompareOperand->getIf(); const Register *NewValue = NewValueOperand == nullptr ? nullptr : NewValueOperand->getIf(); const std::optional ValueType = resultType(Function, Instruction); if (OldValue == nullptr || Success == nullptr || Address == nullptr || Compare == nullptr || NewValue == nullptr || !ValueType || Instruction.memOperands().size() != 1) return makeUnexpected(ErrorCode::VerificationError, "RISC-V compare-exchange-with-success has invalid operands"); auto AtomicResult = Function.registerInfo().createVirtualRegister(*ValueType); if (!AtomicResult) return std::unexpected(AtomicResult.error()); MachineIRBuilder Builder(Function, Target); auto Point = Builder.setInsertPointBefore(Instruction); if (!Point) return std::unexpected(Point.error()); auto Atomic = Builder.buildInstr(opcode(GenericOpcode::G_ATOMIC_CMPXCHG)); if (!Atomic) return std::unexpected(Atomic.error()); if (auto Added = Atomic->addDef(*AtomicResult); !Added) return std::unexpected(Added.error()); if (auto Added = Atomic->addUse(*Address); !Added) return std::unexpected(Added.error()); if (auto Added = Atomic->addUse(*Compare); !Added) return std::unexpected(Added.error()); if (auto Added = Atomic->addUse(*NewValue); !Added) return std::unexpected(Added.error()); if (auto Added = Atomic->addMemOperand(*Instruction.memOperands().front()); !Added) return std::unexpected(Added.error()); auto BuiltAtomic = Atomic->build(); if (!BuiltAtomic) return std::unexpected(BuiltAtomic.error()); Point = Builder.setInsertPointBefore(Instruction); if (!Point) return std::unexpected(Point.error()); auto CompareResult = Builder.buildInstr(opcode(GenericOpcode::G_ICMP)); if (!CompareResult) return std::unexpected(CompareResult.error()); if (auto Added = CompareResult->addDef(*Success); !Added) return std::unexpected(Added.error()); if (auto Added = CompareResult->addIntegerPredicate(IntegerPredicate::Equal); !Added) return std::unexpected(Added.error()); if (auto Added = CompareResult->addUse(*AtomicResult); !Added) return std::unexpected(Added.error()); if (auto Added = CompareResult->addUse(*Compare); !Added) return std::unexpected(Added.error()); auto BuiltCompare = CompareResult->build(); if (!BuiltCompare) return std::unexpected(BuiltCompare.error()); while (!Instruction.memOperands().empty()) { auto Erased = Instruction.eraseMemOperand(0); if (!Erased) return std::unexpected(Erased.error()); } return rewriteUnary(Instruction, GenericOpcode::COPY, *OldValue, *AtomicResult); } [[nodiscard]] Expected widenIntegerCompareSource(MachineInstr &Instruction, MachineFunction &Function, MachineIRBuilder &Builder, Register Source, LLT WideType) { const auto SourceInfo = Function.registerInfo().virtualRegisterInfo(Source); if (!SourceInfo || !SourceInfo->get().Type) return makeUnexpected(ErrorCode::VerificationError, "RISC-V integer compare source has no scalar type"); const LLT SourceType = *SourceInfo->get().Type; if (SourceType.scalarBitWidth() == WideType.scalarBitWidth()) return Source; const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(Source); MachineInstr *Def = Definition == nullptr ? nullptr : const_cast(Definition->parent()); std::optional ExistingExtension; bool AmbiguousExtension = false; auto &RegisterInfo = Function.registerInfo(); for (MachineOperand &Use : RegisterInfo.useOperands(Source)) { MachineInstr *User = Use.parent(); if (User == nullptr || User == &Instruction || User->parent() == nullptr || (User->opcode() != opcode(GenericOpcode::G_ANYEXT) && User->opcode() != opcode(GenericOpcode::G_SEXT))) { continue; } const MachineOperand *ExtendedOperand = User->operandAt(0); const Register *Extended = ExtendedOperand == nullptr ? nullptr : ExtendedOperand->getIf(); const std::optional ExtendedType = ExtendedOperand == nullptr ? std::nullopt : registerType(Function, *ExtendedOperand); if (Extended == nullptr || !ExtendedType || *ExtendedType != WideType || (Def == nullptr || (Def->opcode() != opcode(GenericOpcode::G_CONSTANT) && User->opcode() != opcode(GenericOpcode::G_SEXT)))) { continue; } if (ExistingExtension) { AmbiguousExtension = true; break; } ExistingExtension = *Extended; } if (ExistingExtension && !AmbiguousExtension) return *ExistingExtension; if (Def != nullptr && Def->opcode() == opcode(GenericOpcode::G_CONSTANT) && singleRegisterUse(Function, Source) == &Instruction) { auto Widened = widenConstantToRV64(*Def, Function); if (!Widened) return std::unexpected(Widened.error()); return Source; } if (Def != nullptr && Def->opcode() == opcode(GenericOpcode::G_TRUNC) && Def->operandCount() == 2) { const MachineOperand *WideOperand = Def->operandAt(1); const Register *WideSource = WideOperand == nullptr ? nullptr : WideOperand->getIf(); const std::optional ExistingWideType = WideOperand == nullptr ? std::nullopt : registerType(Function, *WideOperand); if (WideSource != nullptr && ExistingWideType && *ExistingWideType == WideType) { auto Result = Function.registerInfo().createVirtualRegister(WideType); if (!Result) return std::unexpected(Result.error()); auto Point = Builder.setInsertPointBefore(Instruction); if (!Point) return std::unexpected(Point.error()); auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_SEXT_INREG)); if (!Pending) return std::unexpected(Pending.error()); if (auto Added = Pending->addDef(*Result); !Added) return std::unexpected(Added.error()); if (auto Added = Pending->addUse(*WideSource); !Added) return std::unexpected(Added.error()); if (auto Added = Pending->addImmediate(static_cast(SourceType.scalarBitWidth())); !Added) { return std::unexpected(Added.error()); } auto Built = Pending->build(); if (!Built) return std::unexpected(Built.error()); return *Result; } } return buildUnaryBefore(Builder, Instruction, GenericOpcode::G_SEXT, WideType, Source); } [[nodiscard]] Expected lowerIntegerCompare(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target) { const MachineOperand *DestinationOperand = Instruction.operandAt(0); const MachineOperand *LeftOperand = Instruction.operandAt(2); const MachineOperand *RightOperand = Instruction.operandAt(3); const Register *Destination = DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf(); const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf(); if (Destination == nullptr || Left == nullptr || Right == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V integer compare has invalid register operands"); const Register OriginalDestination = *Destination; const Register OriginalLeft = *Left; const Register OriginalRight = *Right; auto WideType = LLT::makeAnyScalar(64); if (!WideType) return std::unexpected(WideType.error()); MachineIRBuilder Builder(Function, Target); auto WideLeft = widenIntegerCompareSource(Instruction, Function, Builder, OriginalLeft, *WideType); if (!WideLeft) return std::unexpected(WideLeft.error()); auto WideRight = widenIntegerCompareSource(Instruction, Function, Builder, OriginalRight, *WideType); if (!WideRight) return std::unexpected(WideRight.error()); if (auto Replaced = Instruction.operandAt(2)->setRegister(*WideLeft); !Replaced) return std::unexpected(Replaced.error()); if (auto Replaced = Instruction.operandAt(3)->setRegister(*WideRight); !Replaced) return std::unexpected(Replaced.error()); const auto DestinationInfo = Function.registerInfo().virtualRegisterInfo(OriginalDestination); if (!DestinationInfo || !DestinationInfo->get().Type) return makeUnexpected(ErrorCode::VerificationError, "RISC-V integer compare destination has no scalar type"); if (DestinationInfo->get().Type->scalarBitWidth() == 64) return {}; MachineInstr *FoldedExtension = singleRegisterUse(Function, OriginalDestination); if (FoldedExtension != nullptr && (FoldedExtension->opcode() == opcode(GenericOpcode::G_ANYEXT) || FoldedExtension->opcode() == opcode(GenericOpcode::G_SEXT) || FoldedExtension->opcode() == opcode(GenericOpcode::G_ZEXT)) && operandType(Function, *FoldedExtension, 0) .transform([](LLT Type) { return Type.scalarBitWidth() == 64; }) .value_or(false)) { const MachineOperand *ExtendedOperand = FoldedExtension->operandAt(0); const Register *Extended = ExtendedOperand == nullptr ? nullptr : ExtendedOperand->getIf(); if (Extended == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V integer compare extension has no destination"); const Register WideDestination = *Extended; if (auto Replaced = Instruction.operandAt(0)->setRegister(WideDestination); !Replaced) return std::unexpected(Replaced.error()); return eraseInstruction(*FoldedExtension); } auto WideDestination = Function.registerInfo().createVirtualRegister(*WideType); if (!WideDestination) return std::unexpected(WideDestination.error()); if (auto Replaced = Instruction.operandAt(0)->setRegister(*WideDestination); !Replaced) return std::unexpected(Replaced.error()); if (Function.registerInfo().useEmpty(OriginalDestination)) return {}; auto Point = Builder.setInsertPointAfter(Instruction); if (!Point) return std::unexpected(Point.error()); auto Truncation = Builder.buildInstr(opcode(GenericOpcode::G_TRUNC)); if (!Truncation) return std::unexpected(Truncation.error()); if (auto Added = Truncation->addDef(OriginalDestination); !Added) return std::unexpected(Added.error()); if (auto Added = Truncation->addUse(*WideDestination); !Added) return std::unexpected(Added.error()); auto Built = Truncation->build(); if (!Built) return std::unexpected(Built.error()); return {}; } [[nodiscard]] Expected foldNarrowAnyExtBinary(MachineInstr &Extension, MachineFunction &Function, const TargetDescription &Target) { const auto Plan = narrowAnyExtBinaryPlan(Extension, Function); if (!Plan) return makeUnexpected(ErrorCode::Unsupported, "RISC-V narrow any-extension does not have a legal binary chain"); for (MachineInstr *Constant : Plan->Constants) { auto Widened = widenConstantToRV64(*Constant, Function); if (!Widened) return std::unexpected(Widened.error()); } for (std::size_t Index = 0; Index != Plan->WideInputs.size(); ++Index) { auto Replaced = Plan->Binary->operandAt(Index + 1U)->setRegister(Plan->WideInputs[Index]); if (!Replaced) return std::unexpected(Replaced.error()); } for (MachineInstr *Artifact : Plan->Artifacts) { if (Artifact->parent() == nullptr) continue; auto Removed = Artifact->parent()->removeInstruction(*Artifact); if (!Removed) return std::unexpected(Removed.error()); } if (Plan->Generic == GenericOpcode::G_SHL) { Register Amount = Plan->WideInputs[1]; if (Plan->SourceWidth < 32 || Plan->SourceWidth > 32) { const std::uint64_t Mask = Plan->SourceWidth == 64 ? std::numeric_limits::max() : (std::uint64_t{1} << Plan->SourceWidth) - 1U; auto WideType = LLT::makeAnyScalar(64); if (!WideType) return std::unexpected(WideType.error()); MachineIRBuilder Builder(Function, Target); auto MaskValue = buildIntegerConstantBefore(Builder, Extension, 64, Mask); if (!MaskValue) return std::unexpected(MaskValue.error()); auto MaskedAmount = buildBinaryBefore(Builder, Extension, GenericOpcode::G_AND, *WideType, Amount, *MaskValue); if (!MaskedAmount) return std::unexpected(MaskedAmount.error()); Amount = *MaskedAmount; } auto SetOpcode = Extension.setOpcode( Plan->SourceWidth == 32 ? targetOpcode(InstructionOpcode::G_SLLW) : opcode(GenericOpcode::G_SHL)); if (!SetOpcode) return std::unexpected(SetOpcode.error()); while (Extension.operandCount() > 1) { auto Erased = Extension.eraseOperand(1); if (!Erased) return std::unexpected(Erased.error()); } for (Register Input : {Plan->WideInputs[0], Amount}) { auto Operand = MachineOperand::registerOperand( Input, RegisterOperandFlags{RegisterOperandFlag::Use}); if (!Operand) return std::unexpected(Operand.error()); auto Added = Extension.addOperand(std::move(*Operand)); if (!Added) return std::unexpected(Added.error()); } MachineBasicBlock *BinaryBlock = Plan->Binary->parent(); if (BinaryBlock == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V narrow shift is detached during legalization"); return BinaryBlock->removeInstruction(*Plan->Binary); } if (Plan->SourceWidth == 32 && (Plan->Generic == GenericOpcode::G_ADD || Plan->Generic == GenericOpcode::G_SUB)) { auto WideType = LLT::makeAnyScalar(64); if (!WideType) return std::unexpected(WideType.error()); auto Temporary = Function.registerInfo().createVirtualRegister(*WideType); if (!Temporary) return std::unexpected(Temporary.error()); auto Retargeted = Plan->Binary->operandAt(0)->setRegister(*Temporary); if (!Retargeted) return std::unexpected(Retargeted.error()); auto SetOpcode = Extension.setOpcode(opcode(GenericOpcode::G_SEXT_INREG)); if (!SetOpcode) return std::unexpected(SetOpcode.error()); auto ReplacedSource = Extension.operandAt(1)->setRegister(*Temporary); if (!ReplacedSource) return std::unexpected(ReplacedSource.error()); auto Width = Extension.addOperand(MachineOperand::immediate(32)); if (!Width) return std::unexpected(Width.error()); return {}; } auto SetOpcode = Extension.setOpcode(opcode(Plan->Generic)); if (!SetOpcode) return std::unexpected(SetOpcode.error()); while (Extension.operandCount() > 1) { auto Erased = Extension.eraseOperand(1); if (!Erased) return std::unexpected(Erased.error()); } for (Register Input : Plan->WideInputs) { auto Operand = MachineOperand::registerOperand( Input, RegisterOperandFlags{RegisterOperandFlag::Use}); if (!Operand) return std::unexpected(Operand.error()); auto Added = Extension.addOperand(std::move(*Operand)); if (!Added) return std::unexpected(Added.error()); } MachineBasicBlock *BinaryBlock = Plan->Binary->parent(); if (BinaryBlock == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V narrow binary instruction is detached during legalization"); auto Removed = BinaryBlock->removeInstruction(*Plan->Binary); if (!Removed) return std::unexpected(Removed.error()); (void)Target; return {}; } [[nodiscard]] bool fitsSigned(std::int64_t Value, std::uint32_t Bits) noexcept { const std::int64_t Minimum = -(std::int64_t{1} << (Bits - 1U)); const std::int64_t Maximum = (std::int64_t{1} << (Bits - 1U)) - 1; return Value >= Minimum && Value <= Maximum; } [[nodiscard]] Expected lowerConstantToPool(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target) { const MachineOperand *DestinationOperand = Instruction.operandAt(0); const Register *Destination = DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); const MachineOperand *ImmediateOperand = Instruction.operandAt(1); const auto *Immediate = ImmediateOperand == nullptr ? nullptr : ImmediateOperand->getIf(); if (Destination == nullptr || !Destination->isVirtual() || Immediate == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V constant-pool legalization has invalid operands"); const auto DestinationInfo = Function.registerInfo().virtualRegisterInfo(*Destination); if (!DestinationInfo || !DestinationInfo->get().Type || DestinationInfo->get().Type->scalarBitWidth() != 64) return makeUnexpected(ErrorCode::VerificationError, "RISC-V constant-pool legalization requires a 64-bit destination"); const LLT DestinationType = *DestinationInfo->get().Type; auto Value = GenericConstantData::integer(Immediate->Value); if (!Value) return std::unexpected(Value.error()); auto Align = Alignment::create(8); if (!Align) return std::unexpected(Align.error()); MachineConstantPoolEntryDesc Description; Description.Value = std::move(*Value); Description.Size = 8; Description.Align = *Align; auto PoolIndex = Function.constantPool().addEntry(std::move(Description)); if (!PoolIndex) return std::unexpected(PoolIndex.error()); auto PointerType = LLT::makePointer(0, 64); if (!PointerType) return std::unexpected(PointerType.error()); auto Address = Function.registerInfo().createVirtualRegister(*PointerType); if (!Address) return std::unexpected(Address.error()); MachineIRBuilder Builder(Function, Target); auto Point = Builder.setInsertPointBefore(Instruction); if (!Point) return std::unexpected(Point.error()); auto AddressInstruction = Builder.buildInstr(opcode(GenericOpcode::G_CONSTANT_POOL)); if (!AddressInstruction) return std::unexpected(AddressInstruction.error()); auto AddressDef = AddressInstruction->addDef(*Address); if (!AddressDef) return std::unexpected(AddressDef.error()); auto AddressPool = AddressInstruction->addConstantPool(*PoolIndex); if (!AddressPool) return std::unexpected(AddressPool.error()); auto BuiltAddress = AddressInstruction->build(); if (!BuiltAddress) return std::unexpected(BuiltAddress.error()); MachineMemOperandDesc MemoryDescription; MemoryDescription.Accesses = {MemoryAccessFlag::Load}; MemoryDescription.Size = *MemorySize::fixed(8); MemoryDescription.MemoryType = DestinationType; MemoryDescription.Align = *Align; MemoryDescription.Base = MemoryBase::constantPoolPseudo(); auto Memory = Function.createMemOperand(std::move(MemoryDescription)); if (!Memory) return std::unexpected(Memory.error()); auto SetOpcode = Instruction.setOpcode(opcode(GenericOpcode::G_LOAD)); if (!SetOpcode) return std::unexpected(SetOpcode.error()); while (Instruction.operandCount() > 1) { auto Erased = Instruction.eraseOperand(1); if (!Erased) return std::unexpected(Erased.error()); } auto AddressUse = MachineOperand::registerOperand( *Address, RegisterOperandFlags{RegisterOperandFlag::Use}); if (!AddressUse) return std::unexpected(AddressUse.error()); auto AddedAddress = Instruction.addOperand(std::move(*AddressUse)); if (!AddedAddress) return std::unexpected(AddedAddress.error()); auto AddedMemory = Instruction.addMemOperand(**Memory); if (!AddedMemory) return std::unexpected(AddedMemory.error()); return {}; } [[nodiscard]] bool isSupportedNonAtomicMemoryAccess(const MachineInstr &Instruction) noexcept { if (Instruction.memOperands().size() != 1) return false; const MachineMemOperand &Memory = *Instruction.memOperands().front(); const MachineMemOperandDesc &Description = Memory.description(); return !Memory.isAtomic() && Description.AddressSpace == 0 && Description.Size.isKnown() && !Description.Size.isScalable() && Description.Size.minimumBytes() != 0; } [[nodiscard]] bool isSupportedAtomicMemoryAccess(const MachineInstr &Instruction, AtomicOperation Operation, std::uint64_t Bytes) noexcept { if (Instruction.memOperands().size() != 1 || (Bytes != 4 && Bytes != 8)) return false; const MachineMemOperand &Memory = *Instruction.memOperands().front(); const MachineMemOperandDesc &Description = Memory.description(); return Description.AtomicKind == Operation && Description.AddressSpace == 0 && Description.Size.isKnown() && !Description.Size.isScalable() && Description.Size.minimumBytes() == Bytes && Description.Align.bytes() >= Bytes && Description.SyncScope.value() == 0; } [[nodiscard]] bool isSupportedAtomicContainerAccess(const MachineInstr &Instruction, AtomicOperation Operation, std::uint64_t ContainerBytes) noexcept { if (Instruction.memOperands().size() != 1 || ContainerBytes == 0) return false; const MachineMemOperandDesc &Description = Instruction.memOperands().front()->description(); if (Description.AtomicKind != Operation || Description.AddressSpace != 0 || !Description.Size.isKnown() || Description.Size.isScalable() || Description.SyncScope.value() != 0) return false; const std::uint64_t MemoryBytes = Description.Size.minimumBytes(); return (MemoryBytes == 1 || MemoryBytes == 2 || MemoryBytes == 4 || MemoryBytes == 8) && MemoryBytes <= ContainerBytes && Description.Align.bytes() >= MemoryBytes; } [[nodiscard]] std::optional memoryAccessBytes(const MachineInstr &Instruction) noexcept { if (Instruction.memOperands().size() != 1) return std::nullopt; const MachineMemOperandDesc &Description = Instruction.memOperands().front()->description(); if (!Description.Size.isKnown() || Description.Size.isScalable()) return std::nullopt; return Description.Size.minimumBytes(); } [[nodiscard]] bool memoryMatchesType(const MachineInstr &Instruction, LLT Type) noexcept { const auto Bytes = memoryAccessBytes(Instruction); if (!Bytes || (!isSupportedIntegerType(Type) && !isRVFloatType(Type))) return false; if (isRVFloatType(Type)) return Type.scalarBitWidth() == 32 ? *Bytes == 4 : *Bytes == 8; const std::uint64_t ValueBytes = (Type.scalarBitWidth() + 7U) / 8U; return (*Bytes == 1 || *Bytes == 2 || *Bytes == 4 || *Bytes == 8) && *Bytes <= ValueBytes; } [[nodiscard]] Expected requiredRegisterClass(const MachineFunction &Function, Register Reg, const TargetDescription &Target) { if (!Reg.isVirtual()) return makeUnexpected(ErrorCode::InvalidArgument, "RISC-V register-class selection requires a virtual register"); auto Info = Function.registerInfo().virtualRegisterInfo(Reg); if (!Info) return std::unexpected(Info.error()); if (!Info->get().Type) { return makeUnexpected(ErrorCode::VerificationError, "RISC-V selection requires a type for every virtual register"); } const VirtualRegisterInfo &RegisterInfo = Info->get(); const LLT Type = RegisterInfo.Type.value_or(LLT{}); RegisterClassID RequiredClass; if (isScalarGPRType(Type)) { RequiredClass = RegisterClassID::fromValue(static_cast(RegisterClass::GPR)); } else if (Type.isFloat() && Type.scalarBitWidth() == 32) { RequiredClass = RegisterClassID::fromValue(static_cast(RegisterClass::FPR32)); } else if (Type.isFloat() && Type.scalarBitWidth() == 64) { RequiredClass = RegisterClassID::fromValue(static_cast(RegisterClass::FPR64)); } else { return makeUnexpected(ErrorCode::Unsupported, "RISC-V RV64GC has no register class for type '" + Type.toString() + "'"); } if (Target.registers().registerClass(RequiredClass) == nullptr) { return makeUnexpected(ErrorCode::InternalError, "RISC-V target description is missing a required register class"); } if (RegisterInfo.Bank && !Target.registers().bankCoversClass(*RegisterInfo.Bank, RequiredClass)) { return makeUnexpected(ErrorCode::VerificationError, "RISC-V virtual register bank cannot cover its required class"); } if (RegisterInfo.RegClass && !Target.registers().isSubclass(*RegisterInfo.RegClass, RequiredClass)) { return makeUnexpected(ErrorCode::VerificationError, "RISC-V virtual register has a conflicting register class"); } return RequiredClass; } [[nodiscard]] Expected appendImmediate(MachineInstr &Instruction, std::int64_t Value) { return Instruction.addOperand(MachineOperand::immediate(Value)); } [[nodiscard]] Expected insertPhysicalRegister(MachineInstr &Instruction, std::size_t Index, PhysicalRegister Reg, RegisterOperandFlag Flag) { auto Operand = MachineOperand::registerOperand(Register::fromPhysical(Reg), {Flag}); if (!Operand) return std::unexpected(Operand.error()); return Instruction.insertOperand(Index, std::move(*Operand)); } [[nodiscard]] Expected insertVirtualUse(MachineInstr &Instruction, std::size_t Index, Register Reg) { auto Operand = MachineOperand::registerOperand(Reg, {RegisterOperandFlag::Use}); if (!Operand) return std::unexpected(Operand.error()); return Instruction.insertOperand(Index, std::move(*Operand)); } [[nodiscard]] Expected createTemporary(MachineFunction &Function, LLT Type, const TargetDescription &Target) { auto Temporary = Function.registerInfo().createVirtualRegister(Type); if (!Temporary) return std::unexpected(Temporary.error()); auto RequiredClass = requiredRegisterClass(Function, *Temporary, Target); if (!RequiredClass) return std::unexpected(RequiredClass.error()); const RegisterBank BankKind = Type.isFloat() ? RegisterBank::FPR : RegisterBank::GPR; auto Bank = Function.registerInfo().setRegisterBank( *Temporary, RegisterBankID::fromValue(static_cast(BankKind))); if (!Bank) return std::unexpected(Bank.error()); auto Class = Function.registerInfo().setRegisterClass(*Temporary, *RequiredClass); if (!Class) return std::unexpected(Class.error()); return *Temporary; } [[nodiscard]] Expected signedImmediate(const MachineOperand &Operand) { if (const std::int64_t *Value = Operand.getIf()) return *Value; const auto *Integer = Operand.getIf(); if (Integer == nullptr || Integer->Value.bitWidth() == 0 || Integer->Value.bitWidth() > 64) return makeUnexpected(ErrorCode::Unsupported, "RISC-V constant selection requires an immediate no wider than 64 bits"); std::uint64_t Raw = Integer->Value.words().empty() ? 0 : Integer->Value.words().front(); const std::uint32_t Width = Integer->Value.bitWidth(); if (Width < 64 && (Raw & (std::uint64_t{1} << (Width - 1U))) != 0) Raw |= ~std::uint64_t{0} << Width; return std::bit_cast(Raw); } [[nodiscard]] Expected replaceRegisterUses(MachineFunction &Function, Register OldReg, Register NewReg) { std::vector Uses; for (MachineOperand &Operand : Function.registerInfo().useOperands(OldReg)) Uses.push_back(&Operand); for (MachineOperand *Operand : Uses) { auto Replaced = Operand->setRegister(NewReg); if (!Replaced) return std::unexpected(Replaced.error()); } return {}; } [[nodiscard]] Expected eraseInstruction(MachineInstr &Instruction) { MachineBasicBlock *Block = Instruction.parent(); if (Block == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V legalizer cannot erase a detached instruction"); return Block->removeInstruction(Instruction); } [[nodiscard]] bool allDefsUnused(const MachineInstr &Instruction, const MachineFunction &Function) { for (const MachineOperand &Operand : Instruction.operands()) { const Register *Reg = Operand.getIf(); if (Reg != nullptr && Operand.isDef() && Reg->isVirtual() && !Function.registerInfo().useEmpty(*Reg)) return false; } return true; } void eraseDeadArtifacts(std::span Candidates, const MachineFunction &Function) { bool Changed = false; do { Changed = false; for (auto It = Candidates.rbegin(); It != Candidates.rend(); ++It) { MachineInstr *Instruction = *It; if (Instruction == nullptr || Instruction->parent() == nullptr || !allDefsUnused(*Instruction, Function)) continue; const Opcode Value = Instruction->opcode(); if (Value != opcode(GenericOpcode::G_TRUNC) && Value != opcode(GenericOpcode::G_ANYEXT) && Value != opcode(GenericOpcode::G_SEXT) && Value != opcode(GenericOpcode::G_ZEXT) && Value != opcode(GenericOpcode::G_MERGE_VALUES) && Value != opcode(GenericOpcode::G_UNMERGE_VALUES) && Value != opcode(GenericOpcode::COPY)) continue; if (eraseInstruction(*Instruction)) Changed = true; } } while (Changed); } [[nodiscard]] Expected buildIntegerConstantBefore(MachineIRBuilder &Builder, MachineInstr &Anchor, std::uint32_t Width, std::uint64_t Value) { auto Type = LLT::makeAnyScalar(Width); if (!Type) return std::unexpected(Type.error()); auto Result = Builder.function().registerInfo().createVirtualRegister(*Type); if (!Result) return std::unexpected(Result.error()); auto Immediate = IntegerImmediate::fromUnsigned(Width, Value); if (!Immediate) return std::unexpected(Immediate.error()); auto Point = Builder.setInsertPointBefore(Anchor); if (!Point) return std::unexpected(Point.error()); auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_CONSTANT)); if (!Pending) return std::unexpected(Pending.error()); if (auto Added = Pending->addDef(*Result); !Added) return std::unexpected(Added.error()); if (auto Added = Pending->addIntegerImmediate(std::move(*Immediate)); !Added) return std::unexpected(Added.error()); auto Built = Pending->build(); if (!Built) return std::unexpected(Built.error()); return *Result; } [[nodiscard]] Expected buildUnaryBefore(MachineIRBuilder &Builder, MachineInstr &Anchor, GenericOpcode Generic, LLT Type, Register Source) { auto Result = Builder.function().registerInfo().createVirtualRegister(Type); if (!Result) return std::unexpected(Result.error()); auto Point = Builder.setInsertPointBefore(Anchor); if (!Point) return std::unexpected(Point.error()); auto Pending = Builder.buildInstr(opcode(Generic)); if (!Pending) return std::unexpected(Pending.error()); if (auto Added = Pending->addDef(*Result); !Added) return std::unexpected(Added.error()); if (auto Added = Pending->addUse(Source); !Added) return std::unexpected(Added.error()); auto Built = Pending->build(); if (!Built) return std::unexpected(Built.error()); return *Result; } [[nodiscard]] Expected buildBinaryBefore(MachineIRBuilder &Builder, MachineInstr &Anchor, GenericOpcode Generic, LLT Type, Register Left, Register Right) { auto Result = Builder.function().registerInfo().createVirtualRegister(Type); if (!Result) return std::unexpected(Result.error()); auto Point = Builder.setInsertPointBefore(Anchor); if (!Point) return std::unexpected(Point.error()); auto Pending = Builder.buildInstr(opcode(Generic)); if (!Pending) return std::unexpected(Pending.error()); if (auto Added = Pending->addDef(*Result); !Added) return std::unexpected(Added.error()); if (auto Added = Pending->addUse(Left); !Added) return std::unexpected(Added.error()); if (auto Added = Pending->addUse(Right); !Added) return std::unexpected(Added.error()); auto Built = Pending->build(); if (!Built) return std::unexpected(Built.error()); return *Result; } [[nodiscard]] Expected rewriteUnary(MachineInstr &Instruction, GenericOpcode Generic, Register Destination, Register Source) { auto SetOpcode = Instruction.setOpcode(opcode(Generic)); if (!SetOpcode) return std::unexpected(SetOpcode.error()); while (Instruction.operandCount() != 0) { auto Erased = Instruction.eraseOperand(Instruction.operandCount() - 1U); if (!Erased) return std::unexpected(Erased.error()); } auto Def = MachineOperand::registerOperand( Destination, RegisterOperandFlags{RegisterOperandFlag::Define}); if (!Def) return std::unexpected(Def.error()); if (auto Added = Instruction.addOperand(std::move(*Def)); !Added) return std::unexpected(Added.error()); auto Use = MachineOperand::registerOperand(Source, RegisterOperandFlags{RegisterOperandFlag::Use}); if (!Use) return std::unexpected(Use.error()); return Instruction.addOperand(std::move(*Use)); } [[nodiscard]] Expected widenNarrowMultiply(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target) { const std::optional Type = resultType(Function, Instruction); const MachineOperand *DestinationOperand = Instruction.operandAt(0); const MachineOperand *LeftOperand = Instruction.operandAt(1); const MachineOperand *RightOperand = Instruction.operandAt(2); const Register *Destination = DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf(); const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf(); if (!Type || Destination == nullptr || Left == nullptr || Right == nullptr || Type->scalarBitWidth() >= 64) { return makeUnexpected(ErrorCode::VerificationError, "RISC-V narrow multiply has invalid operands"); } MachineInstr *Extension = singleRegisterUse(Function, *Destination); if (Extension != nullptr && Extension->opcode() == opcode(GenericOpcode::G_ANYEXT)) { const auto Plan = narrowAnyExtBinaryPlan(*Extension, Function); if (Plan && Plan->Binary == &Instruction) return foldNarrowAnyExtBinary(*Extension, Function, Target); } auto WideType = LLT::makeAnyScalar(64); if (!WideType) return std::unexpected(WideType.error()); MachineIRBuilder Builder(Function, Target); auto WideLeft = buildUnaryBefore(Builder, Instruction, GenericOpcode::G_ANYEXT, *WideType, *Left); if (!WideLeft) return std::unexpected(WideLeft.error()); auto WideRight = buildUnaryBefore(Builder, Instruction, GenericOpcode::G_ANYEXT, *WideType, *Right); if (!WideRight) return std::unexpected(WideRight.error()); auto Product = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_MUL, *WideType, *WideLeft, *WideRight); if (!Product) return std::unexpected(Product.error()); return rewriteUnary(Instruction, GenericOpcode::G_TRUNC, *Destination, *Product); } [[nodiscard]] Expected lowerMultiplyLibcall(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target) { const std::optional Type = resultType(Function, Instruction); const MachineOperand *DestinationOperand = Instruction.operandAt(0); const MachineOperand *LeftOperand = Instruction.operandAt(1); const MachineOperand *RightOperand = Instruction.operandAt(2); const Register *Destination = DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf(); const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf(); if (!Type || Destination == nullptr || Left == nullptr || Right == nullptr || Type->scalarBitWidth() != 64) { return makeUnexpected(ErrorCode::VerificationError, "RISC-V multiply libcall requires three s64 registers"); } auto ABIAlignment = Alignment::create(8); auto Callee = MachineCallTarget::externalSymbol("__muldi3"); if (!ABIAlignment || !Callee) return makeUnexpected(ErrorCode::InternalError, "RISC-V multiply libcall ABI facts are unavailable"); const auto Value = [Type = *Type, Alignment = *ABIAlignment](Register Reg, std::uint32_t Number) { return ABIValue{ .Value = Reg, .Type = Type, .OriginalValue = Number, .FragmentIndex = 0, .FragmentCount = 1, .FragmentOffsetBytes = 0, .OriginalAlignment = Alignment, .Extension = ABIExtensionKind::None, .Flags = {}, .ByValSizeBytes = 0, .ByValAlignment = Alignment, }; }; CallLoweringRequest Request; Request.Convention = callingConventionID(CallingConvention::LP64D); Request.Callee = std::move(*Callee); Request.Arguments = {Value(*Left, 0), Value(*Right, 1)}; Request.Results = {Value(*Destination, 0)}; Request.FixedArgumentCount = Request.Arguments.size(); MachineIRBuilder Builder(Function, Target); auto Point = Builder.setInsertPointBefore(Instruction); if (!Point) return std::unexpected(Point.error()); auto Lowered = callLowering().lowerCall(Builder, Request); if (!Lowered) return std::unexpected(Lowered.error()); return eraseInstruction(Instruction); } [[nodiscard]] Expected rewriteBinary(MachineInstr &Instruction, GenericOpcode Generic, Register Destination, Register Left, Register Right) { auto Rewritten = rewriteUnary(Instruction, Generic, Destination, Left); if (!Rewritten) return std::unexpected(Rewritten.error()); auto Use = MachineOperand::registerOperand(Right, RegisterOperandFlags{RegisterOperandFlag::Use}); if (!Use) return std::unexpected(Use.error()); return Instruction.addOperand(std::move(*Use)); } [[nodiscard]] bool isKnownZeroExtended(const MachineFunction &Function, Register Reg, std::uint32_t ActiveBits, unsigned Depth = 0) { if (Depth > 8 || !Reg.isVirtual()) return false; const auto Info = Function.registerInfo().virtualRegisterInfo(Reg); if (!Info || !Info->get().Type) return false; const std::uint32_t Width = Info->get().Type->scalarBitWidth(); if (Width <= ActiveBits) return true; const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(Reg); const MachineInstr *Def = Definition == nullptr ? nullptr : Definition->parent(); if (Def == nullptr) return false; if (Def->opcode() == opcode(GenericOpcode::G_ASSERT_ZEXT)) { const MachineOperand *BitsOperand = Def->operandAt(2); auto Bits = BitsOperand == nullptr ? Expected(std::unexpected( Error(ErrorCode::InvalidArgument, "missing asserted extension width"))) : signedImmediate(*BitsOperand); return Bits && *Bits >= 0 && static_cast(*Bits) <= ActiveBits; } if (Def->opcode() == opcode(GenericOpcode::G_ZEXT)) { const MachineOperand *SourceOperand = Def->operandAt(1); const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); if (Source == nullptr) return false; const auto SourceInfo = Function.registerInfo().virtualRegisterInfo(*Source); return SourceInfo && SourceInfo->get().Type && SourceInfo->get().Type->scalarBitWidth() <= ActiveBits; } if (Def->opcode() == opcode(GenericOpcode::G_TRUNC)) { const MachineOperand *SourceOperand = Def->operandAt(1); const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); return Source != nullptr && isKnownZeroExtended(Function, *Source, ActiveBits, Depth + 1U); } return false; } [[nodiscard]] Expected zeroExtendToRV64(MachineIRBuilder &Builder, MachineInstr &Anchor, MachineFunction &Function, Register Source, std::vector &DeadCandidates) { const auto Info = Function.registerInfo().virtualRegisterInfo(Source); if (!Info || !Info->get().Type) return makeUnexpected(ErrorCode::VerificationError, "RISC-V merge source has no scalar type"); const LLT SourceType = *Info->get().Type; if (SourceType.scalarBitWidth() == 64) return Source; const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(Source); MachineInstr *Def = Definition == nullptr ? nullptr : const_cast(Definition->parent()); if (Def != nullptr && Def->opcode() == opcode(GenericOpcode::G_TRUNC)) { const MachineOperand *WideOperand = Def->operandAt(1); const Register *Wide = WideOperand == nullptr ? nullptr : WideOperand->getIf(); if (Wide != nullptr) { const auto WideInfo = Function.registerInfo().virtualRegisterInfo(*Wide); if (WideInfo && WideInfo->get().Type && WideInfo->get().Type->scalarBitWidth() == 64 && isKnownZeroExtended(Function, *Wide, SourceType.scalarBitWidth())) { DeadCandidates.push_back(Def); return *Wide; } } } auto WideType = LLT::makeAnyScalar(64); if (!WideType) return std::unexpected(WideType.error()); return buildUnaryBefore(Builder, Anchor, GenericOpcode::G_ZEXT, *WideType, Source); } [[nodiscard]] MachineInstr *singleUseInstruction(const MachineFunction &Function, Register Reg) { MachineInstr *Result = nullptr; auto &RegisterInfo = const_cast(Function.registerInfo()); for (MachineOperand &Operand : RegisterInfo.useOperands(Reg)) { MachineInstr *Instruction = Operand.parent(); if (Instruction == nullptr || Instruction->parent() == nullptr) continue; if (Result != nullptr && Result != Instruction) return nullptr; Result = Instruction; } return Result; } [[nodiscard]] Expected lowerWideMergeValues(MachineInstr &Instruction, MachineFunction &Function) { const MachineOperand *DestinationOperand = Instruction.operandAt(0); const Register *Destination = DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); const MachineOperand *LowOperand = Instruction.operandAt(1); const Register *Low = LowOperand == nullptr ? nullptr : LowOperand->getIf(); if (Destination == nullptr || Low == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide merge has invalid register operands"); std::vector Truncations; auto &RegisterInfo = Function.registerInfo(); std::vector Uses; for (MachineOperand &Use : RegisterInfo.useOperands(*Destination)) Uses.push_back(&Use); for (MachineOperand *Use : Uses) { MachineInstr *Trunc = Use->parent(); if (Trunc == nullptr || Trunc->parent() == nullptr || Trunc->opcode() != opcode(GenericOpcode::G_TRUNC) || Trunc->operandAt(1) != Use) return makeUnexpected(ErrorCode::Unsupported, "RISC-V wide merge is only representable through low-part truncation"); const MachineOperand *TruncResultOperand = Trunc->operandAt(0); const Register *TruncResult = TruncResultOperand == nullptr ? nullptr : TruncResultOperand->getIf(); const auto LowInfo = RegisterInfo.virtualRegisterInfo(*Low); const auto ResultInfo = TruncResult == nullptr ? Expected>( std::unexpected(Error(ErrorCode::InvalidArgument, "missing truncation result"))) : RegisterInfo.virtualRegisterInfo(*TruncResult); if (!LowInfo || !LowInfo->get().Type || !ResultInfo || !ResultInfo->get().Type || ResultInfo->get().Type->scalarBitWidth() > LowInfo->get().Type->scalarBitWidth()) return makeUnexpected(ErrorCode::Unsupported, "RISC-V wide merge truncation spans more than its low source"); if (*ResultInfo->get().Type == *LowInfo->get().Type) { auto Replaced = replaceRegisterUses(Function, *TruncResult, *Low); if (!Replaced) return std::unexpected(Replaced.error()); Truncations.push_back(Trunc); } else { auto Replaced = Use->setRegister(*Low); if (!Replaced) return std::unexpected(Replaced.error()); } } for (MachineInstr *Trunc : Truncations) { auto Erased = eraseInstruction(*Trunc); if (!Erased) return std::unexpected(Erased.error()); } return eraseInstruction(Instruction); } [[nodiscard]] Expected lowerMergeValues(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target) { const std::optional DestinationType = resultType(Function, Instruction); if (!DestinationType || DestinationType->scalarBitWidth() == 0) return makeUnexpected(ErrorCode::VerificationError, "RISC-V merge destination has no scalar type"); if (DestinationType->scalarBitWidth() > 64) return lowerWideMergeValues(Instruction, Function); const MachineOperand *DestinationOperand = Instruction.operandAt(0); const Register *Destination = DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); if (Destination == nullptr || Instruction.operandCount() < 3) return makeUnexpected(ErrorCode::VerificationError, "RISC-V merge has invalid register operands"); Register Result = *Destination; MachineInstr *FoldedExtension = nullptr; if (DestinationType->scalarBitWidth() < 64) { MachineInstr *Use = singleUseInstruction(Function, *Destination); if (Use != nullptr && Use->opcode() == opcode(GenericOpcode::G_ZEXT) && operandType(Function, *Use, 0).transform([](LLT Type) { return Type.scalarBitWidth() == 64; }).value_or(false)) { const MachineOperand *ExtendedOperand = Use->operandAt(0); const Register *Extended = ExtendedOperand == nullptr ? nullptr : ExtendedOperand->getIf(); if (Extended != nullptr) { Result = *Extended; FoldedExtension = Use; } } } auto WideType = LLT::makeAnyScalar(64); if (!WideType) return std::unexpected(WideType.error()); MachineIRBuilder Builder(Function, Target); std::vector DeadCandidates; std::vector Terms; Terms.reserve(Instruction.operandCount() - 1U); for (std::size_t Index = 1; Index < Instruction.operandCount(); ++Index) { const MachineOperand *Operand = Instruction.operandAt(Index); const Register *Source = Operand == nullptr ? nullptr : Operand->getIf(); if (Source == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V merge source is not a register"); auto WideSource = zeroExtendToRV64(Builder, Instruction, Function, *Source, DeadCandidates); if (!WideSource) return std::unexpected(WideSource.error()); if (Index == 1) { Terms.push_back(*WideSource); continue; } const std::optional SourceType = registerType(Function, *Operand); if (!SourceType) return makeUnexpected(ErrorCode::VerificationError, "RISC-V merge source has no scalar type"); const std::uint64_t Shift = static_cast(SourceType->scalarBitWidth()) * (Index - 1U); auto Amount = buildIntegerConstantBefore(Builder, Instruction, 64, Shift); if (!Amount) return std::unexpected(Amount.error()); auto Shifted = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_SHL, *WideType, *WideSource, *Amount); if (!Shifted) return std::unexpected(Shifted.error()); Terms.push_back(*Shifted); } Register Accumulator = Terms.front(); for (std::size_t Index = 1; Index + 1U < Terms.size(); ++Index) { auto Combined = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_OR, *WideType, Accumulator, Terms[Index]); if (!Combined) return std::unexpected(Combined.error()); Accumulator = *Combined; } if (DestinationType->scalarBitWidth() == 64 || FoldedExtension != nullptr) { auto Rewritten = rewriteBinary(Instruction, GenericOpcode::G_OR, Result, Accumulator, Terms.back()); if (!Rewritten) return std::unexpected(Rewritten.error()); } else { auto Combined = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_OR, *WideType, Accumulator, Terms.back()); if (!Combined) return std::unexpected(Combined.error()); auto Rewritten = rewriteUnary(Instruction, GenericOpcode::G_TRUNC, *Destination, *Combined); if (!Rewritten) return std::unexpected(Rewritten.error()); } if (FoldedExtension != nullptr) { auto Erased = eraseInstruction(*FoldedExtension); if (!Erased) return std::unexpected(Erased.error()); } eraseDeadArtifacts(DeadCandidates, Function); return {}; } [[nodiscard]] Expected foldWideExtendedUnmerge(MachineInstr &Instruction, MachineFunction &Function) { const std::size_t OutputCount = Instruction.operandCount() - 1U; const MachineOperand *SourceOperand = Instruction.operandAt(OutputCount); const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); const MachineOperand *Definition = Source == nullptr ? nullptr : Function.registerInfo().getUniqueVRegDef(*Source); MachineInstr *Extension = Definition == nullptr ? nullptr : const_cast(Definition->parent()); if (Extension == nullptr || (Extension->opcode() != opcode(GenericOpcode::G_ZEXT) && Extension->opcode() != opcode(GenericOpcode::G_ANYEXT) && Extension->opcode() != opcode(GenericOpcode::G_SEXT))) return makeUnexpected(ErrorCode::Unsupported, "RISC-V wide unmerge requires an extension-defined source"); const MachineOperand *NarrowOperand = Extension->operandAt(1); const Register *Narrow = NarrowOperand == nullptr ? nullptr : NarrowOperand->getIf(); if (Narrow == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide unmerge extension has no register source"); const auto NarrowInfo = Function.registerInfo().virtualRegisterInfo(*Narrow); if (!NarrowInfo || !NarrowInfo->get().Type) return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide unmerge extension source has no type"); const std::uint32_t NarrowWidth = NarrowInfo->get().Type->scalarBitWidth(); std::vector DeadUsers; for (std::size_t Index = 0; Index < OutputCount; ++Index) { const MachineOperand *OutputOperand = Instruction.operandAt(Index); const Register *Output = OutputOperand == nullptr ? nullptr : OutputOperand->getIf(); if (Output == nullptr || Function.registerInfo().useEmpty(*Output)) continue; const auto OutputInfo = Function.registerInfo().virtualRegisterInfo(*Output); if (!OutputInfo || !OutputInfo->get().Type) return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide unmerge output has no type"); const std::uint64_t Offset = static_cast(Index) * OutputInfo->get().Type->scalarBitWidth(); if (Offset != 0) return makeUnexpected(ErrorCode::Unsupported, "RISC-V wide unmerge cannot materialize a used high wide part"); if (OutputInfo->get().Type->scalarBitWidth() == NarrowWidth) { auto Replaced = replaceRegisterUses(Function, *Output, *Narrow); if (!Replaced) return std::unexpected(Replaced.error()); continue; } if (OutputInfo->get().Type->scalarBitWidth() < NarrowWidth) return makeUnexpected(ErrorCode::Unsupported, "RISC-V wide unmerge output unexpectedly narrows its source"); std::vector Uses; for (MachineOperand &Use : Function.registerInfo().useOperands(*Output)) Uses.push_back(&Use); for (MachineOperand *Use : Uses) { MachineInstr *Trunc = Use->parent(); if (Trunc == nullptr || Trunc->parent() == nullptr || Trunc->opcode() != opcode(GenericOpcode::G_TRUNC) || Trunc->operandAt(1) != Use) return makeUnexpected(ErrorCode::Unsupported, "RISC-V wide unmerge result is only representable through truncation"); const MachineOperand *ResultOperand = Trunc->operandAt(0); const Register *Result = ResultOperand == nullptr ? nullptr : ResultOperand->getIf(); const auto ResultInfo = Result == nullptr ? Expected>( std::unexpected(Error(ErrorCode::InvalidArgument, "missing truncation result"))) : Function.registerInfo().virtualRegisterInfo(*Result); if (!ResultInfo || !ResultInfo->get().Type || ResultInfo->get().Type->scalarBitWidth() > NarrowWidth) return makeUnexpected(ErrorCode::Unsupported, "RISC-V wide unmerge truncation exceeds its source width"); if (ResultInfo->get().Type->scalarBitWidth() == NarrowWidth) { auto Replaced = replaceRegisterUses(Function, *Result, *Narrow); if (!Replaced) return std::unexpected(Replaced.error()); DeadUsers.push_back(Trunc); } else { auto Replaced = Use->setRegister(*Narrow); if (!Replaced) return std::unexpected(Replaced.error()); } } } for (MachineInstr *User : DeadUsers) { auto Erased = eraseInstruction(*User); if (!Erased) return std::unexpected(Erased.error()); } auto Erased = eraseInstruction(Instruction); if (!Erased) return std::unexpected(Erased.error()); if (Function.registerInfo().useEmpty(*Source)) return eraseInstruction(*Extension); return {}; } [[nodiscard]] Expected lowerUnmergeValues(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target) { if (Instruction.operandCount() < 3) return makeUnexpected(ErrorCode::VerificationError, "RISC-V unmerge has invalid register operands"); const std::size_t OutputCount = Instruction.operandCount() - 1U; const std::optional OutputType = operandType(Function, Instruction, 0); const std::optional SourceType = operandType(Function, Instruction, OutputCount); if (!OutputType || !SourceType) return makeUnexpected(ErrorCode::VerificationError, "RISC-V unmerge operands have no scalar types"); if (SourceType->scalarBitWidth() > 64 || OutputType->scalarBitWidth() > 64) return foldWideExtendedUnmerge(Instruction, Function); const MachineOperand *SourceOperand = Instruction.operandAt(OutputCount); const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); if (Source == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V unmerge source is not a register"); auto WideType = LLT::makeAnyScalar(64); if (!WideType) return std::unexpected(WideType.error()); MachineIRBuilder Builder(Function, Target); std::vector FoldedExtensions; for (std::size_t Index = 0; Index < OutputCount; ++Index) { const MachineOperand *OutputOperand = Instruction.operandAt(Index); const Register *Output = OutputOperand == nullptr ? nullptr : OutputOperand->getIf(); if (Output == nullptr || Function.registerInfo().useEmpty(*Output)) continue; Register Extracted = *Source; const std::uint64_t Offset = static_cast(Index) * OutputType->scalarBitWidth(); if (Offset != 0) { auto Amount = buildIntegerConstantBefore(Builder, Instruction, 64, Offset); if (!Amount) return std::unexpected(Amount.error()); auto Shifted = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_LSHR, *WideType, *Source, *Amount); if (!Shifted) return std::unexpected(Shifted.error()); Extracted = *Shifted; } MachineInstr *Extension = singleUseInstruction(Function, *Output); const bool FoldZExt = Extension != nullptr && Extension->opcode() == opcode(GenericOpcode::G_ZEXT) && operandType(Function, *Extension, 0) .transform([](LLT Type) { return Type.scalarBitWidth() == 64; }) .value_or(false); if (FoldZExt) { const MachineOperand *ExtendedOperand = Extension->operandAt(0); const Register *Extended = ExtendedOperand == nullptr ? nullptr : ExtendedOperand->getIf(); if (Extended == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V unmerge extension has no destination"); const std::uint64_t Mask = OutputType->scalarBitWidth() == 64 ? std::numeric_limits::max() : (std::uint64_t{1} << OutputType->scalarBitWidth()) - 1U; auto MaskReg = buildIntegerConstantBefore(Builder, Instruction, 64, Mask); if (!MaskReg) return std::unexpected(MaskReg.error()); auto Masked = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_AND, *WideType, Extracted, *MaskReg); if (!Masked) return std::unexpected(Masked.error()); auto Replaced = replaceRegisterUses(Function, *Extended, *Masked); if (!Replaced) return std::unexpected(Replaced.error()); FoldedExtensions.push_back(Extension); continue; } auto Truncated = buildUnaryBefore(Builder, Instruction, GenericOpcode::G_TRUNC, *OutputType, Extracted); if (!Truncated) return std::unexpected(Truncated.error()); auto Replaced = replaceRegisterUses(Function, *Output, *Truncated); if (!Replaced) return std::unexpected(Replaced.error()); } for (MachineInstr *Extension : FoldedExtensions) { auto Erased = eraseInstruction(*Extension); if (!Erased) return std::unexpected(Erased.error()); } return eraseInstruction(Instruction); } struct WideValueParts final { std::uint32_t BitWidth = 0; std::vector Parts; std::vector Artifacts; }; [[nodiscard]] Expected collectWideValueParts(Register Value, MachineFunction &Function, unsigned Depth = 0) { if (Depth > 16) return makeUnexpected(ErrorCode::Unsupported, "RISC-V wide-value artifact chain is too deep"); const auto Info = Function.registerInfo().virtualRegisterInfo(Value); if (!Info || !Info->get().Type) return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide value has no scalar type"); const LLT Type = *Info->get().Type; if ((!Type.isInteger() && !Type.isAnyScalar()) || Type.scalarBitWidth() == 0) return makeUnexpected(ErrorCode::Unsupported, "RISC-V wide-value decomposition requires an integer scalar"); if (Type.scalarBitWidth() <= 64) return WideValueParts{ .BitWidth = Type.scalarBitWidth(), .Parts = {Value}, .Artifacts = {}}; const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(Value); MachineInstr *Def = Definition == nullptr ? nullptr : const_cast(Definition->parent()); if (Def == nullptr) return makeUnexpected(ErrorCode::Unsupported, "RISC-V wide value has no decomposable definition"); if (Def->opcode() == opcode(GenericOpcode::G_MERGE_VALUES)) { WideValueParts Result; Result.BitWidth = Type.scalarBitWidth(); for (std::size_t Index = 1; Index < Def->operandCount(); ++Index) { const MachineOperand *Operand = Def->operandAt(Index); const Register *Part = Operand == nullptr ? nullptr : Operand->getIf(); const std::optional PartType = Operand == nullptr ? std::nullopt : registerType(Function, *Operand); if (Part == nullptr || !PartType || PartType->scalarBitWidth() != 64) return makeUnexpected(ErrorCode::Unsupported, "RISC-V wide merge must already use 64-bit limbs"); Result.Parts.push_back(*Part); } Result.Artifacts.push_back(Def); return Result; } if (Def->opcode() == opcode(GenericOpcode::G_TRUNC)) { const MachineOperand *SourceOperand = Def->operandAt(1); const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); if (Source == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide truncation has no source register"); auto Result = collectWideValueParts(*Source, Function, Depth + 1U); if (!Result) return std::unexpected(Result.error()); Result->BitWidth = Type.scalarBitWidth(); const std::size_t RequiredParts = (Type.scalarBitWidth() + 63U) / 64U; if (Result->Parts.size() < RequiredParts) return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide truncation source has too few limbs"); Result->Parts.resize(RequiredParts); Result->Artifacts.push_back(Def); return Result; } return makeUnexpected(ErrorCode::Unsupported, "RISC-V wide value is not defined by merge/trunc artifacts"); } struct WideResultPlan final { std::vector Outputs; MachineInstr *Extension = nullptr; MachineInstr *Unmerge = nullptr; }; [[nodiscard]] Expected wideResultPlan(Register Result, MachineFunction &Function, std::size_t RequiredParts) { MachineInstr *Use = singleUseInstruction(Function, Result); WideResultPlan Plan; if (Use != nullptr && (Use->opcode() == opcode(GenericOpcode::G_ANYEXT) || Use->opcode() == opcode(GenericOpcode::G_SEXT) || Use->opcode() == opcode(GenericOpcode::G_ZEXT))) { Plan.Extension = Use; const MachineOperand *ExtendedOperand = Use->operandAt(0); const Register *Extended = ExtendedOperand == nullptr ? nullptr : ExtendedOperand->getIf(); Use = Extended == nullptr ? nullptr : singleUseInstruction(Function, *Extended); } if (Use == nullptr || Use->opcode() != opcode(GenericOpcode::G_UNMERGE_VALUES)) return makeUnexpected(ErrorCode::Unsupported, "RISC-V wide result must feed an unmerge artifact"); Plan.Unmerge = Use; const std::size_t OutputCount = Use->operandCount() - 1U; if (OutputCount < RequiredParts) return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide result unmerge has too few outputs"); Plan.Outputs.reserve(RequiredParts); for (std::size_t Index = 0; Index < RequiredParts; ++Index) { const MachineOperand *Operand = Use->operandAt(Index); const Register *Output = Operand == nullptr ? nullptr : Operand->getIf(); const std::optional Type = Operand == nullptr ? std::nullopt : registerType(Function, *Operand); if (Output == nullptr || !Type || Type->scalarBitWidth() != 64) return makeUnexpected(ErrorCode::Unsupported, "RISC-V wide result must unmerge to 64-bit limbs"); Plan.Outputs.push_back(*Output); } return Plan; } [[nodiscard]] Expected buildIntegerCompareBefore(MachineIRBuilder &Builder, MachineInstr &Anchor, IntegerPredicate Predicate, Register Left, Register Right) { auto Type = LLT::makeAnyScalar(64); if (!Type) return std::unexpected(Type.error()); auto Result = Builder.function().registerInfo().createVirtualRegister(*Type); if (!Result) return std::unexpected(Result.error()); auto Point = Builder.setInsertPointBefore(Anchor); if (!Point) return std::unexpected(Point.error()); auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_ICMP)); if (!Pending) return std::unexpected(Pending.error()); if (auto Added = Pending->addDef(*Result); !Added) return std::unexpected(Added.error()); if (auto Added = Pending->addIntegerPredicate(Predicate); !Added) return std::unexpected(Added.error()); if (auto Added = Pending->addUse(Left); !Added) return std::unexpected(Added.error()); if (auto Added = Pending->addUse(Right); !Added) return std::unexpected(Added.error()); auto Built = Pending->build(); if (!Built) return std::unexpected(Built.error()); return *Result; } [[nodiscard]] Expected buildSelectBefore(MachineIRBuilder &Builder, MachineInstr &Anchor, LLT Type, Register Condition, Register TrueValue, Register FalseValue) { auto Result = Builder.function().registerInfo().createVirtualRegister(Type); if (!Result) return std::unexpected(Result.error()); auto Point = Builder.setInsertPointBefore(Anchor); if (!Point) return std::unexpected(Point.error()); auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_SELECT)); if (!Pending) return std::unexpected(Pending.error()); if (auto Added = Pending->addDef(*Result); !Added) return std::unexpected(Added.error()); if (auto Added = Pending->addUse(Condition); !Added) return std::unexpected(Added.error()); if (auto Added = Pending->addUse(TrueValue); !Added) return std::unexpected(Added.error()); if (auto Added = Pending->addUse(FalseValue); !Added) return std::unexpected(Added.error()); auto Built = Pending->build(); if (!Built) return std::unexpected(Built.error()); return *Result; } struct TwoLimbShiftResult final { Register Low; Register High; }; [[nodiscard]] Expected emitTwoLimbLeftShiftBody(MachineIRBuilder &Builder, MachineInstr &Anchor, Register Low, Register High, Register Amount, Register AmountExcess, Register AmountLack, Register Zero, Register IsShort, Register IsZero) { auto Type = LLT::makeAnyScalar(64); if (!Type) return std::unexpected(Type.error()); auto LowShort = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SHL, *Type, Low, Amount); if (!LowShort) return std::unexpected(LowShort.error()); auto CrossLow = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_LSHR, *Type, Low, AmountLack); if (!CrossLow) return std::unexpected(CrossLow.error()); auto HighShort = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SHL, *Type, High, Amount); if (!HighShort) return std::unexpected(HighShort.error()); auto ShortHigh = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_OR, *Type, *CrossLow, *HighShort); if (!ShortHigh) return std::unexpected(ShortHigh.error()); auto LongHigh = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SHL, *Type, Low, AmountExcess); if (!LongHigh) return std::unexpected(LongHigh.error()); auto ResultLow = buildSelectBefore(Builder, Anchor, *Type, IsShort, *LowShort, Zero); if (!ResultLow) return std::unexpected(ResultLow.error()); auto ShiftedHigh = buildSelectBefore(Builder, Anchor, *Type, IsShort, *ShortHigh, *LongHigh); if (!ShiftedHigh) return std::unexpected(ShiftedHigh.error()); auto ResultHigh = buildSelectBefore(Builder, Anchor, *Type, IsZero, High, *ShiftedHigh); if (!ResultHigh) return std::unexpected(ResultHigh.error()); return TwoLimbShiftResult{.Low = *ResultLow, .High = *ResultHigh}; } [[nodiscard]] Expected emitTwoLimbLeftShift(MachineIRBuilder &Builder, MachineInstr &Anchor, Register Low, Register High, Register Amount) { auto Type = LLT::makeAnyScalar(64); if (!Type) return std::unexpected(Type.error()); auto Boundary = buildIntegerConstantBefore(Builder, Anchor, 64, 64); if (!Boundary) return std::unexpected(Boundary.error()); auto AmountExcess = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, Amount, *Boundary); if (!AmountExcess) return std::unexpected(AmountExcess.error()); auto AmountLack = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, *Boundary, Amount); if (!AmountLack) return std::unexpected(AmountLack.error()); auto Zero = buildIntegerConstantBefore(Builder, Anchor, 64, 0); if (!Zero) return std::unexpected(Zero.error()); auto IsShort = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan, Amount, *Boundary); if (!IsShort) return std::unexpected(IsShort.error()); auto IsZero = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::Equal, Amount, *Zero); if (!IsZero) return std::unexpected(IsZero.error()); return emitTwoLimbLeftShiftBody(Builder, Anchor, Low, High, Amount, *AmountExcess, *AmountLack, *Zero, *IsShort, *IsZero); } [[nodiscard]] Expected emitLowLimbOfTwoLimbLogicalRightShift( MachineIRBuilder &Builder, MachineInstr &Anchor, Register Low, Register High, Register Amount, Register Boundary, Register Zero) { auto Type = LLT::makeAnyScalar(64); if (!Type) return std::unexpected(Type.error()); auto AmountExcess = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, Amount, Boundary); if (!AmountExcess) return std::unexpected(AmountExcess.error()); auto AmountLack = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, Boundary, Amount); if (!AmountLack) return std::unexpected(AmountLack.error()); auto IsShort = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan, Amount, Boundary); if (!IsShort) return std::unexpected(IsShort.error()); auto IsZero = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::Equal, Amount, Zero); if (!IsZero) return std::unexpected(IsZero.error()); auto LowShort = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_LSHR, *Type, Low, Amount); if (!LowShort) return std::unexpected(LowShort.error()); auto HighShort = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SHL, *Type, High, *AmountLack); if (!HighShort) return std::unexpected(HighShort.error()); auto ShortValue = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_OR, *Type, *LowShort, *HighShort); if (!ShortValue) return std::unexpected(ShortValue.error()); auto LongValue = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_LSHR, *Type, High, *AmountExcess); if (!LongValue) return std::unexpected(LongValue.error()); auto Shifted = buildSelectBefore(Builder, Anchor, *Type, *IsShort, *ShortValue, *LongValue); if (!Shifted) return std::unexpected(Shifted.error()); return buildSelectBefore(Builder, Anchor, *Type, *IsZero, Low, *Shifted); } [[nodiscard]] Expected> emitThreeLimbLeftShift(MachineIRBuilder &Builder, MachineInstr &Anchor, std::span Parts, Register Amount) { if (Parts.size() != 3) return makeUnexpected(ErrorCode::VerificationError, "RISC-V three-limb shift requires exactly three limbs"); auto Type = LLT::makeAnyScalar(64); if (!Type) return std::unexpected(Type.error()); auto OuterBoundary = buildIntegerConstantBefore(Builder, Anchor, 64, 128); if (!OuterBoundary) return std::unexpected(OuterBoundary.error()); auto OuterExcess = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, Amount, *OuterBoundary); if (!OuterExcess) return std::unexpected(OuterExcess.error()); auto OuterLack = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, *OuterBoundary, Amount); if (!OuterLack) return std::unexpected(OuterLack.error()); auto Zero = buildIntegerConstantBefore(Builder, Anchor, 64, 0); if (!Zero) return std::unexpected(Zero.error()); auto OuterIsShort = buildIntegerCompareBefore( Builder, Anchor, IntegerPredicate::UnsignedLessThan, Amount, *OuterBoundary); if (!OuterIsShort) return std::unexpected(OuterIsShort.error()); auto OuterIsZero = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::Equal, Amount, *Zero); if (!OuterIsZero) return std::unexpected(OuterIsZero.error()); auto InnerBoundary = buildIntegerConstantBefore(Builder, Anchor, 64, 64); if (!InnerBoundary) return std::unexpected(InnerBoundary.error()); auto InnerExcess = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, Amount, *InnerBoundary); if (!InnerExcess) return std::unexpected(InnerExcess.error()); auto InnerLack = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, *InnerBoundary, Amount); if (!InnerLack) return std::unexpected(InnerLack.error()); auto InnerIsShort = buildIntegerCompareBefore( Builder, Anchor, IntegerPredicate::UnsignedLessThan, Amount, *InnerBoundary); if (!InnerIsShort) return std::unexpected(InnerIsShort.error()); auto InnerIsZero = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::Equal, Amount, *Zero); if (!InnerIsZero) return std::unexpected(InnerIsZero.error()); auto Lower = emitTwoLimbLeftShiftBody( Builder, Anchor, Parts[0], Parts[1], Amount, *InnerExcess, *InnerLack, *Zero, *InnerIsShort, *InnerIsZero); if (!Lower) return std::unexpected(Lower.error()); auto Cross = emitLowLimbOfTwoLimbLogicalRightShift( Builder, Anchor, Parts[0], Parts[1], *OuterLack, *InnerBoundary, *Zero); if (!Cross) return std::unexpected(Cross.error()); auto HighIsShort = buildIntegerCompareBefore( Builder, Anchor, IntegerPredicate::UnsignedLessThan, Amount, *InnerBoundary); if (!HighIsShort) return std::unexpected(HighIsShort.error()); auto HighShort = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SHL, *Type, Parts[2], Amount); if (!HighShort) return std::unexpected(HighShort.error()); auto BoundedHigh = buildSelectBefore(Builder, Anchor, *Type, *HighIsShort, *HighShort, *Zero); if (!BoundedHigh) return std::unexpected(BoundedHigh.error()); auto ShortHigh = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_OR, *Type, *Cross, *BoundedHigh); if (!ShortHigh) return std::unexpected(ShortHigh.error()); auto LongIsShort = buildIntegerCompareBefore( Builder, Anchor, IntegerPredicate::UnsignedLessThan, *OuterExcess, *InnerBoundary); if (!LongIsShort) return std::unexpected(LongIsShort.error()); auto LongShift = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SHL, *Type, Parts[0], *OuterExcess); if (!LongShift) return std::unexpected(LongShift.error()); auto LongHigh = buildSelectBefore(Builder, Anchor, *Type, *LongIsShort, *LongShift, *Zero); if (!LongHigh) return std::unexpected(LongHigh.error()); auto ResultLow = buildSelectBefore(Builder, Anchor, *Type, *OuterIsShort, Lower->Low, *Zero); if (!ResultLow) return std::unexpected(ResultLow.error()); auto ResultMiddle = buildSelectBefore(Builder, Anchor, *Type, *OuterIsShort, Lower->High, *Zero); if (!ResultMiddle) return std::unexpected(ResultMiddle.error()); auto ShiftedHigh = buildSelectBefore(Builder, Anchor, *Type, *OuterIsShort, *ShortHigh, *LongHigh); if (!ShiftedHigh) return std::unexpected(ShiftedHigh.error()); auto ResultHigh = buildSelectBefore(Builder, Anchor, *Type, *OuterIsZero, Parts[2], *ShiftedHigh); if (!ResultHigh) return std::unexpected(ResultHigh.error()); return std::vector{*ResultLow, *ResultMiddle, *ResultHigh}; } [[nodiscard]] Expected lowerWideShiftLeft(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target) { const MachineOperand *DestinationOperand = Instruction.operandAt(0); const MachineOperand *ValueOperand = Instruction.operandAt(1); const MachineOperand *AmountOperand = Instruction.operandAt(2); const Register *Destination = DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); const Register *Value = ValueOperand == nullptr ? nullptr : ValueOperand->getIf(); const Register *Amount = AmountOperand == nullptr ? nullptr : AmountOperand->getIf(); if (Destination == nullptr || Value == nullptr || Amount == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide shift has invalid register operands"); auto ValueParts = collectWideValueParts(*Value, Function); if (!ValueParts) return std::unexpected(ValueParts.error()); auto AmountParts = collectWideValueParts(*Amount, Function); if (!AmountParts) return std::unexpected(AmountParts.error()); if (ValueParts->Parts.size() < 2 || AmountParts->Parts.empty()) return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide shift has an invalid limb decomposition"); auto ResultPlan = wideResultPlan(*Destination, Function, ValueParts->Parts.size()); if (!ResultPlan) return std::unexpected(ResultPlan.error()); MachineIRBuilder Builder(Function, Target); Expected> Results = ValueParts->Parts.size() == 2 ? emitTwoLimbLeftShift(Builder, Instruction, ValueParts->Parts[0], ValueParts->Parts[1], AmountParts->Parts[0]) .transform([](TwoLimbShiftResult Result) { return std::vector{Result.Low, Result.High}; }) : ValueParts->Parts.size() == 3 ? emitThreeLimbLeftShift(Builder, Instruction, ValueParts->Parts, AmountParts->Parts[0]) : Expected>(std::unexpected( Error(ErrorCode::Unsupported, "RISC-V wide shift currently requires two or three 64-bit limbs"))); if (!Results) return std::unexpected(Results.error()); for (std::size_t Index = 0; Index < Results->size(); ++Index) { auto Replaced = replaceRegisterUses(Function, ResultPlan->Outputs[Index], (*Results)[Index]); if (!Replaced) return std::unexpected(Replaced.error()); } if (auto Erased = eraseInstruction(*ResultPlan->Unmerge); !Erased) return std::unexpected(Erased.error()); if (ResultPlan->Extension != nullptr) { if (auto Erased = eraseInstruction(*ResultPlan->Extension); !Erased) return std::unexpected(Erased.error()); } if (auto Erased = eraseInstruction(Instruction); !Erased) return std::unexpected(Erased.error()); for (std::size_t Index = 1; Index < AmountParts->Parts.size(); ++Index) { const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(AmountParts->Parts[Index]); if (Definition != nullptr && Definition->parent() != nullptr) AmountParts->Artifacts.push_back( const_cast(Definition->parent())); } ValueParts->Artifacts.insert(ValueParts->Artifacts.end(), AmountParts->Artifacts.begin(), AmountParts->Artifacts.end()); eraseDeadArtifacts(ValueParts->Artifacts, Function); return {}; } [[nodiscard]] Expected> emitWideAdd(MachineIRBuilder &Builder, MachineInstr &Anchor, std::span Left, std::span Right) { if (Left.size() != Right.size() || Left.size() < 2) return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide add requires matching multi-limb operands"); auto Type = LLT::makeAnyScalar(64); if (!Type) return std::unexpected(Type.error()); std::vector Results; Results.reserve(Left.size()); auto Low = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_ADD, *Type, Left[0], Right[0]); if (!Low) return std::unexpected(Low.error()); auto Carry = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan, *Low, Right[0]); if (!Carry) return std::unexpected(Carry.error()); auto LowCopy = buildUnaryBefore(Builder, Anchor, GenericOpcode::COPY, *Type, *Low); if (!LowCopy) return std::unexpected(LowCopy.error()); Results.push_back(*LowCopy); for (std::size_t Index = 1; Index < Left.size(); ++Index) { auto Base = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_ADD, *Type, Left[Index], Right[Index]); if (!Base) return std::unexpected(Base.error()); std::optional BaseCarry; if (Index + 1U < Left.size()) { auto Compared = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan, *Base, Left[Index]); if (!Compared) return std::unexpected(Compared.error()); BaseCarry = *Compared; } auto Sum = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_ADD, *Type, *Base, *Carry); if (!Sum) return std::unexpected(Sum.error()); if (Index + 1U < Left.size()) { auto Zero = buildIntegerConstantBefore(Builder, Anchor, 64, 0); if (!Zero) return std::unexpected(Zero.error()); auto IsZero = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::Equal, *Sum, *Zero); if (!IsZero) return std::unexpected(IsZero.error()); auto CarryFromIncoming = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_AND, *Type, *IsZero, *Carry); if (!CarryFromIncoming) return std::unexpected(CarryFromIncoming.error()); auto NextCarry = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_OR, *Type, *BaseCarry, *CarryFromIncoming); if (!NextCarry) return std::unexpected(NextCarry.error()); Carry = *NextCarry; } auto Copy = buildUnaryBefore(Builder, Anchor, GenericOpcode::COPY, *Type, *Sum); if (!Copy) return std::unexpected(Copy.error()); Results.push_back(*Copy); } return Results; } [[nodiscard]] Expected> emitWideSub(MachineIRBuilder &Builder, MachineInstr &Anchor, std::span Left, std::span Right) { if (Left.size() != Right.size() || Left.size() < 2) return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide sub requires matching multi-limb operands"); auto Type = LLT::makeAnyScalar(64); if (!Type) return std::unexpected(Type.error()); std::vector Results; Results.reserve(Left.size()); auto Low = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, Left[0], Right[0]); if (!Low) return std::unexpected(Low.error()); Results.push_back(*Low); auto Borrow = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan, Left[0], Right[0]); if (!Borrow) return std::unexpected(Borrow.error()); for (std::size_t Index = 1; Index < Left.size(); ++Index) { auto Base = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, Left[Index], Right[Index]); if (!Base) return std::unexpected(Base.error()); std::optional BaseBorrow; if (Index + 1U < Left.size()) { auto Compared = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedGreaterThan, *Base, Left[Index]); if (!Compared) return std::unexpected(Compared.error()); BaseBorrow = *Compared; } auto Difference = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, *Base, *Borrow); if (!Difference) return std::unexpected(Difference.error()); Results.push_back(*Difference); if (Index + 1U < Left.size()) { auto Zero = buildIntegerConstantBefore(Builder, Anchor, 64, 0); if (!Zero) return std::unexpected(Zero.error()); auto IsZero = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::Equal, *Base, *Zero); if (!IsZero) return std::unexpected(IsZero.error()); auto BorrowFromIncoming = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_AND, *Type, *IsZero, *Borrow); if (!BorrowFromIncoming) return std::unexpected(BorrowFromIncoming.error()); auto NextBorrow = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_OR, *Type, *BaseBorrow, *BorrowFromIncoming); if (!NextBorrow) return std::unexpected(NextBorrow.error()); Borrow = *NextBorrow; } } return Results; } [[nodiscard]] Expected> emitWideBitwise(MachineIRBuilder &Builder, MachineInstr &Anchor, GenericOpcode Generic, std::span Left, std::span Right) { if (Left.size() != Right.size() || Left.size() < 2) return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide bitwise operation requires matching multi-limb operands"); auto Type = LLT::makeAnyScalar(64); if (!Type) return std::unexpected(Type.error()); std::vector Results; Results.reserve(Left.size()); for (std::size_t Index = 0; Index < Left.size(); ++Index) { auto Part = buildBinaryBefore(Builder, Anchor, Generic, *Type, Left[Index], Right[Index]); if (!Part) return std::unexpected(Part.error()); Results.push_back(*Part); } return Results; } [[nodiscard]] Expected lowerWideBinary(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target, GenericOpcode Generic) { const MachineOperand *DestinationOperand = Instruction.operandAt(0); const MachineOperand *LeftOperand = Instruction.operandAt(1); const MachineOperand *RightOperand = Instruction.operandAt(2); const Register *Destination = DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf(); const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf(); if (Destination == nullptr || Left == nullptr || Right == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide binary instruction has invalid operands"); auto LeftParts = collectWideValueParts(*Left, Function); if (!LeftParts) return std::unexpected(LeftParts.error()); auto RightParts = collectWideValueParts(*Right, Function); if (!RightParts) return std::unexpected(RightParts.error()); if (LeftParts->Parts.size() != RightParts->Parts.size()) return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide binary operands have different limb counts"); auto ResultPlan = wideResultPlan(*Destination, Function, LeftParts->Parts.size()); if (!ResultPlan) return std::unexpected(ResultPlan.error()); MachineIRBuilder Builder(Function, Target); Expected> Results = Generic == GenericOpcode::G_ADD ? emitWideAdd(Builder, Instruction, LeftParts->Parts, RightParts->Parts) : Generic == GenericOpcode::G_SUB ? emitWideSub(Builder, Instruction, LeftParts->Parts, RightParts->Parts) : emitWideBitwise(Builder, Instruction, Generic, LeftParts->Parts, RightParts->Parts); if (!Results) return std::unexpected(Results.error()); for (std::size_t Index = 0; Index < Results->size(); ++Index) { auto Replaced = replaceRegisterUses(Function, ResultPlan->Outputs[Index], (*Results)[Index]); if (!Replaced) return std::unexpected(Replaced.error()); } if (auto Erased = eraseInstruction(*ResultPlan->Unmerge); !Erased) return std::unexpected(Erased.error()); if (ResultPlan->Extension != nullptr) { if (auto Erased = eraseInstruction(*ResultPlan->Extension); !Erased) return std::unexpected(Erased.error()); } if (auto Erased = eraseInstruction(Instruction); !Erased) return std::unexpected(Erased.error()); LeftParts->Artifacts.insert(LeftParts->Artifacts.end(), RightParts->Artifacts.begin(), RightParts->Artifacts.end()); eraseDeadArtifacts(LeftParts->Artifacts, Function); return {}; } [[nodiscard]] bool canLowerNarrowBranchCondition(const MachineInstr &Instruction, const MachineFunction &Function) { if (Instruction.opcode() != opcode(GenericOpcode::G_BRCOND) || Instruction.operandCount() < 2) return false; const MachineOperand *ConditionOperand = Instruction.operandAt(0); const Register *Condition = ConditionOperand == nullptr ? nullptr : ConditionOperand->getIf(); if (Condition == nullptr || !Condition->isVirtual() || singleRegisterUse(Function, *Condition) != &Instruction) return false; const auto ConditionInfo = Function.registerInfo().virtualRegisterInfo(*Condition); const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(*Condition); const MachineInstr *Truncation = Definition == nullptr ? nullptr : Definition->parent(); if (!ConditionInfo || !ConditionInfo->get().Type || ConditionInfo->get().Type->scalarBitWidth() != 1 || Truncation == nullptr || Truncation->opcode() != opcode(GenericOpcode::G_TRUNC) || Truncation->operandCount() != 2) return false; return operandType(Function, *Truncation, 1) .transform([](LLT Type) { return isScalarGPRType(Type) && Type.scalarBitWidth() == 64; }) .value_or(false); } [[nodiscard]] Expected lowerNarrowBranchCondition(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target) { if (!canLowerNarrowBranchCondition(Instruction, Function)) return makeUnexpected(ErrorCode::Unsupported, "RISC-V branch condition is not a narrow truncation"); const MachineOperand *ConditionOperand = Instruction.operandAt(0); const Register Condition = *ConditionOperand->getIf(); const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(Condition); MachineInstr *Truncation = const_cast(Definition->parent()); const Register Source = *Truncation->operandAt(1)->getIf(); auto Type = LLT::makeAnyScalar(64); if (!Type) return std::unexpected(Type.error()); MachineIRBuilder Builder(Function, Target); auto One = buildIntegerConstantBefore(Builder, Instruction, 64, 1); if (!One) return std::unexpected(One.error()); auto Masked = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_AND, *Type, Source, *One); if (!Masked) return std::unexpected(Masked.error()); auto Replaced = Instruction.operandAt(0)->setRegister(*Masked); if (!Replaced) return std::unexpected(Replaced.error()); if (Function.registerInfo().useEmpty(Condition)) return eraseInstruction(*Truncation); return {}; } struct NarrowPhiIncoming final { Register Value; MachineBasicBlock *Block = nullptr; MachineInstr *ReusableTruncation = nullptr; }; [[nodiscard]] MachineInstr *firstInstructionAfterLeadingCopies(MachineBasicBlock &Block) { MachineInstr *Instruction = Block.firstInstruction(); while (Instruction != nullptr && Instruction->opcode() == opcode(GenericOpcode::COPY)) Instruction = Instruction->next(); return Instruction; } [[nodiscard]] Expected buildIncomingConversion(MachineFunction &Function, const TargetDescription &Target, MachineBasicBlock &Block, MachineInstr *Anchor, GenericOpcode Generic, LLT Type, Register Source) { auto Result = Function.registerInfo().createVirtualRegister(Type); if (!Result) return std::unexpected(Result.error()); MachineIRBuilder Builder(Function, Target); auto Point = Anchor == nullptr ? Builder.setInsertPoint(Block) : Builder.setInsertPointBefore(*Anchor); if (!Point) return std::unexpected(Point.error()); auto Pending = Builder.buildInstr(opcode(Generic)); if (!Pending) return std::unexpected(Pending.error()); if (auto Added = Pending->addDef(*Result); !Added) return std::unexpected(Added.error()); if (auto Added = Pending->addUse(Source); !Added) return std::unexpected(Added.error()); auto Built = Pending->build(); if (!Built) return std::unexpected(Built.error()); return *Result; } [[nodiscard]] Expected lowerNarrowPhi(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target) { const std::optional OriginalType = resultType(Function, Instruction); const MachineOperand *DestinationOperand = Instruction.operandAt(0); const Register *Destination = DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); if (!OriginalType || Destination == nullptr || OriginalType->scalarBitWidth() >= 32 || Instruction.operandCount() < 3 || (Instruction.operandCount() % 2U) == 0) return makeUnexpected(ErrorCode::VerificationError, "RISC-V narrow PHI has invalid operands"); auto LegalType = LLT::makeAnyScalar(32); if (!LegalType) return std::unexpected(LegalType.error()); std::vector Incoming; Incoming.reserve((Instruction.operandCount() - 1U) / 2U); for (std::size_t Index = 1; Index < Instruction.operandCount(); Index += 2) { const MachineOperand *ValueOperand = Instruction.operandAt(Index); const Register *Value = ValueOperand == nullptr ? nullptr : ValueOperand->getIf(); const MachineOperand *BlockOperand = Instruction.operandAt(Index + 1U); const auto *Block = BlockOperand == nullptr ? nullptr : BlockOperand->getIf(); if (Value == nullptr || Block == nullptr || Block->Block == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V narrow PHI has an invalid incoming edge"); MachineInstr *Reusable = nullptr; const MachineOperand *Definition = Value->isVirtual() ? Function.registerInfo().getUniqueVRegDef(*Value) : nullptr; MachineInstr *Def = Definition == nullptr ? nullptr : const_cast(Definition->parent()); if (Def != nullptr && Def->opcode() == opcode(GenericOpcode::G_TRUNC) && Def->operandCount() == 2 && singleRegisterUse(Function, *Value) == &Instruction && operandType(Function, *Def, 1) .transform([](LLT Type) { return Type.scalarBitWidth() >= 32; }) .value_or(false)) { Reusable = Def; } Incoming.push_back( NarrowPhiIncoming{.Value = *Value, .Block = Block->Block, .ReusableTruncation = Reusable}); } for (NarrowPhiIncoming &Entry : Incoming) { if (Entry.ReusableTruncation == nullptr) continue; const std::size_t Count = static_cast(std::ranges::count( Incoming, Entry.ReusableTruncation, &NarrowPhiIncoming::ReusableTruncation)); if (Count != 1) Entry.ReusableTruncation = nullptr; } for (NarrowPhiIncoming &Entry : Incoming) { if (Entry.ReusableTruncation == nullptr) continue; MachineBasicBlock *OriginalBlock = Entry.ReusableTruncation->parent(); if (OriginalBlock == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V narrow PHI input truncation is detached"); auto Removed = OriginalBlock->removeInstruction(*Entry.ReusableTruncation); if (!Removed) return std::unexpected(Removed.error()); } std::vector> Anchors; for (const NarrowPhiIncoming &Entry : Incoming) { if (std::ranges::find(Anchors, Entry.Block, &decltype(Anchors)::value_type::first) == Anchors.end()) { Anchors.emplace_back(Entry.Block, firstInstructionAfterLeadingCopies(*Entry.Block)); } } std::vector LegalIncoming; LegalIncoming.reserve(Incoming.size()); for (NarrowPhiIncoming &Entry : Incoming) { const auto AnchorPosition = std::ranges::find( Anchors, Entry.Block, &decltype(Anchors)::value_type::first); MachineInstr *Anchor = AnchorPosition->second; if (Entry.ReusableTruncation != nullptr) { auto SetType = Function.registerInfo().setType(Entry.Value, *LegalType); if (!SetType) return std::unexpected(SetType.error()); auto Inserted = Anchor == nullptr ? Entry.Block->appendInstruction(*Entry.ReusableTruncation) : Entry.Block->insertInstructionBefore( *Anchor, *Entry.ReusableTruncation); if (!Inserted) return std::unexpected(Inserted.error()); LegalIncoming.push_back(Entry.Value); continue; } const auto ValueInfo = Function.registerInfo().virtualRegisterInfo(Entry.Value); if (!ValueInfo || !ValueInfo->get().Type) return makeUnexpected(ErrorCode::VerificationError, "RISC-V narrow PHI input has no scalar type"); const std::uint32_t Width = ValueInfo->get().Type->scalarBitWidth(); const GenericOpcode Conversion = Width < 32 ? GenericOpcode::G_ANYEXT : Width > 32 ? GenericOpcode::G_TRUNC : GenericOpcode::COPY; auto Converted = buildIncomingConversion(Function, Target, *Entry.Block, Anchor, Conversion, *LegalType, Entry.Value); if (!Converted) return std::unexpected(Converted.error()); LegalIncoming.push_back(*Converted); } auto LegalResult = Function.registerInfo().createVirtualRegister(*LegalType); if (!LegalResult) return std::unexpected(LegalResult.error()); MachineIRBuilder Builder(Function, Target); auto Point = Builder.setInsertPointBefore(Instruction); if (!Point) return std::unexpected(Point.error()); auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_PHI)); if (!Pending) return std::unexpected(Pending.error()); if (auto Added = Pending->addDef(*LegalResult); !Added) return std::unexpected(Added.error()); for (std::size_t Index = 0; Index < Incoming.size(); ++Index) { if (auto Added = Pending->addUse(LegalIncoming[Index]); !Added) return std::unexpected(Added.error()); if (auto Added = Pending->addBasicBlock(*Incoming[Index].Block); !Added) return std::unexpected(Added.error()); } auto Built = Pending->build(); if (!Built) return std::unexpected(Built.error()); auto Replaced = replaceRegisterUses(Function, *Destination, *LegalResult); if (!Replaced) return std::unexpected(Replaced.error()); return eraseInstruction(Instruction); } [[nodiscard]] Expected lowerWidePhi(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target) { const MachineOperand *DestinationOperand = Instruction.operandAt(0); const Register *Destination = DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); if (Destination == nullptr || Instruction.operandCount() < 5 || (Instruction.operandCount() % 2U) == 0) return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide PHI has invalid operands"); std::vector IncomingParts; std::vector IncomingBlocks; for (std::size_t Index = 1; Index < Instruction.operandCount(); Index += 2) { const MachineOperand *ValueOperand = Instruction.operandAt(Index); const Register *Value = ValueOperand == nullptr ? nullptr : ValueOperand->getIf(); const MachineOperand *BlockOperand = Instruction.operandAt(Index + 1U); const auto *Block = BlockOperand == nullptr ? nullptr : BlockOperand->getIf(); if (Value == nullptr || Block == nullptr || Block->Block == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide PHI has an invalid incoming edge"); auto Parts = collectWideValueParts(*Value, Function); if (!Parts) return std::unexpected(Parts.error()); if (!IncomingParts.empty() && Parts->Parts.size() != IncomingParts.front().Parts.size()) return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide PHI incoming values have different limb counts"); IncomingParts.push_back(std::move(*Parts)); IncomingBlocks.push_back(Block->Block); } const std::size_t PartCount = IncomingParts.front().Parts.size(); auto ResultPlan = wideResultPlan(*Destination, Function, PartCount); if (!ResultPlan) return std::unexpected(ResultPlan.error()); auto Type = LLT::makeAnyScalar(64); if (!Type) return std::unexpected(Type.error()); MachineIRBuilder Builder(Function, Target); std::vector Results; Results.reserve(PartCount); for (std::size_t PartIndex = 0; PartIndex < PartCount; ++PartIndex) { auto Result = Function.registerInfo().createVirtualRegister(*Type); if (!Result) return std::unexpected(Result.error()); auto Point = Builder.setInsertPointBefore(Instruction); if (!Point) return std::unexpected(Point.error()); auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_PHI)); if (!Pending) return std::unexpected(Pending.error()); if (auto Added = Pending->addDef(*Result); !Added) return std::unexpected(Added.error()); for (std::size_t IncomingIndex = 0; IncomingIndex < IncomingParts.size(); ++IncomingIndex) { if (auto Added = Pending->addUse(IncomingParts[IncomingIndex].Parts[PartIndex]); !Added) return std::unexpected(Added.error()); if (auto Added = Pending->addBasicBlock(*IncomingBlocks[IncomingIndex]); !Added) return std::unexpected(Added.error()); } auto Built = Pending->build(); if (!Built) return std::unexpected(Built.error()); Results.push_back(*Result); } for (std::size_t Index = 0; Index < PartCount; ++Index) { auto Replaced = replaceRegisterUses(Function, ResultPlan->Outputs[Index], Results[Index]); if (!Replaced) return std::unexpected(Replaced.error()); } if (auto Erased = eraseInstruction(*ResultPlan->Unmerge); !Erased) return std::unexpected(Erased.error()); if (ResultPlan->Extension != nullptr) { if (auto Erased = eraseInstruction(*ResultPlan->Extension); !Erased) return std::unexpected(Erased.error()); } if (auto Erased = eraseInstruction(Instruction); !Erased) return std::unexpected(Erased.error()); std::vector Artifacts; for (WideValueParts &Parts : IncomingParts) Artifacts.insert(Artifacts.end(), Parts.Artifacts.begin(), Parts.Artifacts.end()); eraseDeadArtifacts(Artifacts, Function); return {}; } [[nodiscard]] bool canFoldWideMergeImmediately(const MachineInstr &Instruction, const MachineFunction &Function) { const MachineOperand *LowOperand = Instruction.operandAt(1); const Register *Low = LowOperand == nullptr ? nullptr : LowOperand->getIf(); if (Low == nullptr) return false; const auto LowInfo = Function.registerInfo().virtualRegisterInfo(*Low); if (!LowInfo || !LowInfo->get().Type) return false; const MachineOperand *DestinationOperand = Instruction.operandAt(0); const Register *Destination = DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); if (Destination == nullptr) return false; auto &RegisterInfo = const_cast(Function.registerInfo()); for (MachineOperand &Use : RegisterInfo.useOperands(*Destination)) { MachineInstr *Trunc = Use.parent(); if (Trunc == nullptr || Trunc->opcode() != opcode(GenericOpcode::G_TRUNC) || Trunc->operandAt(1) != &Use) return false; const std::optional Result = resultType(Function, *Trunc); if (!Result || Result->scalarBitWidth() > LowInfo->get().Type->scalarBitWidth()) return false; } return true; } class RISCVLegalizerInfo final : public TargetLegalizerInfo { public: explicit RISCVLegalizerInfo(RISCVSubtargetFeatures Features) : Features(Features) {} [[nodiscard]] Expected classify(const MachineInstr &Instruction, const MachineFunction &Function, const TargetDescription &) const override { if (!Features.has(RISCVExtension::I)) { return makeUnexpected(ErrorCode::InternalError, "RISC-V legalizer requires the base I extension"); } if (!isGenericOpcode(Instruction.opcode())) return LegalizationAction::Legal; const GenericOpcode Generic = static_cast(Instruction.opcode().value()); const std::optional Type = resultType(Function, Instruction); switch (Generic) { case GenericOpcode::G_BR: return LegalizationAction::Legal; case GenericOpcode::G_BRCOND: { const std::optional Condition = operandType(Function, Instruction, 0); if (!Condition || !isScalarGPRType(*Condition)) return LegalizationAction::Unsupported; return canLowerNarrowBranchCondition(Instruction, Function) ? LegalizationAction::Custom : LegalizationAction::Legal; } case GenericOpcode::G_PHI: { if (!Type) return LegalizationAction::Unsupported; for (std::size_t Index = 1; Index < Instruction.operandCount(); Index += 2) { const std::optional Incoming = operandType(Function, Instruction, Index); if (!Incoming || *Incoming != *Type) return LegalizationAction::Unsupported; } if ((Type->isInteger() || Type->isAnyScalar()) && Type->scalarBitWidth() > 64) return LegalizationAction::Custom; if ((Type->isInteger() || Type->isAnyScalar()) && Type->scalarBitWidth() < 32) return LegalizationAction::Custom; if (!isSupportedIntegerType(*Type) && !isRVFloatType(*Type)) return LegalizationAction::Unsupported; return LegalizationAction::Legal; } case GenericOpcode::G_IMPLICIT_DEF: return Type && (isSupportedIntegerType(*Type) || isRVFloatType(*Type)) ? LegalizationAction::Legal : LegalizationAction::Unsupported; case GenericOpcode::G_STORE: if (Instruction.operandCount() == 0) return LegalizationAction::Unsupported; if (const std::optional ValueType = registerType(Function, *Instruction.operandAt(0)); ValueType) { const std::optional AddressType = operandType(Function, Instruction, 1); if (!AddressType || !AddressType->isPointer() || AddressType->addressSpace() != 0 || AddressType->scalarBitWidth() != 64) { return LegalizationAction::Unsupported; } if (Instruction.memOperands().size() == 1 && Instruction.memOperands().front()->isAtomic()) { const std::uint64_t Bytes = ValueType->scalarBitWidth() / 8U; return isSupportedIntegerType(*ValueType) && isSupportedAtomicMemoryAccess(Instruction, AtomicOperation::Store, Bytes) ? LegalizationAction::Legal : LegalizationAction::Unsupported; } if ((isSupportedIntegerType(*ValueType) || isRVFloatType(*ValueType)) && isSupportedNonAtomicMemoryAccess(Instruction) && memoryMatchesType(Instruction, *ValueType)) { return LegalizationAction::Legal; } } return LegalizationAction::Unsupported; case GenericOpcode::G_SEXTLOAD: case GenericOpcode::G_ZEXTLOAD: { const auto Bytes = memoryAccessBytes(Instruction); const std::optional AddressType = operandType(Function, Instruction, 1); return Type && Bytes && AddressType && isSupportedIntegerType(*Type) && AddressType->isPointer() && AddressType->addressSpace() == 0 && AddressType->scalarBitWidth() == 64 && *Bytes < ((Type->scalarBitWidth() + 7U) / 8U) && (*Bytes == 1 || *Bytes == 2 || *Bytes == 4) && isSupportedNonAtomicMemoryAccess(Instruction) ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_FPEXTLOAD: { const auto Bytes = memoryAccessBytes(Instruction); const std::optional AddressType = operandType(Function, Instruction, 1); return Type && Bytes && AddressType && Type->isFloat() && Type->scalarBitWidth() == 64 && AddressType->isPointer() && AddressType->addressSpace() == 0 && AddressType->scalarBitWidth() == 64 && *Bytes == 4 && isSupportedNonAtomicMemoryAccess(Instruction) ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_FPTRUNCSTORE: { const std::optional ValueType = operandType(Function, Instruction, 0); const std::optional AddressType = operandType(Function, Instruction, 1); const auto Bytes = memoryAccessBytes(Instruction); return ValueType && AddressType && Bytes && ValueType->isFloat() && ValueType->scalarBitWidth() == 64 && AddressType->isPointer() && AddressType->addressSpace() == 0 && AddressType->scalarBitWidth() == 64 && *Bytes == 4 && isSupportedNonAtomicMemoryAccess(Instruction) ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_LOAD: if (Type) { const std::optional AddressType = operandType(Function, Instruction, 1); if (!AddressType || !AddressType->isPointer() || AddressType->addressSpace() != 0 || AddressType->scalarBitWidth() != 64) { return LegalizationAction::Unsupported; } if (Instruction.memOperands().size() == 1 && Instruction.memOperands().front()->isAtomic()) { const std::uint64_t Bytes = Type->scalarBitWidth() / 8U; return isSupportedIntegerType(*Type) && isSupportedAtomicMemoryAccess(Instruction, AtomicOperation::Load, Bytes) ? LegalizationAction::Legal : LegalizationAction::Unsupported; } if ((isSupportedIntegerType(*Type) || isRVFloatType(*Type)) && isSupportedNonAtomicMemoryAccess(Instruction) && memoryMatchesType(Instruction, *Type)) { return LegalizationAction::Legal; } } return LegalizationAction::Unsupported; case GenericOpcode::G_FADD: case GenericOpcode::G_FSUB: case GenericOpcode::G_FMUL: case GenericOpcode::G_FDIV: { const std::optional Left = operandType(Function, Instruction, 1); const std::optional Right = operandType(Function, Instruction, 2); return Type && Left && Right && isRVFloatType(*Type) && *Left == *Type && *Right == *Type ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_FMA: { const std::optional First = operandType(Function, Instruction, 1); const std::optional Second = operandType(Function, Instruction, 2); const std::optional Third = operandType(Function, Instruction, 3); return Type && First && Second && Third && isRVFloatType(*Type) && *First == *Type && *Second == *Type && *Third == *Type ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_FNEG: case GenericOpcode::G_FABS: case GenericOpcode::G_FSQRT: { const std::optional Source = operandType(Function, Instruction, 1); return Type && Source && isRVFloatType(*Type) && *Source == *Type ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_FCOPYSIGN: { const std::optional Left = operandType(Function, Instruction, 1); const std::optional Right = operandType(Function, Instruction, 2); return Type && Left && Right && isRVFloatType(*Type) && *Left == *Type && *Right == *Type ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_FCONSTANT: { const MachineOperand *Immediate = Instruction.operandAt(1); const auto *Floating = Immediate == nullptr ? nullptr : Immediate->getIf(); const std::optional Semantics = Type ? Type->floatSemantics() : std::nullopt; return Type && isRVFloatType(*Type) && Floating != nullptr && Semantics && Floating->Bits.semantics() == *Semantics ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_ASSERT_SEXT: case GenericOpcode::G_ASSERT_ZEXT: case GenericOpcode::G_ASSERT_ALIGN: { const std::optional Source = operandType(Function, Instruction, 1); return Type && Source && *Type == *Source && isScalarGPRType(*Type) && Instruction.operandCount() == 3 ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_FREEZE: case GenericOpcode::G_CONSTANT_FOLD_BARRIER: { const std::optional Source = operandType(Function, Instruction, 1); return Type && Source && *Type == *Source && (isSupportedIntegerType(*Type) || isRVFloatType(*Type)) ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_PTRTOINT: { const std::optional Source = operandType(Function, Instruction, 1); return Type && Source && Type->isInteger() && Type->scalarBitWidth() == 64 && Source->isPointer() && Source->scalarBitWidth() == 64 ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_INTTOPTR: { const std::optional Source = operandType(Function, Instruction, 1); return Type && Source && Type->isPointer() && Type->scalarBitWidth() == 64 && Source->isInteger() && Source->scalarBitWidth() == 64 ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_BITCAST: { const std::optional Source = operandType(Function, Instruction, 1); const bool IntegerFloatPair = Type && Source && Type->scalarBitWidth() == Source->scalarBitWidth() && ((Type->isFloat() && Source->isInteger()) || (Type->isInteger() && Source->isFloat())) && (Type->scalarBitWidth() == 32 || Type->scalarBitWidth() == 64); return IntegerFloatPair ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_ANYEXT: case GenericOpcode::G_SEXT: case GenericOpcode::G_ZEXT: { const std::optional Source = operandType(Function, Instruction, 1); if (!Type || !Source) return LegalizationAction::Unsupported; if (!isScalarIntegerExtension(*Type, *Source)) { const MachineOperand *DestinationOperand = Instruction.operandAt(0); const Register *Destination = DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); MachineInstr *OnlyUse = Destination == nullptr ? nullptr : singleUseInstruction(Function, *Destination); return (Type->isInteger() || Type->isAnyScalar()) && (Source->isInteger() || Source->isAnyScalar()) && Type->scalarBitWidth() > 64 && Source->scalarBitWidth() < Type->scalarBitWidth() && OnlyUse != nullptr && OnlyUse->opcode() == opcode(GenericOpcode::G_UNMERGE_VALUES) ? LegalizationAction::Legal : LegalizationAction::Unsupported; } const MachineOperand *SourceOperand = Instruction.operandAt(1); const Register *SourceRegister = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); const MachineOperand *SourceDefinition = SourceRegister == nullptr || !SourceRegister->isVirtual() ? nullptr : Function.registerInfo().getUniqueVRegDef(*SourceRegister); const MachineInstr *SourceDef = SourceDefinition == nullptr ? nullptr : SourceDefinition->parent(); if (SourceDef != nullptr && SourceDef->opcode() == opcode(GenericOpcode::G_CONSTANT)) return LegalizationAction::Custom; if (Generic == GenericOpcode::G_ANYEXT && narrowAnyExtBinaryPlan(Instruction, Function).has_value()) return LegalizationAction::Custom; return LegalizationAction::Legal; } case GenericOpcode::G_TRUNC: { const std::optional Source = operandType(Function, Instruction, 1); if (Type && Source && (Type->isInteger() || Type->isAnyScalar()) && (Source->isInteger() || Source->isAnyScalar()) && Type->scalarBitWidth() > 64 && Source->scalarBitWidth() > Type->scalarBitWidth()) return LegalizationAction::Legal; return Type && Source && isSupportedIntegerScalar(*Type) && isSupportedIntegerScalar(*Source) && Type->scalarBitWidth() != 0 && Source->scalarBitWidth() >= Type->scalarBitWidth() ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_MERGE_VALUES: case GenericOpcode::G_UNMERGE_VALUES: { if (Instruction.operandCount() < 3) return LegalizationAction::Unsupported; const std::size_t SmallTypeIndex = Generic == GenericOpcode::G_MERGE_VALUES ? 1 : 0; const std::size_t BigTypeIndex = Generic == GenericOpcode::G_MERGE_VALUES ? 0 : Instruction.operandCount() - 1U; const std::optional SmallType = operandType(Function, Instruction, SmallTypeIndex); const std::optional BigType = operandType(Function, Instruction, BigTypeIndex); if (SmallType && BigType && Generic == GenericOpcode::G_MERGE_VALUES && BigType->scalarBitWidth() > 64 && !canFoldWideMergeImmediately(Instruction, Function)) return LegalizationAction::Legal; return SmallType && BigType && (SmallType->isInteger() || SmallType->isAnyScalar()) && (BigType->isInteger() || BigType->isAnyScalar()) && SmallType->scalarBitWidth() != 0 && BigType->scalarBitWidth() != 0 ? LegalizationAction::Custom : LegalizationAction::Unsupported; } case GenericOpcode::G_SEXT_INREG: { const std::optional Source = operandType(Function, Instruction, 1); const MachineOperand *WidthOperand = Instruction.operandAt(2); const auto Width = WidthOperand == nullptr ? Expected(std::unexpected( Error(ErrorCode::InvalidArgument, "missing in-register extension width"))) : signedImmediate(*WidthOperand); return Type && Source && *Type == *Source && isScalarGPRType(*Type) && Width && *Width > 0 && static_cast(*Width) <= Type->scalarBitWidth() ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_ICMP: { const MachineOperand *Predicate = Instruction.operandAt(1); const auto *IntegerPredicate = Predicate == nullptr ? nullptr : Predicate->getIf(); const std::optional Left = operandType(Function, Instruction, 2); const std::optional Right = operandType(Function, Instruction, 3); if (!Type || !isSupportedIntegerScalar(*Type) || IntegerPredicate == nullptr || IntegerPredicate->integerPredicate() == nullptr || !Left || !Right || *Left != *Right || !isSupportedIntegerType(*Left)) { return LegalizationAction::Unsupported; } return Type->scalarBitWidth() == 64 && Left->scalarBitWidth() == 64 ? LegalizationAction::Legal : LegalizationAction::Custom; } case GenericOpcode::G_FCMP: { const MachineOperand *Predicate = Instruction.operandAt(1); const auto *FloatingPredicate = Predicate == nullptr ? nullptr : Predicate->getIf(); const std::optional Left = operandType(Function, Instruction, 2); const std::optional Right = operandType(Function, Instruction, 3); return Type && isI1Type(*Type) && FloatingPredicate != nullptr && FloatingPredicate->floatingPredicate() != nullptr && Left && Right && *Left == *Right && isRVFloatType(*Left) ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_FPEXT: case GenericOpcode::G_FPTRUNC: { const std::optional Source = operandType(Function, Instruction, 1); const bool IsExtension = Generic == GenericOpcode::G_FPEXT; return Type && Source && isRVFloatType(*Type) && isRVFloatType(*Source) && ((IsExtension && Source->scalarBitWidth() == 32 && Type->scalarBitWidth() == 64) || (!IsExtension && Source->scalarBitWidth() == 64 && Type->scalarBitWidth() == 32)) ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_FPTOSI: case GenericOpcode::G_FPTOUI: { const std::optional Source = operandType(Function, Instruction, 1); return Type && Source && isSupportedIntegerType(*Type) && isRVFloatType(*Source) ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_SITOFP: case GenericOpcode::G_UITOFP: { const std::optional Source = operandType(Function, Instruction, 1); return Type && Source && isRVFloatType(*Type) && isSupportedIntegerType(*Source) ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_FMINNUM: case GenericOpcode::G_FMAXNUM: { const std::optional Left = operandType(Function, Instruction, 1); const std::optional Right = operandType(Function, Instruction, 2); return Type && Left && Right && isRVFloatType(*Type) && *Left == *Type && *Right == *Type ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_SELECT: { const std::optional Condition = operandType(Function, Instruction, 1); const std::optional TrueValue = operandType(Function, Instruction, 2); const std::optional FalseValue = operandType(Function, Instruction, 3); return Type && Condition && TrueValue && FalseValue && isScalarGPRType(*Condition) && *TrueValue == *Type && *FalseValue == *Type && (isSupportedIntegerType(*Type) || isRVFloatType(*Type)) ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_PTRMASK: { const std::optional Pointer = operandType(Function, Instruction, 1); const std::optional Mask = operandType(Function, Instruction, 2); return Type && Pointer && Mask && Type->isPointer() && Type->addressSpace() == 0 && Type->scalarBitWidth() == 64 && *Pointer == *Type && isSupportedIntegerScalar(*Mask) && Mask->scalarBitWidth() == 64 ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_ABS: { const std::optional Source = operandType(Function, Instruction, 1); return Type && Source && *Source == *Type && isSupportedIntegerScalar(*Type) ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_SDIVREM: case GenericOpcode::G_UDIVREM: { const std::optional Remainder = operandType(Function, Instruction, 1); const std::optional Left = operandType(Function, Instruction, 2); const std::optional Right = operandType(Function, Instruction, 3); return Type && Remainder && Left && Right && isSupportedIntegerScalar(*Type) && *Remainder == *Type && *Left == *Type && *Right == *Type ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_SMULH: case GenericOpcode::G_UMULH: { const std::optional Left = operandType(Function, Instruction, 1); const std::optional Right = operandType(Function, Instruction, 2); return Type && Left && Right && isNativeIntegerScalar(*Type) && *Left == *Type && *Right == *Type ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_SMIN: case GenericOpcode::G_SMAX: case GenericOpcode::G_UMIN: case GenericOpcode::G_UMAX: { const std::optional Left = operandType(Function, Instruction, 1); const std::optional Right = operandType(Function, Instruction, 2); return Type && Left && Right && isSupportedIntegerScalar(*Type) && *Left == *Type && *Right == *Type ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_UADDO: case GenericOpcode::G_USUBO: case GenericOpcode::G_SADDO: case GenericOpcode::G_SSUBO: case GenericOpcode::G_UMULO: case GenericOpcode::G_SMULO: { const std::optional Overflow = operandType(Function, Instruction, 1); const std::optional Left = operandType(Function, Instruction, 2); const std::optional Right = operandType(Function, Instruction, 3); return Type && Overflow && Left && Right && isNativeIntegerScalar(*Type) && isI1Type(*Overflow) && *Left == *Type && *Right == *Type ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_UADDE: case GenericOpcode::G_USUBE: case GenericOpcode::G_SADDE: case GenericOpcode::G_SSUBE: { const std::optional CarryOut = operandType(Function, Instruction, 1); const std::optional Left = operandType(Function, Instruction, 2); const std::optional Right = operandType(Function, Instruction, 3); const std::optional CarryIn = operandType(Function, Instruction, 4); return Type && CarryOut && Left && Right && CarryIn && isNativeIntegerScalar(*Type) && isI1Type(*CarryOut) && isI1Type(*CarryIn) && *Left == *Type && *Right == *Type ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_ATOMICRMW_XCHG: case GenericOpcode::G_ATOMICRMW_ADD: case GenericOpcode::G_ATOMICRMW_SUB: case GenericOpcode::G_ATOMICRMW_AND: case GenericOpcode::G_ATOMICRMW_OR: case GenericOpcode::G_ATOMICRMW_XOR: case GenericOpcode::G_ATOMICRMW_MAX: case GenericOpcode::G_ATOMICRMW_MIN: case GenericOpcode::G_ATOMICRMW_UMAX: case GenericOpcode::G_ATOMICRMW_UMIN: { const std::optional Address = operandType(Function, Instruction, 1); const std::optional Value = operandType(Function, Instruction, 2); const std::uint64_t Bytes = Type ? Type->scalarBitWidth() / 8U : 0; return Type && Address && Value && isNativeIntegerScalar(*Type) && *Value == *Type && Address->isPointer() && Address->addressSpace() == 0 && Address->scalarBitWidth() == 64 && isSupportedAtomicMemoryAccess(Instruction, AtomicOperation::ReadModifyWrite, Bytes) ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_ATOMIC_CMPXCHG: { const std::optional Address = operandType(Function, Instruction, 1); const std::optional Compare = operandType(Function, Instruction, 2); const std::optional NewValue = operandType(Function, Instruction, 3); if (!Type || !Address || !Compare || !NewValue || *Compare != *Type || *NewValue != *Type || (!Type->isInteger() && !Type->isAnyScalar()) || (Type->scalarBitWidth() != 8 && Type->scalarBitWidth() != 16 && Type->scalarBitWidth() != 32 && Type->scalarBitWidth() != 64) || !Address->isPointer() || Address->addressSpace() != 0 || Address->scalarBitWidth() != 64 || !isSupportedAtomicContainerAccess(Instruction, AtomicOperation::CompareExchange, (Type->scalarBitWidth() + 7U) / 8U)) { return LegalizationAction::Unsupported; } return Type->scalarBitWidth() == 64 ? LegalizationAction::Legal : LegalizationAction::Custom; } case GenericOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: { const std::optional Success = operandType(Function, Instruction, 1); const std::optional Address = operandType(Function, Instruction, 2); const std::optional Compare = operandType(Function, Instruction, 3); const std::optional NewValue = operandType(Function, Instruction, 4); return Type && Success && Address && Compare && NewValue && isI1Type(*Success) && *Compare == *Type && *NewValue == *Type && (Type->isInteger() || Type->isAnyScalar()) && (Type->scalarBitWidth() == 8 || Type->scalarBitWidth() == 16 || Type->scalarBitWidth() == 32 || Type->scalarBitWidth() == 64) && Address->isPointer() && Address->addressSpace() == 0 && Address->scalarBitWidth() == 64 && isSupportedAtomicContainerAccess(Instruction, AtomicOperation::CompareExchange, (Type->scalarBitWidth() + 7U) / 8U) ? LegalizationAction::Custom : LegalizationAction::Unsupported; } case GenericOpcode::G_FENCE: { const MachineOperand *OrderingOperand = Instruction.operandAt(0); const MachineOperand *ScopeOperand = Instruction.operandAt(1); const auto Ordering = OrderingOperand == nullptr ? Expected(std::unexpected( Error(ErrorCode::InvalidArgument, "missing fence ordering"))) : signedImmediate(*OrderingOperand); const auto Scope = ScopeOperand == nullptr ? Expected(std::unexpected( Error(ErrorCode::InvalidArgument, "missing fence scope"))) : signedImmediate(*ScopeOperand); return Ordering && Scope && *Ordering >= 4 && *Ordering <= 7 && *Scope == 0 ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_CONSTANT_POOL: { const MachineOperand *PoolOperand = Instruction.operandAt(1); const auto *Pool = PoolOperand == nullptr ? nullptr : PoolOperand->getIf(); return Type && Type->isPointer() && Type->addressSpace() == 0 && Type->scalarBitWidth() == 64 && Pool != nullptr && Function.constantPool().entry(Pool->Index) != nullptr ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_GLOBAL_VALUE: { const MachineOperand *GlobalOperand = Instruction.operandAt(1); const auto *Global = GlobalOperand == nullptr ? nullptr : GlobalOperand->getIf(); return Type && Type->isPointer() && Type->addressSpace() == 0 && Type->scalarBitWidth() == 64 && Global != nullptr && !Global->Name.empty() ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_BRINDIRECT: return operandType(Function, Instruction, 0) .transform([](LLT Value) { return Value.isPointer() && Value.addressSpace() == 0 && Value.scalarBitWidth() == 64; }) .value_or(false) ? LegalizationAction::Legal : LegalizationAction::Unsupported; case GenericOpcode::G_TRAP: case GenericOpcode::G_DEBUGTRAP: return Instruction.operandCount() == 0 ? LegalizationAction::Legal : LegalizationAction::Unsupported; case GenericOpcode::G_UBSANTRAP: return Instruction.operandCount() == 1 ? LegalizationAction::Legal : LegalizationAction::Unsupported; case GenericOpcode::G_CONSTANT: { const MachineOperand *Immediate = Instruction.operandAt(1); if (!Type || Immediate == nullptr || (Immediate->kind() != MachineOperand::Kind::Immediate && Immediate->kind() != MachineOperand::Kind::IntegerImmediate)) return LegalizationAction::Unsupported; if (Type->scalarBitWidth() > 64) return wideConstantUnmergePlan(Instruction, Function).has_value() ? LegalizationAction::Custom : LegalizationAction::Unsupported; if ((Type->isInteger() || Type->isAnyScalar()) && Type->scalarBitWidth() == 64) { auto Value = signedImmediate(*Immediate); if (Value && shouldUseRV64IntegerConstantPool(*Value)) return LegalizationAction::Custom; } return isSupportedIntegerType(*Type) ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_ADD: case GenericOpcode::G_SUB: case GenericOpcode::G_MUL: case GenericOpcode::G_SDIV: case GenericOpcode::G_UDIV: case GenericOpcode::G_SREM: case GenericOpcode::G_UREM: case GenericOpcode::G_AND: case GenericOpcode::G_OR: case GenericOpcode::G_XOR: case GenericOpcode::G_SHL: case GenericOpcode::G_LSHR: case GenericOpcode::G_ASHR: { const std::optional Left = operandType(Function, Instruction, 1); const std::optional Right = operandType(Function, Instruction, 2); if (!Type || !Left || !Right || *Left != *Type || *Right != *Type || (!Type->isInteger() && !Type->isAnyScalar())) return LegalizationAction::Unsupported; if (Type->scalarBitWidth() > 64) { return Generic == GenericOpcode::G_ADD || Generic == GenericOpcode::G_SUB || Generic == GenericOpcode::G_AND || Generic == GenericOpcode::G_OR || Generic == GenericOpcode::G_XOR || Generic == GenericOpcode::G_SHL ? LegalizationAction::Custom : LegalizationAction::Unsupported; } if (Generic == GenericOpcode::G_MUL && !Features.has(RISCVExtension::M)) { if (Type->scalarBitWidth() == 64) return LegalizationAction::Libcall; const MachineOperand *DestinationOperand = Instruction.operandAt(0); const Register *Destination = DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); MachineInstr *Extension = Destination == nullptr ? nullptr : singleRegisterUse(Function, *Destination); const std::optional ExtensionType = Extension == nullptr ? std::nullopt : resultType(Function, *Extension); if (Extension != nullptr && Extension->opcode() == opcode(GenericOpcode::G_ANYEXT) && ExtensionType && ExtensionType->scalarBitWidth() == 64) { return LegalizationAction::Legal; } return LegalizationAction::Custom; } return isSupportedIntegerScalar(*Type) ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_PTR_ADD: { const std::optional Pointer = operandType(Function, Instruction, 1); const std::optional Offset = operandType(Function, Instruction, 2); return Type && Pointer && Offset && Type->isPointer() && Type->addressSpace() == 0 && Type->scalarBitWidth() == 64 && *Pointer == *Type && Offset->isInteger() && Offset->scalarBitWidth() == 64 ? LegalizationAction::Legal : LegalizationAction::Unsupported; } default: break; } // Keep the unsupported policy mechanically synchronized with the complete // target-independent opcode inventory. Supported cases return above. switch (Generic) { #define RCC_CORE_OPCODE(Name) #define RCC_GENERIC_OPCODE(Name) case GenericOpcode::Name: #include "rcc/CodeGen/GenericOpcode.def" #undef RCC_GENERIC_OPCODE #undef RCC_CORE_OPCODE return LegalizationAction::Unsupported; case GenericOpcode::Invalid: case GenericOpcode::CORE_PSEUDO_END: case GenericOpcode::G_GENERIC_END: default: break; } return makeUnexpected(ErrorCode::InternalError, "RISC-V legalizer received an invalid generic opcode value"); } [[nodiscard]] Expected lower(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target, LegalizationAction Action) const override { if (Instruction.opcode() == opcode(GenericOpcode::G_MUL) && !Features.has(RISCVExtension::M)) { if (Action == LegalizationAction::Custom) return widenNarrowMultiply(Instruction, Function, Target); if (Action == LegalizationAction::Libcall) return lowerMultiplyLibcall(Instruction, Function, Target); } if (Action == LegalizationAction::Custom && resultType(Function, Instruction) .transform([](LLT Type) { return Type.scalarBitWidth() > 64; }) .value_or(false)) { const Opcode Value = Instruction.opcode(); if (Value == opcode(GenericOpcode::G_ADD) || Value == opcode(GenericOpcode::G_SUB) || Value == opcode(GenericOpcode::G_AND) || Value == opcode(GenericOpcode::G_OR) || Value == opcode(GenericOpcode::G_XOR)) { return lowerWideBinary(Instruction, Function, Target, static_cast(Value.value())); } if (Value == opcode(GenericOpcode::G_SHL)) return lowerWideShiftLeft(Instruction, Function, Target); if (Value == opcode(GenericOpcode::G_PHI)) return lowerWidePhi(Instruction, Function, Target); } if (Action == LegalizationAction::Custom && Instruction.opcode() == opcode(GenericOpcode::G_MERGE_VALUES)) return lowerMergeValues(Instruction, Function, Target); if (Action == LegalizationAction::Custom && Instruction.opcode() == opcode(GenericOpcode::G_UNMERGE_VALUES)) return lowerUnmergeValues(Instruction, Function, Target); if (Action == LegalizationAction::Custom && Instruction.opcode() == opcode(GenericOpcode::G_BRCOND)) return lowerNarrowBranchCondition(Instruction, Function, Target); if (Action == LegalizationAction::Custom && Instruction.opcode() == opcode(GenericOpcode::G_PHI)) return lowerNarrowPhi(Instruction, Function, Target); if (Action == LegalizationAction::Custom && Instruction.opcode() == opcode(GenericOpcode::G_ICMP)) return lowerIntegerCompare(Instruction, Function, Target); if (Action == LegalizationAction::Custom && Instruction.opcode() == opcode(GenericOpcode::G_ATOMIC_CMPXCHG)) return lowerAtomicCompareExchange(Instruction, Function, Target); if (Action == LegalizationAction::Custom && Instruction.opcode() == opcode(GenericOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS)) return lowerAtomicCompareExchangeWithSuccess(Instruction, Function, Target); if (Action == LegalizationAction::Custom && (Instruction.opcode() == opcode(GenericOpcode::G_ANYEXT) || Instruction.opcode() == opcode(GenericOpcode::G_SEXT) || Instruction.opcode() == opcode(GenericOpcode::G_ZEXT))) { if (Instruction.opcode() == opcode(GenericOpcode::G_ANYEXT) && narrowAnyExtBinaryPlan(Instruction, Function).has_value()) return foldNarrowAnyExtBinary(Instruction, Function, Target); return foldConstantExtension( Instruction, Function, static_cast(Instruction.opcode().value())); } if (Action == LegalizationAction::Custom && Instruction.opcode() == opcode(GenericOpcode::G_CONSTANT)) { const MachineOperand *Immediate = Instruction.operandAt(1); const std::optional Type = resultType(Function, Instruction); if (Immediate != nullptr && Type && (Type->isInteger() || Type->isAnyScalar()) && Type->scalarBitWidth() == 64) { auto Value = signedImmediate(*Immediate); if (Value && shouldUseRV64IntegerConstantPool(*Value)) return lowerConstantToPool(Instruction, Function, Target); } return foldWideConstant(Instruction, Function, Target); } return makeUnexpected(ErrorCode::Unsupported, "RISC-V legalizer has no lowering rule for this instruction"); } private: RISCVSubtargetFeatures Features; }; class RISCVRegisterBankInfo final : public TargetRegisterBankInfo { public: [[nodiscard]] Expected selectBank(const MachineFunction &, Register, const VirtualRegisterInfo &Info, const MachineInstr *, const TargetDescription &) const override { if (!Info.Type) { return makeUnexpected(ErrorCode::VerificationError, "RISC-V register-bank selection requires a virtual-register type"); } if (isScalarGPRType(*Info.Type)) { return RegisterBankID::fromValue(static_cast(RegisterBank::GPR)); } if (isRVFloatType(*Info.Type)) return RegisterBankID::fromValue(static_cast(RegisterBank::FPR)); return makeUnexpected(ErrorCode::Unsupported, "RISC-V RV64GC has no register bank for virtual register type '" + Info.Type->toString() + "'"); } [[nodiscard]] Expected constrainInstruction(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target) const override { std::vector> PlannedClasses; for (const MachineOperand &Operand : Instruction.operands()) { const Register *Reg = Operand.getIf(); if (Reg == nullptr || !Reg->isVirtual()) continue; auto Required = requiredRegisterClass(Function, *Reg, Target); if (!Required) return std::unexpected(Required.error()); if (std::ranges::find_if(PlannedClasses, [Reg](const auto &Entry) { return Entry.first == *Reg; }) == PlannedClasses.end()) PlannedClasses.emplace_back(*Reg, *Required); } for (const auto &[Reg, Required] : PlannedClasses) { auto Info = Function.registerInfo().virtualRegisterInfo(Reg); if (!Info) return std::unexpected(Info.error()); if (Info->get().RegClass) continue; auto SetClass = Function.registerInfo().setRegisterClass(Reg, Required); if (!SetClass) return std::unexpected(SetClass.error()); } return {}; } [[nodiscard]] Expected validateInstruction(const MachineInstr &Instruction, const MachineFunction &Function, const TargetDescription &Target) const override { for (const MachineOperand &Operand : Instruction.operands()) { const Register *Reg = Operand.getIf(); if (Reg == nullptr || !Reg->isVirtual()) continue; auto Required = requiredRegisterClass(Function, *Reg, Target); if (!Required) return std::unexpected(Required.error()); } return {}; } }; class RISCVInstructionSelector final : public TargetInstructionSelector { public: [[nodiscard]] Expected constrainSelectedInstruction(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target) const override { return registerBankInfo().constrainInstruction(Instruction, Function, Target); } [[nodiscard]] Expected canSelect(const MachineInstr &Instruction, const MachineFunction &Function, const TargetDescription &Target) const override { if (Instruction.opcode() == targetOpcode(InstructionOpcode::G_SLLW)) { const std::optional Type = resultType(Function, Instruction); const std::optional Left = operandType(Function, Instruction, 1); const std::optional Right = operandType(Function, Instruction, 2); return Type && Left && Right && isScalarGPRType(*Type) && Type->scalarBitWidth() == 64 && *Left == *Type && *Right == *Type ? Expected{} : makeUnexpected(ErrorCode::Unsupported, "RISC-V G_SLLW requires three 64-bit GPR operands"); } if (!isGenericOpcode(Instruction.opcode())) return {}; const GenericOpcode Generic = static_cast(Instruction.opcode().value()); const std::optional Type = resultType(Function, Instruction); const auto Integer = [Type] { return Type && isSupportedIntegerType(*Type); }; const auto Float = [Type] { return Type && isRVFloatType(*Type); }; auto Classification = legalizerInfo().classify(Instruction, Function, Target); if (!Classification) return std::unexpected(Classification.error()); if (*Classification != LegalizationAction::Legal) { std::string Message = "RISC-V selector cannot select opcode '" + std::string(getGenericOpcodeName(Generic)) + "'"; if (Type) Message += " with result type '" + Type->toString() + "'"; Message += ": the legalization rule explicitly marks this combination unsupported"; return makeUnexpected(ErrorCode::Unsupported, std::move(Message)); } switch (Generic) { case GenericOpcode::G_ADD: case GenericOpcode::G_SUB: case GenericOpcode::G_MUL: case GenericOpcode::G_SDIV: case GenericOpcode::G_UDIV: case GenericOpcode::G_SREM: case GenericOpcode::G_UREM: case GenericOpcode::G_AND: case GenericOpcode::G_OR: case GenericOpcode::G_XOR: case GenericOpcode::G_SHL: case GenericOpcode::G_LSHR: case GenericOpcode::G_ASHR: case GenericOpcode::G_PTR_ADD: case GenericOpcode::G_PTRMASK: case GenericOpcode::G_CONSTANT: if (!Integer()) return makeUnexpected(ErrorCode::Unsupported, "RISC-V selector requires an i1-i64 integer or 64-bit pointer value"); if (Generic == GenericOpcode::G_CONSTANT) { const MachineOperand *Immediate = Instruction.operandAt(1); if (Immediate == nullptr || (Immediate->kind() != MachineOperand::Kind::Immediate && Immediate->kind() != MachineOperand::Kind::IntegerImmediate)) return makeUnexpected(ErrorCode::VerificationError, "RISC-V integer constant requires an immediate operand"); } return {}; case GenericOpcode::G_LOAD: case GenericOpcode::G_SEXTLOAD: case GenericOpcode::G_ZEXTLOAD: case GenericOpcode::G_FPEXTLOAD: case GenericOpcode::G_FPTRUNCSTORE: case GenericOpcode::G_STORE: case GenericOpcode::G_BR: case GenericOpcode::G_BRCOND: return {}; case GenericOpcode::G_FADD: case GenericOpcode::G_FSUB: case GenericOpcode::G_FMUL: case GenericOpcode::G_FDIV: case GenericOpcode::G_FMA: case GenericOpcode::G_FSQRT: case GenericOpcode::G_FNEG: case GenericOpcode::G_FABS: case GenericOpcode::G_FCOPYSIGN: if (!Float()) return makeUnexpected(ErrorCode::Unsupported, "RISC-V selector requires f32 or f64 floating values"); return {}; case GenericOpcode::G_FCONSTANT: { const MachineOperand *Immediate = Instruction.operandAt(1); const auto *Floating = Immediate == nullptr ? nullptr : Immediate->getIf(); const std::optional Semantics = Type ? Type->floatSemantics() : std::nullopt; if (!Float() || Floating == nullptr || !Semantics || Floating->Bits.semantics() != *Semantics) return makeUnexpected(ErrorCode::Unsupported, "RISC-V selector requires a matching floating constant"); return {}; } case GenericOpcode::G_ANYEXT: case GenericOpcode::G_ASSERT_SEXT: case GenericOpcode::G_ASSERT_ZEXT: case GenericOpcode::G_ASSERT_ALIGN: case GenericOpcode::G_FREEZE: case GenericOpcode::G_CONSTANT_FOLD_BARRIER: case GenericOpcode::G_PTRTOINT: case GenericOpcode::G_INTTOPTR: case GenericOpcode::G_BITCAST: case GenericOpcode::G_TRUNC: case GenericOpcode::G_SEXT: case GenericOpcode::G_SEXT_INREG: case GenericOpcode::G_ZEXT: case GenericOpcode::G_ICMP: case GenericOpcode::G_FCMP: case GenericOpcode::G_FPEXT: case GenericOpcode::G_FPTRUNC: case GenericOpcode::G_FPTOSI: case GenericOpcode::G_FPTOUI: case GenericOpcode::G_SITOFP: case GenericOpcode::G_UITOFP: case GenericOpcode::G_FMINNUM: case GenericOpcode::G_FMAXNUM: case GenericOpcode::G_SDIVREM: case GenericOpcode::G_UDIVREM: case GenericOpcode::G_SMULH: case GenericOpcode::G_UMULH: case GenericOpcode::G_SMIN: case GenericOpcode::G_SMAX: case GenericOpcode::G_UMIN: case GenericOpcode::G_UMAX: case GenericOpcode::G_SELECT: case GenericOpcode::G_ABS: case GenericOpcode::G_UADDO: case GenericOpcode::G_USUBO: case GenericOpcode::G_SADDO: case GenericOpcode::G_SSUBO: case GenericOpcode::G_UMULO: case GenericOpcode::G_SMULO: case GenericOpcode::G_UADDE: case GenericOpcode::G_USUBE: case GenericOpcode::G_SADDE: case GenericOpcode::G_SSUBE: case GenericOpcode::G_ATOMICRMW_XCHG: case GenericOpcode::G_ATOMICRMW_ADD: case GenericOpcode::G_ATOMICRMW_SUB: case GenericOpcode::G_ATOMICRMW_AND: case GenericOpcode::G_ATOMICRMW_OR: case GenericOpcode::G_ATOMICRMW_XOR: case GenericOpcode::G_ATOMICRMW_MAX: case GenericOpcode::G_ATOMICRMW_MIN: case GenericOpcode::G_ATOMICRMW_UMAX: case GenericOpcode::G_ATOMICRMW_UMIN: case GenericOpcode::G_FENCE: case GenericOpcode::G_BRINDIRECT: case GenericOpcode::G_TRAP: case GenericOpcode::G_DEBUGTRAP: case GenericOpcode::G_UBSANTRAP: case GenericOpcode::G_PHI: case GenericOpcode::G_IMPLICIT_DEF: return {}; default: return makeUnexpected(ErrorCode::Unsupported, "RISC-V instruction selector has no rule for generic opcode"); } } [[nodiscard]] Expected select(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target) const override { auto Match = canSelect(Instruction, Function, Target); if (!Match) return std::unexpected(Match.error()); if (Instruction.opcode() == targetOpcode(InstructionOpcode::G_SLLW)) return Instruction.setOpcode(targetOpcode(InstructionOpcode::SLLW)); const GenericOpcode Generic = static_cast(Instruction.opcode().value()); const std::optional Type = resultType(Function, Instruction); const auto SelectDirect = [&Instruction](InstructionOpcode Selected) { return Instruction.setOpcode(targetOpcode(Selected)); }; switch (Generic) { case GenericOpcode::G_ADD: return selectIntegerBinary(Instruction, Type, InstructionOpcode::ADD, InstructionOpcode::ADDW); case GenericOpcode::G_SUB: return selectIntegerBinary(Instruction, Type, InstructionOpcode::SUB, InstructionOpcode::SUBW); case GenericOpcode::G_MUL: return selectIntegerBinary(Instruction, Type, InstructionOpcode::MUL, InstructionOpcode::MULW); case GenericOpcode::G_SDIV: return selectNormalizedIntegerBinary(Instruction, Function, Target, Type, InstructionOpcode::DIV, InstructionOpcode::DIVW, true, true); case GenericOpcode::G_UDIV: return selectNormalizedIntegerBinary(Instruction, Function, Target, Type, InstructionOpcode::DIVU, InstructionOpcode::DIVUW, false, true); case GenericOpcode::G_SREM: return selectNormalizedIntegerBinary(Instruction, Function, Target, Type, InstructionOpcode::REM, InstructionOpcode::REMW, true, true); case GenericOpcode::G_UREM: return selectNormalizedIntegerBinary(Instruction, Function, Target, Type, InstructionOpcode::REMU, InstructionOpcode::REMUW, false, true); case GenericOpcode::G_SDIVREM: return selectDivRem(Instruction, Function, Target, Type, true); case GenericOpcode::G_UDIVREM: return selectDivRem(Instruction, Function, Target, Type, false); case GenericOpcode::G_AND: return SelectDirect(InstructionOpcode::AND); case GenericOpcode::G_OR: return SelectDirect(InstructionOpcode::OR); case GenericOpcode::G_XOR: return SelectDirect(InstructionOpcode::XOR); case GenericOpcode::G_SHL: return selectIntegerBinary(Instruction, Type, InstructionOpcode::SLL, InstructionOpcode::SLLW); case GenericOpcode::G_LSHR: return selectNormalizedIntegerBinary(Instruction, Function, Target, Type, InstructionOpcode::SRL, InstructionOpcode::SRLW, false, false); case GenericOpcode::G_ASHR: return selectNormalizedIntegerBinary(Instruction, Function, Target, Type, InstructionOpcode::SRA, InstructionOpcode::SRAW, true, false); case GenericOpcode::G_PTR_ADD: return SelectDirect(InstructionOpcode::ADD); case GenericOpcode::G_PTRMASK: return SelectDirect(InstructionOpcode::AND); case GenericOpcode::G_LOAD: return selectLoad(Instruction, Type); case GenericOpcode::G_SEXTLOAD: return selectExtLoad(Instruction, Type, true); case GenericOpcode::G_ZEXTLOAD: return selectExtLoad(Instruction, Type, false); case GenericOpcode::G_FPEXTLOAD: return selectFloatExtLoad(Instruction, Function, Target); case GenericOpcode::G_STORE: return selectStore(Instruction, Function); case GenericOpcode::G_FPTRUNCSTORE: return selectFloatTruncStore(Instruction, Function, Target); case GenericOpcode::G_BR: return selectBranch(Instruction); case GenericOpcode::G_BRCOND: return selectConditionalBranch(Instruction); case GenericOpcode::G_CONSTANT: return selectIntegerConstant(Instruction, Function, Target, Type); case GenericOpcode::G_FCONSTANT: return selectFloatingConstant(Instruction, Function, Target, Type); case GenericOpcode::G_ASSERT_SEXT: case GenericOpcode::G_ASSERT_ZEXT: case GenericOpcode::G_ASSERT_ALIGN: if (Instruction.operandCount() == 3) { auto EraseImmediate = Instruction.eraseOperand(2); if (!EraseImmediate) return std::unexpected(EraseImmediate.error()); } return Instruction.setOpcode(opcode(GenericOpcode::COPY)); case GenericOpcode::G_FREEZE: case GenericOpcode::G_CONSTANT_FOLD_BARRIER: case GenericOpcode::G_PTRTOINT: case GenericOpcode::G_INTTOPTR: return Instruction.setOpcode(opcode(GenericOpcode::COPY)); case GenericOpcode::G_BITCAST: return selectBitcast(Instruction, Function); case GenericOpcode::G_FADD: return selectRoundingBinary(Instruction, Type, InstructionOpcode::FADD_S, InstructionOpcode::FADD_D); case GenericOpcode::G_FSUB: return selectRoundingBinary(Instruction, Type, InstructionOpcode::FSUB_S, InstructionOpcode::FSUB_D); case GenericOpcode::G_FMUL: return selectRoundingBinary(Instruction, Type, InstructionOpcode::FMUL_S, InstructionOpcode::FMUL_D); case GenericOpcode::G_FDIV: return selectRoundingBinary(Instruction, Type, InstructionOpcode::FDIV_S, InstructionOpcode::FDIV_D); case GenericOpcode::G_FMA: return selectRoundingTernary(Instruction, Type, InstructionOpcode::FMADD_S, InstructionOpcode::FMADD_D); case GenericOpcode::G_FSQRT: return selectRoundingUnary(Instruction, Type, InstructionOpcode::FSQRT_S, InstructionOpcode::FSQRT_D); case GenericOpcode::G_FNEG: return selectSignInjection(Instruction, Type, InstructionOpcode::FSGNJN_S, InstructionOpcode::FSGNJN_D); case GenericOpcode::G_FABS: return selectSignInjection(Instruction, Type, InstructionOpcode::FSGNJX_S, InstructionOpcode::FSGNJX_D); case GenericOpcode::G_FCOPYSIGN: return selectFloatingMinMax(Instruction, Type, InstructionOpcode::FSGNJ_S, InstructionOpcode::FSGNJ_D); case GenericOpcode::G_ANYEXT: case GenericOpcode::G_SEXT: case GenericOpcode::G_ZEXT: return selectIntegerExtension(Instruction, Function, Target, Generic, Type); case GenericOpcode::G_TRUNC: return Instruction.setOpcode(opcode(GenericOpcode::COPY)); case GenericOpcode::G_SEXT_INREG: return selectSignExtendInRegister(Instruction, Function, Target, Type); case GenericOpcode::G_ICMP: return selectIntegerCompare(Instruction, Function, Target); case GenericOpcode::G_FCMP: return selectFloatingCompare(Instruction, Function, Target); case GenericOpcode::G_FPEXT: return selectRoundingUnary(Instruction, Type, InstructionOpcode::FCVT_D_S, InstructionOpcode::FCVT_D_S); case GenericOpcode::G_FPTRUNC: return selectRoundingUnary(Instruction, Type, InstructionOpcode::FCVT_S_D, InstructionOpcode::FCVT_S_D); case GenericOpcode::G_FPTOSI: case GenericOpcode::G_FPTOUI: return selectFloatToInteger(Instruction, Function, Type, Generic == GenericOpcode::G_FPTOUI); case GenericOpcode::G_SITOFP: case GenericOpcode::G_UITOFP: return selectIntegerToFloat(Instruction, Function, Target, Type, Generic == GenericOpcode::G_UITOFP); case GenericOpcode::G_FMINNUM: return selectFloatingMinMax(Instruction, Type, InstructionOpcode::FMIN_S, InstructionOpcode::FMIN_D); case GenericOpcode::G_FMAXNUM: return selectFloatingMinMax(Instruction, Type, InstructionOpcode::FMAX_S, InstructionOpcode::FMAX_D); case GenericOpcode::G_SMULH: return selectMultiplyHigh(Instruction, Function, Target, Type, true); case GenericOpcode::G_UMULH: return selectMultiplyHigh(Instruction, Function, Target, Type, false); case GenericOpcode::G_SMIN: case GenericOpcode::G_SMAX: case GenericOpcode::G_UMIN: case GenericOpcode::G_UMAX: return selectIntegerMinMax(Instruction, Function, Target, Type, Generic); case GenericOpcode::G_SELECT: return selectValue(Instruction, Function, Target, Type); case GenericOpcode::G_ABS: return selectAbsoluteValue(Instruction, Function, Target, Type); case GenericOpcode::G_UADDO: case GenericOpcode::G_USUBO: case GenericOpcode::G_SADDO: case GenericOpcode::G_SSUBO: case GenericOpcode::G_UMULO: case GenericOpcode::G_SMULO: return selectOverflow(Instruction, Function, Target, Type, Generic); case GenericOpcode::G_UADDE: case GenericOpcode::G_USUBE: case GenericOpcode::G_SADDE: case GenericOpcode::G_SSUBE: return selectExtendedOverflow(Instruction, Function, Target, Type, Generic); case GenericOpcode::G_ATOMICRMW_XCHG: case GenericOpcode::G_ATOMICRMW_ADD: case GenericOpcode::G_ATOMICRMW_SUB: case GenericOpcode::G_ATOMICRMW_AND: case GenericOpcode::G_ATOMICRMW_OR: case GenericOpcode::G_ATOMICRMW_XOR: case GenericOpcode::G_ATOMICRMW_MAX: case GenericOpcode::G_ATOMICRMW_MIN: case GenericOpcode::G_ATOMICRMW_UMAX: case GenericOpcode::G_ATOMICRMW_UMIN: return selectAtomicRMW(Instruction, Function, Target, Type, Generic); case GenericOpcode::G_FENCE: return selectFence(Instruction); case GenericOpcode::G_BRINDIRECT: return selectIndirectBranch(Instruction); case GenericOpcode::G_TRAP: case GenericOpcode::G_DEBUGTRAP: case GenericOpcode::G_UBSANTRAP: return selectTrap(Instruction); case GenericOpcode::G_PHI: return Instruction.setOpcode(opcode(GenericOpcode::PHI)); case GenericOpcode::G_IMPLICIT_DEF: return Instruction.setOpcode(opcode(GenericOpcode::IMPLICIT_DEF)); case GenericOpcode::COPY: case GenericOpcode::IMPLICIT_DEF: case GenericOpcode::PHI: case GenericOpcode::REG_SEQUENCE: case GenericOpcode::INSERT_SUBREG: case GenericOpcode::EXTRACT_SUBREG: case GenericOpcode::SUBREG_TO_REG: case GenericOpcode::KILL: case GenericOpcode::BUNDLE: case GenericOpcode::INLINEASM: case GenericOpcode::CORE_PSEUDO_END: break; default: return makeUnexpected(ErrorCode::Unsupported, "RISC-V instruction selector has no rule for generic opcode"); } return makeUnexpected(ErrorCode::Unsupported, "RISC-V instruction selector has no rule for generic opcode"); } private: [[nodiscard]] static Expected replaceImmediate(MachineInstr &Instruction, std::size_t Index, std::int64_t Value) { if (Instruction.operandAt(Index) == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V instruction is missing an immediate operand"); auto Erased = Instruction.eraseOperand(Index); if (!Erased) return std::unexpected(Erased.error()); return Instruction.insertOperand(Index, MachineOperand::immediate(Value)); } [[nodiscard]] static Expected buildBinaryAfter(MachineIRBuilder &Builder, MachineInstr &Anchor, InstructionOpcode Opcode, Register Def, Register Left, Register Right) { auto Point = Builder.setInsertPointAfter(Anchor); if (!Point) return std::unexpected(Point.error()); auto Pending = Builder.buildInstr(targetOpcode(Opcode)); if (!Pending) return std::unexpected(Pending.error()); auto DefResult = Pending->addDef(Def); if (!DefResult) return std::unexpected(DefResult.error()); auto LeftResult = Pending->addUse(Left); if (!LeftResult) return std::unexpected(LeftResult.error()); auto RightResult = Pending->addUse(Right); if (!RightResult) return std::unexpected(RightResult.error()); auto Built = Pending->build(); if (!Built) return std::unexpected(Built.error()); return *Built; } [[nodiscard]] static Expected buildImmediateAfter(MachineIRBuilder &Builder, MachineInstr &Anchor, InstructionOpcode Opcode, Register Def, Register Source, std::int64_t Value) { auto Point = Builder.setInsertPointAfter(Anchor); if (!Point) return std::unexpected(Point.error()); auto Pending = Builder.buildInstr(targetOpcode(Opcode)); if (!Pending) return std::unexpected(Pending.error()); auto DefResult = Pending->addDef(Def); if (!DefResult) return std::unexpected(DefResult.error()); auto SourceResult = Pending->addUse(Source); if (!SourceResult) return std::unexpected(SourceResult.error()); auto ImmediateResult = Pending->addImmediate(Value); if (!ImmediateResult) return std::unexpected(ImmediateResult.error()); auto Built = Pending->build(); if (!Built) return std::unexpected(Built.error()); return *Built; } [[nodiscard]] static Expected buildImmediateBefore(MachineIRBuilder &Builder, MachineInstr &Anchor, InstructionOpcode Opcode, Register Def, Register Source, std::int64_t Value) { auto Point = Builder.setInsertPointBefore(Anchor); if (!Point) return std::unexpected(Point.error()); auto Pending = Builder.buildInstr(targetOpcode(Opcode)); if (!Pending) return std::unexpected(Pending.error()); auto DefResult = Pending->addDef(Def); if (!DefResult) return std::unexpected(DefResult.error()); auto SourceResult = Pending->addUse(Source); if (!SourceResult) return std::unexpected(SourceResult.error()); auto ImmediateResult = Pending->addImmediate(Value); if (!ImmediateResult) return std::unexpected(ImmediateResult.error()); auto Built = Pending->build(); if (!Built) return std::unexpected(Built.error()); return *Built; } [[nodiscard]] static Expected normalizeIntegerBefore(MachineInstr &Anchor, MachineFunction &Function, const TargetDescription &Target, Register Source, LLT Type, bool Signed) { if (!isSupportedIntegerScalar(Type)) return makeUnexpected(ErrorCode::Unsupported, "RISC-V integer normalization requires a scalar i1-i64 type"); const std::uint32_t Bits = Type.scalarBitWidth(); if (Bits == 64) return Source; MachineIRBuilder Builder(Function, Target); if (Signed && Bits == 32) { auto Result = createTemporary(Function, Type, Target); if (!Result) return std::unexpected(Result.error()); auto Extended = buildImmediateBefore(Builder, Anchor, InstructionOpcode::ADDIW, *Result, Source, 0); if (!Extended) return std::unexpected(Extended.error()); return *Result; } const std::uint32_t ContainerBits = Bits < 32 ? 32 : 64; const std::int64_t Shift = static_cast(ContainerBits - Bits); auto Shifted = createTemporary(Function, Type, Target); auto Result = createTemporary(Function, Type, Target); if (!Shifted || !Result) return std::unexpected(!Shifted ? Shifted.error() : Result.error()); const InstructionOpcode ShiftLeft = ContainerBits == 32 ? InstructionOpcode::SLLIW : InstructionOpcode::SLLI; const InstructionOpcode ShiftRight = Signed ? (ContainerBits == 32 ? InstructionOpcode::SRAIW : InstructionOpcode::SRAI) : (ContainerBits == 32 ? InstructionOpcode::SRLIW : InstructionOpcode::SRLI); auto Left = buildImmediateBefore(Builder, Anchor, ShiftLeft, *Shifted, Source, Shift); if (!Left) return std::unexpected(Left.error()); auto Right = buildImmediateBefore(Builder, Anchor, ShiftRight, *Result, *Shifted, Shift); if (!Right) return std::unexpected(Right.error()); return *Result; } [[nodiscard]] static Expected buildUnaryAfter(MachineIRBuilder &Builder, MachineInstr &Anchor, InstructionOpcode Opcode, Register Def, Register Source) { auto Point = Builder.setInsertPointAfter(Anchor); if (!Point) return std::unexpected(Point.error()); auto Pending = Builder.buildInstr(targetOpcode(Opcode)); if (!Pending) return std::unexpected(Pending.error()); auto DefResult = Pending->addDef(Def); if (!DefResult) return std::unexpected(DefResult.error()); auto SourceResult = Pending->addUse(Source); if (!SourceResult) return std::unexpected(SourceResult.error()); auto Built = Pending->build(); if (!Built) return std::unexpected(Built.error()); return *Built; } [[nodiscard]] static Expected buildGPRSelectAfter(MachineIRBuilder &Builder, MachineInstr &Anchor, MachineFunction &Function, const TargetDescription &Target, LLT ValueType, Register Destination, Register Condition, Register TrueValue, Register FalseValue) { auto IntegerType = LLT::makeInteger(ValueType.scalarBitWidth()); if (!IntegerType) return std::unexpected(IntegerType.error()); auto Mask = createTemporary(Function, *IntegerType, Target); auto Difference = createTemporary(Function, *IntegerType, Target); auto MaskedDifference = createTemporary(Function, *IntegerType, Target); if (!Mask || !Difference || !MaskedDifference) { return std::unexpected( (!Mask ? Mask.error() : (!Difference ? Difference.error() : MaskedDifference.error()))); } auto Zero = physicalRegister(PhysicalRegisterID::X0); if (!Zero) return std::unexpected(Zero.error()); const InstructionOpcode Subtract = ValueType.scalarBitWidth() == 32 ? InstructionOpcode::SUBW : InstructionOpcode::SUB; auto MaskInstruction = buildBinaryAfter(Builder, Anchor, Subtract, *Mask, Register::fromPhysical(*Zero), Condition); if (!MaskInstruction) return std::unexpected(MaskInstruction.error()); auto DifferenceInstruction = buildBinaryAfter( Builder, **MaskInstruction, InstructionOpcode::XOR, *Difference, TrueValue, FalseValue); if (!DifferenceInstruction) return std::unexpected(DifferenceInstruction.error()); auto MaskInstructionResult = buildBinaryAfter(Builder, **DifferenceInstruction, InstructionOpcode::AND, *MaskedDifference, *Difference, *Mask); if (!MaskInstructionResult) return std::unexpected(MaskInstructionResult.error()); return buildBinaryAfter(Builder, **MaskInstructionResult, InstructionOpcode::XOR, Destination, FalseValue, *MaskedDifference); } [[nodiscard]] static Expected selectDivRem(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target, std::optional Type, bool Signed) { const MachineOperand *RemainderOperand = Instruction.operandAt(1); const MachineOperand *LeftOperand = Instruction.operandAt(2); const MachineOperand *RightOperand = Instruction.operandAt(3); const Register *Remainder = RemainderOperand == nullptr ? nullptr : RemainderOperand->getIf(); const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf(); const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf(); if (!Type || Remainder == nullptr || Left == nullptr || Right == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V division-with-remainder has invalid operands"); const Register RemainderRegister = *Remainder; auto LeftRegister = normalizeIntegerBefore(Instruction, Function, Target, *Left, *Type, Signed); auto RightRegister = normalizeIntegerBefore(Instruction, Function, Target, *Right, *Type, Signed); if (!LeftRegister || !RightRegister) return std::unexpected(!LeftRegister ? LeftRegister.error() : RightRegister.error()); auto RemoveRemainder = Instruction.eraseOperand(1); if (!RemoveRemainder) return std::unexpected(RemoveRemainder.error()); auto SetLeft = Instruction.operandAt(1)->setRegister(*LeftRegister); if (!SetLeft) return std::unexpected(SetLeft.error()); auto SetRight = Instruction.operandAt(2)->setRegister(*RightRegister); if (!SetRight) return std::unexpected(SetRight.error()); const bool IsWord = Type->scalarBitWidth() <= 32; const InstructionOpcode Division = IsWord ? (Signed ? InstructionOpcode::DIVW : InstructionOpcode::DIVUW) : (Signed ? InstructionOpcode::DIV : InstructionOpcode::DIVU); const InstructionOpcode RemainderOpcode = IsWord ? (Signed ? InstructionOpcode::REMW : InstructionOpcode::REMUW) : (Signed ? InstructionOpcode::REM : InstructionOpcode::REMU); auto SetDivision = Instruction.setOpcode(targetOpcode(Division)); if (!SetDivision) return std::unexpected(SetDivision.error()); MachineIRBuilder Builder(Function, Target); auto BuiltRemainder = buildBinaryAfter(Builder, Instruction, RemainderOpcode, RemainderRegister, *LeftRegister, *RightRegister); if (!BuiltRemainder) return std::unexpected(BuiltRemainder.error()); return {}; } [[nodiscard]] static Expected selectMultiplyHigh(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target, std::optional Type, bool Signed) { if (!Type) return makeUnexpected(ErrorCode::VerificationError, "RISC-V high multiplication has no result type"); if (Type->scalarBitWidth() == 64) { return Instruction.setOpcode( targetOpcode(Signed ? InstructionOpcode::MULH : InstructionOpcode::MULHU)); } const MachineOperand *DestinationOperand = Instruction.operandAt(0); const MachineOperand *LeftOperand = Instruction.operandAt(1); const MachineOperand *RightOperand = Instruction.operandAt(2); const Register *Destination = DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf(); const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf(); if (Destination == nullptr || Left == nullptr || Right == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V high multiplication has invalid register operands"); const Register DestinationRegister = *Destination; const Register RightRegister = *Right; auto I64 = LLT::makeInteger(64); if (!I64) return std::unexpected(I64.error()); auto ExtendedLeft = createTemporary(Function, *I64, Target); auto ExtendedRight = createTemporary(Function, *I64, Target); auto Product = createTemporary(Function, *I64, Target); if (!ExtendedLeft || !ExtendedRight || !Product) { return std::unexpected(!ExtendedLeft ? ExtendedLeft.error() : (!ExtendedRight ? ExtendedRight.error() : Product.error())); } auto RemoveRight = Instruction.eraseOperand(2); if (!RemoveRight) return std::unexpected(RemoveRight.error()); MachineIRBuilder Builder(Function, Target); MachineInstr *Anchor = &Instruction; if (Signed) { auto SetDestination = Instruction.operandAt(0)->setRegister(*ExtendedLeft); if (!SetDestination) return std::unexpected(SetDestination.error()); auto Immediate = appendImmediate(Instruction, 0); if (!Immediate) return std::unexpected(Immediate.error()); auto SetOpcode = Instruction.setOpcode(targetOpcode(InstructionOpcode::ADDIW)); if (!SetOpcode) return std::unexpected(SetOpcode.error()); auto ExtendRight = buildImmediateAfter(Builder, *Anchor, InstructionOpcode::ADDIW, *ExtendedRight, RightRegister, 0); if (!ExtendRight) return std::unexpected(ExtendRight.error()); Anchor = *ExtendRight; } else { auto ShiftedLeft = createTemporary(Function, *I64, Target); auto ShiftedRight = createTemporary(Function, *I64, Target); if (!ShiftedLeft || !ShiftedRight) return std::unexpected(!ShiftedLeft ? ShiftedLeft.error() : ShiftedRight.error()); auto SetDestination = Instruction.operandAt(0)->setRegister(*ShiftedLeft); if (!SetDestination) return std::unexpected(SetDestination.error()); auto Immediate = appendImmediate(Instruction, 32); if (!Immediate) return std::unexpected(Immediate.error()); auto SetOpcode = Instruction.setOpcode(targetOpcode(InstructionOpcode::SLLI)); if (!SetOpcode) return std::unexpected(SetOpcode.error()); auto ExtendLeft = buildImmediateAfter(Builder, *Anchor, InstructionOpcode::SRLI, *ExtendedLeft, *ShiftedLeft, 32); if (!ExtendLeft) return std::unexpected(ExtendLeft.error()); auto ShiftRight = buildImmediateAfter(Builder, **ExtendLeft, InstructionOpcode::SLLI, *ShiftedRight, RightRegister, 32); if (!ShiftRight) return std::unexpected(ShiftRight.error()); auto ExtendRight = buildImmediateAfter(Builder, **ShiftRight, InstructionOpcode::SRLI, *ExtendedRight, *ShiftedRight, 32); if (!ExtendRight) return std::unexpected(ExtendRight.error()); Anchor = *ExtendRight; } auto Multiply = buildBinaryAfter(Builder, *Anchor, InstructionOpcode::MUL, *Product, *ExtendedLeft, *ExtendedRight); if (!Multiply) return std::unexpected(Multiply.error()); auto Extract = buildImmediateAfter(Builder, **Multiply, Signed ? InstructionOpcode::SRAI : InstructionOpcode::SRLI, DestinationRegister, *Product, 32); if (!Extract) return std::unexpected(Extract.error()); return {}; } [[nodiscard]] static Expected selectIntegerMinMax(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target, std::optional Type, GenericOpcode Generic) { const MachineOperand *DestinationOperand = Instruction.operandAt(0); const MachineOperand *LeftOperand = Instruction.operandAt(1); const MachineOperand *RightOperand = Instruction.operandAt(2); const Register *Destination = DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf(); const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf(); if (!Type || Destination == nullptr || Left == nullptr || Right == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V integer min/max has invalid operands"); auto I1 = LLT::makeInteger(1); if (!I1) return std::unexpected(I1.error()); auto Condition = createTemporary(Function, *I1, Target); if (!Condition) return std::unexpected(Condition.error()); const Register DestinationRegister = *Destination; const Register LeftRegister = *Left; const Register RightRegister = *Right; const bool Unsigned = Generic == GenericOpcode::G_UMIN || Generic == GenericOpcode::G_UMAX; auto NormalizedLeft = normalizeIntegerBefore(Instruction, Function, Target, LeftRegister, *Type, !Unsigned); auto NormalizedRight = normalizeIntegerBefore(Instruction, Function, Target, RightRegister, *Type, !Unsigned); if (!NormalizedLeft || !NormalizedRight) return std::unexpected(!NormalizedLeft ? NormalizedLeft.error() : NormalizedRight.error()); auto SetCondition = Instruction.operandAt(0)->setRegister(*Condition); if (!SetCondition) return std::unexpected(SetCondition.error()); auto SetLeft = Instruction.operandAt(1)->setRegister(*NormalizedLeft); if (!SetLeft) return std::unexpected(SetLeft.error()); auto SetRight = Instruction.operandAt(2)->setRegister(*NormalizedRight); if (!SetRight) return std::unexpected(SetRight.error()); auto SetCompare = Instruction.setOpcode( targetOpcode(Unsigned ? InstructionOpcode::SLTU : InstructionOpcode::SLT)); if (!SetCompare) return std::unexpected(SetCompare.error()); const bool IsMaximum = Generic == GenericOpcode::G_SMAX || Generic == GenericOpcode::G_UMAX; const Register TrueValue = IsMaximum ? RightRegister : LeftRegister; const Register FalseValue = IsMaximum ? LeftRegister : RightRegister; MachineIRBuilder Builder(Function, Target); auto Selected = buildGPRSelectAfter(Builder, Instruction, Function, Target, *Type, DestinationRegister, *Condition, TrueValue, FalseValue); if (!Selected) return std::unexpected(Selected.error()); return {}; } [[nodiscard]] static Expected selectValue(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target, std::optional Type) { const MachineOperand *DestinationOperand = Instruction.operandAt(0); const MachineOperand *ConditionOperand = Instruction.operandAt(1); const MachineOperand *TrueOperand = Instruction.operandAt(2); const MachineOperand *FalseOperand = Instruction.operandAt(3); const Register *Destination = DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); const Register *Condition = ConditionOperand == nullptr ? nullptr : ConditionOperand->getIf(); const Register *TrueValue = TrueOperand == nullptr ? nullptr : TrueOperand->getIf(); const Register *FalseValue = FalseOperand == nullptr ? nullptr : FalseOperand->getIf(); if (!Type || Destination == nullptr || Condition == nullptr || TrueValue == nullptr || FalseValue == nullptr) { return makeUnexpected(ErrorCode::VerificationError, "RISC-V value selection has invalid operands"); } const Register DestinationRegister = *Destination; const Register TrueRegister = *TrueValue; const Register FalseRegister = *FalseValue; auto I1 = LLT::makeInteger(1); if (!I1) return std::unexpected(I1.error()); auto NormalizedCondition = createTemporary(Function, *I1, Target); if (!NormalizedCondition) return std::unexpected(NormalizedCondition.error()); auto SetCondition = Instruction.operandAt(0)->setRegister(*NormalizedCondition); if (!SetCondition) return std::unexpected(SetCondition.error()); auto EraseFalse = Instruction.eraseOperand(3); if (!EraseFalse) return std::unexpected(EraseFalse.error()); auto EraseTrue = Instruction.eraseOperand(2); if (!EraseTrue) return std::unexpected(EraseTrue.error()); auto Mask = appendImmediate(Instruction, 1); if (!Mask) return std::unexpected(Mask.error()); auto SetNormalize = Instruction.setOpcode(targetOpcode(InstructionOpcode::ANDI)); if (!SetNormalize) return std::unexpected(SetNormalize.error()); MachineIRBuilder Builder(Function, Target); if (!Type->isFloat()) { auto Selected = buildGPRSelectAfter(Builder, Instruction, Function, Target, *Type, DestinationRegister, *NormalizedCondition, TrueRegister, FalseRegister); if (!Selected) return std::unexpected(Selected.error()); return {}; } auto IntegerType = LLT::makeInteger(Type->scalarBitWidth()); if (!IntegerType) return std::unexpected(IntegerType.error()); auto IntegerTrue = createTemporary(Function, *IntegerType, Target); auto IntegerFalse = createTemporary(Function, *IntegerType, Target); auto IntegerResult = createTemporary(Function, *IntegerType, Target); if (!IntegerTrue || !IntegerFalse || !IntegerResult) { return std::unexpected(!IntegerTrue ? IntegerTrue.error() : (!IntegerFalse ? IntegerFalse.error() : IntegerResult.error())); } const InstructionOpcode ToInteger = Type->scalarBitWidth() == 32 ? InstructionOpcode::FMV_X_W : InstructionOpcode::FMV_X_D; const InstructionOpcode ToFloat = Type->scalarBitWidth() == 32 ? InstructionOpcode::FMV_W_X : InstructionOpcode::FMV_D_X; auto MoveTrue = buildUnaryAfter(Builder, Instruction, ToInteger, *IntegerTrue, TrueRegister); if (!MoveTrue) return std::unexpected(MoveTrue.error()); auto MoveFalse = buildUnaryAfter(Builder, **MoveTrue, ToInteger, *IntegerFalse, FalseRegister); if (!MoveFalse) return std::unexpected(MoveFalse.error()); auto Selected = buildGPRSelectAfter(Builder, **MoveFalse, Function, Target, *IntegerType, *IntegerResult, *NormalizedCondition, *IntegerTrue, *IntegerFalse); if (!Selected) return std::unexpected(Selected.error()); auto MoveResult = buildUnaryAfter(Builder, **Selected, ToFloat, DestinationRegister, *IntegerResult); if (!MoveResult) return std::unexpected(MoveResult.error()); return {}; } [[nodiscard]] static Expected selectAbsoluteValue(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target, std::optional Type) { const MachineOperand *DestinationOperand = Instruction.operandAt(0); const MachineOperand *SourceOperand = Instruction.operandAt(1); const Register *Destination = DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); if (!Type || Destination == nullptr || Source == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V absolute value has invalid operands"); const Register DestinationRegister = *Destination; const Register SourceRegister = *Source; auto NormalizedSource = normalizeIntegerBefore(Instruction, Function, Target, SourceRegister, *Type, true); if (!NormalizedSource) return std::unexpected(NormalizedSource.error()); auto SignMask = createTemporary(Function, *Type, Target); auto Flipped = createTemporary(Function, *Type, Target); if (!SignMask || !Flipped) return std::unexpected(!SignMask ? SignMask.error() : Flipped.error()); auto SetMask = Instruction.operandAt(0)->setRegister(*SignMask); if (!SetMask) return std::unexpected(SetMask.error()); auto SetSource = Instruction.operandAt(1)->setRegister(*NormalizedSource); if (!SetSource) return std::unexpected(SetSource.error()); const std::uint32_t ContainerBits = Type->scalarBitWidth() <= 32 ? 32 : 64; auto Shift = appendImmediate(Instruction, ContainerBits - 1U); if (!Shift) return std::unexpected(Shift.error()); const bool IsWord = Type->scalarBitWidth() <= 32; auto SetShift = Instruction.setOpcode( targetOpcode(IsWord ? InstructionOpcode::SRAIW : InstructionOpcode::SRAI)); if (!SetShift) return std::unexpected(SetShift.error()); MachineIRBuilder Builder(Function, Target); auto Flip = buildBinaryAfter(Builder, Instruction, InstructionOpcode::XOR, *Flipped, *NormalizedSource, *SignMask); if (!Flip) return std::unexpected(Flip.error()); auto Result = buildBinaryAfter(Builder, **Flip, IsWord ? InstructionOpcode::SUBW : InstructionOpcode::SUB, DestinationRegister, *Flipped, *SignMask); if (!Result) return std::unexpected(Result.error()); return {}; } [[nodiscard]] static Expected> buildSignedOverflowAfter(MachineIRBuilder &Builder, MachineInstr &Anchor, MachineFunction &Function, const TargetDescription &Target, LLT Type, Register Result, Register Left, Register Right, bool IsAdd) { auto FirstDifference = createTemporary(Function, Type, Target); auto SecondDifference = createTemporary(Function, Type, Target); auto SignBits = createTemporary(Function, Type, Target); auto I1 = LLT::makeInteger(1); if (!I1) return std::unexpected(I1.error()); auto Overflow = createTemporary(Function, *I1, Target); if (!FirstDifference || !SecondDifference || !SignBits || !Overflow) { return std::unexpected(!FirstDifference ? FirstDifference.error() : !SecondDifference ? SecondDifference.error() : !SignBits ? SignBits.error() : Overflow.error()); } const Register FirstLeft = IsAdd ? Result : Left; const Register FirstRight = IsAdd ? Left : Right; const Register SecondLeft = Result; const Register SecondRight = IsAdd ? Right : Left; auto First = buildBinaryAfter(Builder, Anchor, InstructionOpcode::XOR, *FirstDifference, FirstLeft, FirstRight); if (!First) return std::unexpected(First.error()); auto Second = buildBinaryAfter(Builder, **First, InstructionOpcode::XOR, *SecondDifference, SecondLeft, SecondRight); if (!Second) return std::unexpected(Second.error()); auto Both = buildBinaryAfter(Builder, **Second, InstructionOpcode::AND, *SignBits, *FirstDifference, *SecondDifference); if (!Both) return std::unexpected(Both.error()); const InstructionOpcode Shift = Type.scalarBitWidth() == 32 ? InstructionOpcode::SRLIW : InstructionOpcode::SRLI; auto Extract = buildImmediateAfter(Builder, **Both, Shift, *Overflow, *SignBits, Type.scalarBitWidth() - 1U); if (!Extract) return std::unexpected(Extract.error()); return std::pair{*Overflow, *Extract}; } [[nodiscard]] static Expected selectOverflow(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target, std::optional Type, GenericOpcode Generic) { const MachineOperand *ResultOperand = Instruction.operandAt(0); const MachineOperand *OverflowOperand = Instruction.operandAt(1); const MachineOperand *LeftOperand = Instruction.operandAt(2); const MachineOperand *RightOperand = Instruction.operandAt(3); const Register *Result = ResultOperand == nullptr ? nullptr : ResultOperand->getIf(); const Register *Overflow = OverflowOperand == nullptr ? nullptr : OverflowOperand->getIf(); const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf(); const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf(); if (!Type || Result == nullptr || Overflow == nullptr || Left == nullptr || Right == nullptr) { return makeUnexpected(ErrorCode::VerificationError, "RISC-V overflow operation has invalid operands"); } const Register ResultRegister = *Result; const Register OverflowRegister = *Overflow; const Register LeftRegister = *Left; const Register RightRegister = *Right; auto RemoveOverflow = Instruction.eraseOperand(1); if (!RemoveOverflow) return std::unexpected(RemoveOverflow.error()); const bool IsWord = Type->scalarBitWidth() <= 32; const bool IsAdd = Generic == GenericOpcode::G_UADDO || Generic == GenericOpcode::G_SADDO; const bool IsSub = Generic == GenericOpcode::G_USUBO || Generic == GenericOpcode::G_SSUBO; const bool IsUnsigned = Generic == GenericOpcode::G_UADDO || Generic == GenericOpcode::G_USUBO || Generic == GenericOpcode::G_UMULO; const InstructionOpcode Arithmetic = IsAdd ? (IsWord ? InstructionOpcode::ADDW : InstructionOpcode::ADD) : IsSub ? (IsWord ? InstructionOpcode::SUBW : InstructionOpcode::SUB) : (IsWord ? InstructionOpcode::MULW : InstructionOpcode::MUL); auto SetArithmetic = Instruction.setOpcode(targetOpcode(Arithmetic)); if (!SetArithmetic) return std::unexpected(SetArithmetic.error()); MachineIRBuilder Builder(Function, Target); if (IsUnsigned && IsAdd) { auto Carry = buildBinaryAfter(Builder, Instruction, InstructionOpcode::SLTU, OverflowRegister, ResultRegister, LeftRegister); if (!Carry) return std::unexpected(Carry.error()); return {}; } if (IsUnsigned && IsSub) { auto Borrow = buildBinaryAfter(Builder, Instruction, InstructionOpcode::SLTU, OverflowRegister, LeftRegister, RightRegister); if (!Borrow) return std::unexpected(Borrow.error()); return {}; } if (IsAdd || IsSub) { auto FirstDifference = createTemporary(Function, *Type, Target); auto SecondDifference = createTemporary(Function, *Type, Target); auto SignBits = createTemporary(Function, *Type, Target); if (!FirstDifference || !SecondDifference || !SignBits) { return std::unexpected( !FirstDifference ? FirstDifference.error() : (!SecondDifference ? SecondDifference.error() : SignBits.error())); } const Register FirstLeft = IsAdd ? ResultRegister : LeftRegister; const Register FirstRight = IsAdd ? LeftRegister : RightRegister; const Register SecondLeft = ResultRegister; const Register SecondRight = IsAdd ? RightRegister : LeftRegister; auto First = buildBinaryAfter(Builder, Instruction, InstructionOpcode::XOR, *FirstDifference, FirstLeft, FirstRight); if (!First) return std::unexpected(First.error()); auto Second = buildBinaryAfter(Builder, **First, InstructionOpcode::XOR, *SecondDifference, SecondLeft, SecondRight); if (!Second) return std::unexpected(Second.error()); auto Both = buildBinaryAfter(Builder, **Second, InstructionOpcode::AND, *SignBits, *FirstDifference, *SecondDifference); if (!Both) return std::unexpected(Both.error()); auto OverflowResult = buildImmediateAfter( Builder, **Both, IsWord ? InstructionOpcode::SRLIW : InstructionOpcode::SRLI, OverflowRegister, *SignBits, Type->scalarBitWidth() - 1U); if (!OverflowResult) return std::unexpected(OverflowResult.error()); return {}; } auto I64 = LLT::makeInteger(64); if (!I64) return std::unexpected(I64.error()); auto Zero = physicalRegister(PhysicalRegisterID::X0); if (!Zero) return std::unexpected(Zero.error()); if (!IsWord) { auto High = createTemporary(Function, *I64, Target); if (!High) return std::unexpected(High.error()); auto HighProduct = buildBinaryAfter( Builder, Instruction, IsUnsigned ? InstructionOpcode::MULHU : InstructionOpcode::MULH, *High, LeftRegister, RightRegister); if (!HighProduct) return std::unexpected(HighProduct.error()); if (IsUnsigned) { auto OverflowResult = buildBinaryAfter(Builder, **HighProduct, InstructionOpcode::SLTU, OverflowRegister, Register::fromPhysical(*Zero), *High); if (!OverflowResult) return std::unexpected(OverflowResult.error()); return {}; } auto ExpectedHigh = createTemporary(Function, *I64, Target); auto Difference = createTemporary(Function, *I64, Target); if (!ExpectedHigh || !Difference) return std::unexpected(!ExpectedHigh ? ExpectedHigh.error() : Difference.error()); auto Sign = buildImmediateAfter(Builder, **HighProduct, InstructionOpcode::SRAI, *ExpectedHigh, ResultRegister, 63); if (!Sign) return std::unexpected(Sign.error()); auto Compare = buildBinaryAfter(Builder, **Sign, InstructionOpcode::XOR, *Difference, *High, *ExpectedHigh); if (!Compare) return std::unexpected(Compare.error()); auto OverflowResult = buildBinaryAfter(Builder, **Compare, InstructionOpcode::SLTU, OverflowRegister, Register::fromPhysical(*Zero), *Difference); if (!OverflowResult) return std::unexpected(OverflowResult.error()); return {}; } auto ExtendedLeft = createTemporary(Function, *I64, Target); auto ExtendedRight = createTemporary(Function, *I64, Target); auto FullProduct = createTemporary(Function, *I64, Target); if (!ExtendedLeft || !ExtendedRight || !FullProduct) { return std::unexpected(!ExtendedLeft ? ExtendedLeft.error() : (!ExtendedRight ? ExtendedRight.error() : FullProduct.error())); } MachineInstr *Anchor = &Instruction; if (IsUnsigned) { auto ShiftedLeft = createTemporary(Function, *I64, Target); auto ShiftedRight = createTemporary(Function, *I64, Target); if (!ShiftedLeft || !ShiftedRight) return std::unexpected(!ShiftedLeft ? ShiftedLeft.error() : ShiftedRight.error()); auto LeftShift = buildImmediateAfter(Builder, *Anchor, InstructionOpcode::SLLI, *ShiftedLeft, LeftRegister, 32); if (!LeftShift) return std::unexpected(LeftShift.error()); auto LeftExtend = buildImmediateAfter(Builder, **LeftShift, InstructionOpcode::SRLI, *ExtendedLeft, *ShiftedLeft, 32); if (!LeftExtend) return std::unexpected(LeftExtend.error()); auto RightShift = buildImmediateAfter(Builder, **LeftExtend, InstructionOpcode::SLLI, *ShiftedRight, RightRegister, 32); if (!RightShift) return std::unexpected(RightShift.error()); auto RightExtend = buildImmediateAfter(Builder, **RightShift, InstructionOpcode::SRLI, *ExtendedRight, *ShiftedRight, 32); if (!RightExtend) return std::unexpected(RightExtend.error()); Anchor = *RightExtend; } else { auto LeftExtend = buildImmediateAfter(Builder, *Anchor, InstructionOpcode::ADDIW, *ExtendedLeft, LeftRegister, 0); if (!LeftExtend) return std::unexpected(LeftExtend.error()); auto RightExtend = buildImmediateAfter(Builder, **LeftExtend, InstructionOpcode::ADDIW, *ExtendedRight, RightRegister, 0); if (!RightExtend) return std::unexpected(RightExtend.error()); Anchor = *RightExtend; } auto Product = buildBinaryAfter(Builder, *Anchor, InstructionOpcode::MUL, *FullProduct, *ExtendedLeft, *ExtendedRight); if (!Product) return std::unexpected(Product.error()); if (IsUnsigned) { auto High = createTemporary(Function, *I64, Target); if (!High) return std::unexpected(High.error()); auto ExtractHigh = buildImmediateAfter(Builder, **Product, InstructionOpcode::SRLI, *High, *FullProduct, 32); if (!ExtractHigh) return std::unexpected(ExtractHigh.error()); auto OverflowResult = buildBinaryAfter(Builder, **ExtractHigh, InstructionOpcode::SLTU, OverflowRegister, Register::fromPhysical(*Zero), *High); if (!OverflowResult) return std::unexpected(OverflowResult.error()); return {}; } auto Difference = createTemporary(Function, *I64, Target); if (!Difference) return std::unexpected(Difference.error()); auto Compare = buildBinaryAfter(Builder, **Product, InstructionOpcode::XOR, *Difference, *FullProduct, ResultRegister); if (!Compare) return std::unexpected(Compare.error()); auto OverflowResult = buildBinaryAfter(Builder, **Compare, InstructionOpcode::SLTU, OverflowRegister, Register::fromPhysical(*Zero), *Difference); if (!OverflowResult) return std::unexpected(OverflowResult.error()); return {}; } [[nodiscard]] static Expected selectExtendedOverflow(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target, std::optional Type, GenericOpcode Generic) { const MachineOperand *ResultOperand = Instruction.operandAt(0); const MachineOperand *CarryOutOperand = Instruction.operandAt(1); const MachineOperand *LeftOperand = Instruction.operandAt(2); const MachineOperand *RightOperand = Instruction.operandAt(3); const MachineOperand *CarryInOperand = Instruction.operandAt(4); const Register *Result = ResultOperand == nullptr ? nullptr : ResultOperand->getIf(); const Register *CarryOut = CarryOutOperand == nullptr ? nullptr : CarryOutOperand->getIf(); const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf(); const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf(); const Register *CarryIn = CarryInOperand == nullptr ? nullptr : CarryInOperand->getIf(); if (!Type || Result == nullptr || CarryOut == nullptr || Left == nullptr || Right == nullptr || CarryIn == nullptr) { return makeUnexpected(ErrorCode::VerificationError, "RISC-V extended overflow operation has invalid operands"); } const Register ResultRegister = *Result; const Register CarryOutRegister = *CarryOut; const Register LeftRegister = *Left; const Register RightRegister = *Right; const Register CarryInRegister = *CarryIn; const bool IsAdd = Generic == GenericOpcode::G_UADDE || Generic == GenericOpcode::G_SADDE; const bool IsUnsigned = Generic == GenericOpcode::G_UADDE || Generic == GenericOpcode::G_USUBE; const bool IsWord = Type->scalarBitWidth() <= 32; const InstructionOpcode Arithmetic = IsAdd ? (IsWord ? InstructionOpcode::ADDW : InstructionOpcode::ADD) : (IsWord ? InstructionOpcode::SUBW : InstructionOpcode::SUB); auto Partial = createTemporary(Function, *Type, Target); auto I1 = LLT::makeInteger(1); if (!Partial || !I1) return std::unexpected(!Partial ? Partial.error() : I1.error()); auto NormalizedCarry = createTemporary(Function, *I1, Target); if (!NormalizedCarry) return std::unexpected(NormalizedCarry.error()); auto SetPartial = Instruction.operandAt(0)->setRegister(*Partial); if (!SetPartial) return std::unexpected(SetPartial.error()); auto RemoveCarryOut = Instruction.eraseOperand(1); if (!RemoveCarryOut) return std::unexpected(RemoveCarryOut.error()); auto RemoveCarryIn = Instruction.eraseOperand(3); if (!RemoveCarryIn) return std::unexpected(RemoveCarryIn.error()); auto SetArithmetic = Instruction.setOpcode(targetOpcode(Arithmetic)); if (!SetArithmetic) return std::unexpected(SetArithmetic.error()); MachineIRBuilder Builder(Function, Target); auto Normalize = buildImmediateAfter(Builder, Instruction, InstructionOpcode::ANDI, *NormalizedCarry, CarryInRegister, 1); if (!Normalize) return std::unexpected(Normalize.error()); auto Final = buildBinaryAfter(Builder, **Normalize, Arithmetic, ResultRegister, *Partial, *NormalizedCarry); if (!Final) return std::unexpected(Final.error()); if (IsUnsigned) { auto FirstCarry = createTemporary(Function, *I1, Target); auto SecondCarry = createTemporary(Function, *I1, Target); if (!FirstCarry || !SecondCarry) return std::unexpected(!FirstCarry ? FirstCarry.error() : SecondCarry.error()); auto First = buildBinaryAfter(Builder, **Final, InstructionOpcode::SLTU, *FirstCarry, IsAdd ? *Partial : LeftRegister, IsAdd ? LeftRegister : RightRegister); if (!First) return std::unexpected(First.error()); auto Second = buildBinaryAfter(Builder, **First, InstructionOpcode::SLTU, *SecondCarry, IsAdd ? ResultRegister : *Partial, IsAdd ? *Partial : *NormalizedCarry); if (!Second) return std::unexpected(Second.error()); auto Combined = buildBinaryAfter(Builder, **Second, InstructionOpcode::OR, CarryOutRegister, *FirstCarry, *SecondCarry); if (!Combined) return std::unexpected(Combined.error()); return {}; } auto FirstOverflow = buildSignedOverflowAfter(Builder, **Final, Function, Target, *Type, *Partial, LeftRegister, RightRegister, IsAdd); if (!FirstOverflow) return std::unexpected(FirstOverflow.error()); auto SecondOverflow = buildSignedOverflowAfter(Builder, *FirstOverflow->second, Function, Target, *Type, ResultRegister, *Partial, *NormalizedCarry, IsAdd); if (!SecondOverflow) return std::unexpected(SecondOverflow.error()); auto Combined = buildBinaryAfter(Builder, *SecondOverflow->second, InstructionOpcode::OR, CarryOutRegister, FirstOverflow->first, SecondOverflow->first); if (!Combined) return std::unexpected(Combined.error()); return {}; } [[nodiscard]] static Expected selectAtomicRMW(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target, std::optional Type, GenericOpcode Generic) { const MachineOperand *AddressOperand = Instruction.operandAt(1); const MachineOperand *ValueOperand = Instruction.operandAt(2); const Register *Address = AddressOperand == nullptr ? nullptr : AddressOperand->getIf(); const Register *Value = ValueOperand == nullptr ? nullptr : ValueOperand->getIf(); if (!Type || Address == nullptr || Value == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V atomic RMW has invalid operands"); Register SelectedValue = *Value; const bool IsWord = Type->scalarBitWidth() == 32; if (Generic == GenericOpcode::G_ATOMICRMW_SUB) { auto Negated = createTemporary(Function, *Type, Target); auto Zero = physicalRegister(PhysicalRegisterID::X0); if (!Negated || !Zero) return std::unexpected(!Negated ? Negated.error() : Zero.error()); MachineIRBuilder Builder(Function, Target); auto Point = Builder.setInsertPointBefore(Instruction); if (!Point) return std::unexpected(Point.error()); auto Pending = Builder.buildInstr( targetOpcode(IsWord ? InstructionOpcode::SUBW : InstructionOpcode::SUB)); if (!Pending) return std::unexpected(Pending.error()); auto Def = Pending->addDef(*Negated); if (!Def) return std::unexpected(Def.error()); auto Left = Pending->addUse(Register::fromPhysical(*Zero)); if (!Left) return std::unexpected(Left.error()); auto Right = Pending->addUse(*Value); if (!Right) return std::unexpected(Right.error()); auto Built = Pending->build(); if (!Built) return std::unexpected(Built.error()); SelectedValue = *Negated; } const auto SelectOpcode = [Generic, IsWord]() -> InstructionOpcode { switch (Generic) { case GenericOpcode::G_ATOMICRMW_XCHG: return IsWord ? InstructionOpcode::AMOSWAP_W : InstructionOpcode::AMOSWAP_D; case GenericOpcode::G_ATOMICRMW_ADD: case GenericOpcode::G_ATOMICRMW_SUB: return IsWord ? InstructionOpcode::AMOADD_W : InstructionOpcode::AMOADD_D; case GenericOpcode::G_ATOMICRMW_AND: return IsWord ? InstructionOpcode::AMOAND_W : InstructionOpcode::AMOAND_D; case GenericOpcode::G_ATOMICRMW_OR: return IsWord ? InstructionOpcode::AMOOR_W : InstructionOpcode::AMOOR_D; case GenericOpcode::G_ATOMICRMW_XOR: return IsWord ? InstructionOpcode::AMOXOR_W : InstructionOpcode::AMOXOR_D; case GenericOpcode::G_ATOMICRMW_MAX: return IsWord ? InstructionOpcode::AMOMAX_W : InstructionOpcode::AMOMAX_D; case GenericOpcode::G_ATOMICRMW_MIN: return IsWord ? InstructionOpcode::AMOMIN_W : InstructionOpcode::AMOMIN_D; case GenericOpcode::G_ATOMICRMW_UMAX: return IsWord ? InstructionOpcode::AMOMAXU_W : InstructionOpcode::AMOMAXU_D; case GenericOpcode::G_ATOMICRMW_UMIN: return IsWord ? InstructionOpcode::AMOMINU_W : InstructionOpcode::AMOMINU_D; default: return InstructionOpcode::Invalid; } }; const InstructionOpcode Selected = SelectOpcode(); if (Selected == InstructionOpcode::Invalid) return makeUnexpected(ErrorCode::InternalError, "RISC-V atomic RMW selection reached an unhandled opcode"); auto SetValue = Instruction.operandAt(1)->setRegister(SelectedValue); if (!SetValue) return std::unexpected(SetValue.error()); auto SetAddress = Instruction.operandAt(2)->setRegister(*Address); if (!SetAddress) return std::unexpected(SetAddress.error()); return Instruction.setOpcode(targetOpcode(Selected)); } [[nodiscard]] static Expected selectIntegerExtension(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target, GenericOpcode Generic, std::optional Type) { const std::optional SourceType = operandType(Function, Instruction, 1); const MachineOperand *DestinationOperand = Instruction.operandAt(0); const MachineOperand *SourceOperand = Instruction.operandAt(1); const Register *Destination = DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); if (!Type || !SourceType || Destination == nullptr || Source == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V integer extension has invalid register operands"); const Register DestinationRegister = *Destination; if (Generic == GenericOpcode::G_ANYEXT) return Instruction.setOpcode(opcode(GenericOpcode::COPY)); const std::uint32_t DestinationBits = Type->scalarBitWidth(); const std::uint32_t SourceBits = SourceType->scalarBitWidth(); if (Generic == GenericOpcode::G_SEXT && DestinationBits == 64 && SourceBits == 32) { auto ImmediateResult = appendImmediate(Instruction, 0); if (!ImmediateResult) return std::unexpected(ImmediateResult.error()); return Instruction.setOpcode(targetOpcode(InstructionOpcode::ADDIW)); } const std::uint32_t ContainerBits = DestinationBits <= 32 ? 32 : 64; const std::int64_t Shift = static_cast(ContainerBits - SourceBits); const InstructionOpcode ShiftLeft = ContainerBits == 32 ? InstructionOpcode::SLLIW : InstructionOpcode::SLLI; const InstructionOpcode ShiftRight = Generic == GenericOpcode::G_SEXT ? (ContainerBits == 32 ? InstructionOpcode::SRAIW : InstructionOpcode::SRAI) : (ContainerBits == 32 ? InstructionOpcode::SRLIW : InstructionOpcode::SRLI); auto Temporary = createTemporary(Function, *Type, Target); if (!Temporary) return std::unexpected(Temporary.error()); auto SetDestination = Instruction.operandAt(0)->setRegister(*Temporary); if (!SetDestination) return std::unexpected(SetDestination.error()); auto SetOpcode = Instruction.setOpcode(targetOpcode(ShiftLeft)); if (!SetOpcode) return std::unexpected(SetOpcode.error()); auto ShiftImmediate = appendImmediate(Instruction, Shift); if (!ShiftImmediate) return std::unexpected(ShiftImmediate.error()); MachineIRBuilder Builder(Function, Target); auto Shifted = buildImmediateAfter(Builder, Instruction, ShiftRight, DestinationRegister, *Temporary, Shift); if (!Shifted) return std::unexpected(Shifted.error()); return {}; } [[nodiscard]] static Expected selectSignExtendInRegister(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target, std::optional Type) { const MachineOperand *DestinationOperand = Instruction.operandAt(0); const MachineOperand *SourceOperand = Instruction.operandAt(1); const Register *Destination = DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); std::optional Width; if (const MachineOperand *WidthOperand = Instruction.operandAt(2); WidthOperand != nullptr) { auto ParsedWidth = signedImmediate(*WidthOperand); if (ParsedWidth) Width = *ParsedWidth; } if (!Type || Destination == nullptr || Source == nullptr || !Width || *Width <= 0 || static_cast(*Width) > Type->scalarBitWidth()) return makeUnexpected(ErrorCode::VerificationError, "RISC-V sign-extension-in-register has invalid operands"); const Register DestinationRegister = *Destination; if (static_cast(*Width) == Type->scalarBitWidth()) { auto EraseWidth = Instruction.eraseOperand(2); if (!EraseWidth) return std::unexpected(EraseWidth.error()); return Instruction.setOpcode(opcode(GenericOpcode::COPY)); } const std::uint32_t TypeBits = Type->scalarBitWidth(); if (TypeBits == 64 && *Width == 32) { auto ImmediateResult = replaceImmediate(Instruction, 2, 0); if (!ImmediateResult) return std::unexpected(ImmediateResult.error()); return Instruction.setOpcode(targetOpcode(InstructionOpcode::ADDIW)); } const std::uint32_t ContainerBits = TypeBits <= 32 ? 32 : 64; const std::int64_t Shift = static_cast(ContainerBits) - *Width; const InstructionOpcode ShiftLeft = ContainerBits == 32 ? InstructionOpcode::SLLIW : InstructionOpcode::SLLI; const InstructionOpcode ShiftRight = ContainerBits == 32 ? InstructionOpcode::SRAIW : InstructionOpcode::SRAI; auto Temporary = createTemporary(Function, *Type, Target); if (!Temporary) return std::unexpected(Temporary.error()); auto SetDestination = Instruction.operandAt(0)->setRegister(*Temporary); if (!SetDestination) return std::unexpected(SetDestination.error()); auto SetOpcode = Instruction.setOpcode(targetOpcode(ShiftLeft)); if (!SetOpcode) return std::unexpected(SetOpcode.error()); auto ShiftImmediate = replaceImmediate(Instruction, 2, Shift); if (!ShiftImmediate) return std::unexpected(ShiftImmediate.error()); MachineIRBuilder Builder(Function, Target); auto Shifted = buildImmediateAfter(Builder, Instruction, ShiftRight, DestinationRegister, *Temporary, Shift); if (!Shifted) return std::unexpected(Shifted.error()); return {}; } [[nodiscard]] static Expected selectIntegerCompare(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target) { const MachineOperand *PredicateMO = Instruction.operandAt(1); const auto *Predicate = PredicateMO == nullptr ? nullptr : PredicateMO->getIf(); const MachineOperand *DestinationOperand = Instruction.operandAt(0); const MachineOperand *LeftOperand = Instruction.operandAt(2); const MachineOperand *RightOperand = Instruction.operandAt(3); const Register *Destination = DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf(); const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf(); const std::optional LeftType = operandType(Function, Instruction, 2); if (Predicate == nullptr || Predicate->integerPredicate() == nullptr || Destination == nullptr || Left == nullptr || Right == nullptr || !LeftType) return makeUnexpected(ErrorCode::VerificationError, "RISC-V integer comparison has invalid operands"); const IntegerPredicate Condition = *Predicate->integerPredicate(); const Register DestinationRegister = *Destination; const bool SignedOperands = Condition == IntegerPredicate::SignedGreaterThan || Condition == IntegerPredicate::SignedGreaterEqual || Condition == IntegerPredicate::SignedLessThan || Condition == IntegerPredicate::SignedLessEqual; auto LeftRegister = normalizeIntegerBefore(Instruction, Function, Target, *Left, *LeftType, SignedOperands); auto RightRegister = normalizeIntegerBefore(Instruction, Function, Target, *Right, *LeftType, SignedOperands); if (!LeftRegister || !RightRegister) return std::unexpected(!LeftRegister ? LeftRegister.error() : RightRegister.error()); auto ErasePredicate = Instruction.eraseOperand(1); if (!ErasePredicate) return std::unexpected(ErasePredicate.error()); const auto SetSources = [&Instruction](Register NewLeft, Register NewRight) -> Expected { auto SetLeft = Instruction.operandAt(1)->setRegister(NewLeft); if (!SetLeft) return std::unexpected(SetLeft.error()); return Instruction.operandAt(2)->setRegister(NewRight); }; const auto StrictOpcode = [](IntegerPredicate Value) -> std::optional { switch (Value) { case IntegerPredicate::SignedLessThan: return InstructionOpcode::SLT; case IntegerPredicate::UnsignedLessThan: return InstructionOpcode::SLTU; default: return std::nullopt; } }; IntegerPredicate StrictCondition = Condition; bool Swap = false; bool Invert = false; switch (Condition) { case IntegerPredicate::Equal: case IntegerPredicate::NotEqual: break; case IntegerPredicate::SignedGreaterThan: StrictCondition = IntegerPredicate::SignedLessThan; Swap = true; break; case IntegerPredicate::SignedGreaterEqual: StrictCondition = IntegerPredicate::SignedLessThan; Invert = true; break; case IntegerPredicate::SignedLessThan: StrictCondition = IntegerPredicate::SignedLessThan; break; case IntegerPredicate::SignedLessEqual: StrictCondition = IntegerPredicate::SignedLessThan; Swap = true; Invert = true; break; case IntegerPredicate::UnsignedGreaterThan: StrictCondition = IntegerPredicate::UnsignedLessThan; Swap = true; break; case IntegerPredicate::UnsignedGreaterEqual: StrictCondition = IntegerPredicate::UnsignedLessThan; Invert = true; break; case IntegerPredicate::UnsignedLessThan: StrictCondition = IntegerPredicate::UnsignedLessThan; break; case IntegerPredicate::UnsignedLessEqual: StrictCondition = IntegerPredicate::UnsignedLessThan; Swap = true; Invert = true; break; case IntegerPredicate::Count: return makeUnexpected(ErrorCode::Unsupported, "RISC-V integer comparison predicate is invalid"); } MachineIRBuilder Builder(Function, Target); if (Condition == IntegerPredicate::Equal || Condition == IntegerPredicate::NotEqual) { auto Temporary = createTemporary(Function, *LeftType, Target); if (!Temporary) return std::unexpected(Temporary.error()); auto SetDestination = Instruction.operandAt(0)->setRegister(*Temporary); if (!SetDestination) return std::unexpected(SetDestination.error()); auto Set = SetSources(*LeftRegister, *RightRegister); if (!Set) return std::unexpected(Set.error()); auto SetOpcode = Instruction.setOpcode(targetOpcode(InstructionOpcode::XOR)); if (!SetOpcode) return std::unexpected(SetOpcode.error()); if (Condition == IntegerPredicate::Equal) { auto Result = buildImmediateAfter(Builder, Instruction, InstructionOpcode::SLTIU, DestinationRegister, *Temporary, 1); if (!Result) return std::unexpected(Result.error()); } else { auto Zero = physicalRegister(PhysicalRegisterID::X0); if (!Zero) return std::unexpected(Zero.error()); auto Result = buildBinaryAfter(Builder, Instruction, InstructionOpcode::SLTU, DestinationRegister, Register::fromPhysical(*Zero), *Temporary); if (!Result) return std::unexpected(Result.error()); } return {}; } const std::optional OpcodeValue = StrictOpcode(StrictCondition); if (!OpcodeValue) return makeUnexpected(ErrorCode::Unsupported, "RISC-V integer comparison predicate is invalid"); if (Invert) { auto Temporary = createTemporary(Function, *LeftType, Target); if (!Temporary) return std::unexpected(Temporary.error()); auto SetDestination = Instruction.operandAt(0)->setRegister(*Temporary); if (!SetDestination) return std::unexpected(SetDestination.error()); const Register Base = *Temporary; if (Swap) { auto Set = SetSources(*RightRegister, *LeftRegister); if (!Set) return std::unexpected(Set.error()); } else { auto Set = SetSources(*LeftRegister, *RightRegister); if (!Set) return std::unexpected(Set.error()); } auto SetOpcode = Instruction.setOpcode(targetOpcode(*OpcodeValue)); if (!SetOpcode) return std::unexpected(SetOpcode.error()); auto Result = buildImmediateAfter(Builder, Instruction, InstructionOpcode::XORI, DestinationRegister, Base, 1); if (!Result) return std::unexpected(Result.error()); return {}; } if (Swap) { auto Set = SetSources(*RightRegister, *LeftRegister); if (!Set) return std::unexpected(Set.error()); } else { auto Set = SetSources(*LeftRegister, *RightRegister); if (!Set) return std::unexpected(Set.error()); } return Instruction.setOpcode(targetOpcode(*OpcodeValue)); } [[nodiscard]] static Expected selectFloatingCompare(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target) { const MachineOperand *PredicateMO = Instruction.operandAt(1); const auto *Predicate = PredicateMO == nullptr ? nullptr : PredicateMO->getIf(); const MachineOperand *DestinationOperand = Instruction.operandAt(0); const MachineOperand *LeftOperand = Instruction.operandAt(2); const MachineOperand *RightOperand = Instruction.operandAt(3); const Register *Destination = DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf(); const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf(); const std::optional FloatType = operandType(Function, Instruction, 2); const std::optional ResultType = resultType(Function, Instruction); if (Predicate == nullptr || Predicate->floatingPredicate() == nullptr || Destination == nullptr || Left == nullptr || Right == nullptr || !FloatType || !ResultType) return makeUnexpected(ErrorCode::VerificationError, "RISC-V floating comparison has invalid operands"); const FloatingPredicate Condition = *Predicate->floatingPredicate(); const Register DestinationRegister = *Destination; const Register LeftRegister = *Left; const Register RightRegister = *Right; const InstructionOpcode EqualOpcode = FloatType->scalarBitWidth() == 32 ? InstructionOpcode::FEQ_S : InstructionOpcode::FEQ_D; const InstructionOpcode LessOpcode = FloatType->scalarBitWidth() == 32 ? InstructionOpcode::FLT_S : InstructionOpcode::FLT_D; const InstructionOpcode LessEqualOpcode = FloatType->scalarBitWidth() == 32 ? InstructionOpcode::FLE_S : InstructionOpcode::FLE_D; auto ErasePredicate = Instruction.eraseOperand(1); if (!ErasePredicate) return std::unexpected(ErasePredicate.error()); const auto SetBase = [&Instruction](InstructionOpcode OpcodeValue, Register Def, Register NewLeft, Register NewRight) -> Expected { auto SetDestination = Instruction.operandAt(0)->setRegister(Def); if (!SetDestination) return std::unexpected(SetDestination.error()); auto SetLeft = Instruction.operandAt(1)->setRegister(NewLeft); if (!SetLeft) return std::unexpected(SetLeft.error()); auto SetRight = Instruction.operandAt(2)->setRegister(NewRight); if (!SetRight) return std::unexpected(SetRight.error()); return Instruction.setOpcode(targetOpcode(OpcodeValue)); }; const auto BuildInverted = [&](InstructionOpcode BaseOpcode, Register BaseLeft, Register BaseRight) -> Expected { auto Temporary = createTemporary(Function, *ResultType, Target); if (!Temporary) return std::unexpected(Temporary.error()); auto Base = SetBase(BaseOpcode, *Temporary, BaseLeft, BaseRight); if (!Base) return std::unexpected(Base.error()); MachineIRBuilder Builder(Function, Target); auto Invert = buildImmediateAfter(Builder, Instruction, InstructionOpcode::XORI, DestinationRegister, *Temporary, 1); if (!Invert) return std::unexpected(Invert.error()); return {}; }; const auto BuildSelfAnd = [&](bool Invert) -> Expected { auto First = createTemporary(Function, *ResultType, Target); auto Second = createTemporary(Function, *ResultType, Target); if (!First || !Second) return std::unexpected((!First ? First.error() : Second.error())); auto Base = SetBase(EqualOpcode, *First, LeftRegister, LeftRegister); if (!Base) return std::unexpected(Base.error()); MachineIRBuilder Builder(Function, Target); auto SecondInstruction = buildBinaryAfter(Builder, Instruction, EqualOpcode, *Second, RightRegister, RightRegister); if (!SecondInstruction) return std::unexpected(SecondInstruction.error()); Register CombinedDestination = DestinationRegister; if (Invert) { auto Temporary = createTemporary(Function, *ResultType, Target); if (!Temporary) return std::unexpected(Temporary.error()); CombinedDestination = *Temporary; } auto Combined = buildBinaryAfter(Builder, **SecondInstruction, InstructionOpcode::AND, CombinedDestination, *First, *Second); if (!Combined) return std::unexpected(Combined.error()); if (!Invert) return {}; auto Inverted = buildImmediateAfter(Builder, **Combined, InstructionOpcode::XORI, DestinationRegister, CombinedDestination, 1); if (!Inverted) return std::unexpected(Inverted.error()); return {}; }; const auto BuildOrderedNotEqual = [&](bool Invert) -> Expected { auto First = createTemporary(Function, *ResultType, Target); auto Second = createTemporary(Function, *ResultType, Target); if (!First || !Second) return std::unexpected((!First ? First.error() : Second.error())); auto Base = SetBase(LessOpcode, *First, LeftRegister, RightRegister); if (!Base) return std::unexpected(Base.error()); MachineIRBuilder Builder(Function, Target); auto SecondInstruction = buildBinaryAfter(Builder, Instruction, LessOpcode, *Second, RightRegister, LeftRegister); if (!SecondInstruction) return std::unexpected(SecondInstruction.error()); Register CombinedDestination = DestinationRegister; if (Invert) { auto Temporary = createTemporary(Function, *ResultType, Target); if (!Temporary) return std::unexpected(Temporary.error()); CombinedDestination = *Temporary; } auto Combined = buildBinaryAfter(Builder, **SecondInstruction, InstructionOpcode::OR, CombinedDestination, *First, *Second); if (!Combined) return std::unexpected(Combined.error()); if (!Invert) return {}; auto Inverted = buildImmediateAfter(Builder, **Combined, InstructionOpcode::XORI, DestinationRegister, CombinedDestination, 1); if (!Inverted) return std::unexpected(Inverted.error()); return {}; }; switch (Condition) { case FloatingPredicate::AlwaysFalse: case FloatingPredicate::AlwaysTrue: { auto EraseRight = Instruction.eraseOperand(2); if (!EraseRight) return std::unexpected(EraseRight.error()); auto EraseLeft = Instruction.eraseOperand(1); if (!EraseLeft) return std::unexpected(EraseLeft.error()); auto Zero = physicalRegister(PhysicalRegisterID::X0); if (!Zero) return std::unexpected(Zero.error()); auto InsertZero = insertPhysicalRegister(Instruction, 1, *Zero, RegisterOperandFlag::Use); if (!InsertZero) return std::unexpected(InsertZero.error()); auto Immediate = appendImmediate(Instruction, Condition == FloatingPredicate::AlwaysTrue ? 1 : 0); if (!Immediate) return std::unexpected(Immediate.error()); return Instruction.setOpcode(targetOpcode(InstructionOpcode::ADDI)); } case FloatingPredicate::OrderedEqual: return SetBase(EqualOpcode, DestinationRegister, LeftRegister, RightRegister); case FloatingPredicate::OrderedGreaterThan: return SetBase(LessOpcode, DestinationRegister, RightRegister, LeftRegister); case FloatingPredicate::OrderedGreaterEqual: return SetBase(LessEqualOpcode, DestinationRegister, RightRegister, LeftRegister); case FloatingPredicate::OrderedLessThan: return SetBase(LessOpcode, DestinationRegister, LeftRegister, RightRegister); case FloatingPredicate::OrderedLessEqual: return SetBase(LessEqualOpcode, DestinationRegister, LeftRegister, RightRegister); case FloatingPredicate::OrderedNotEqual: return BuildOrderedNotEqual(false); case FloatingPredicate::Ordered: return BuildSelfAnd(false); case FloatingPredicate::Unordered: return BuildSelfAnd(true); case FloatingPredicate::UnorderedEqual: return BuildOrderedNotEqual(true); case FloatingPredicate::UnorderedGreaterThan: return BuildInverted(LessEqualOpcode, LeftRegister, RightRegister); case FloatingPredicate::UnorderedGreaterEqual: return BuildInverted(LessOpcode, LeftRegister, RightRegister); case FloatingPredicate::UnorderedLessThan: return BuildInverted(LessEqualOpcode, RightRegister, LeftRegister); case FloatingPredicate::UnorderedLessEqual: return BuildInverted(LessOpcode, RightRegister, LeftRegister); case FloatingPredicate::UnorderedNotEqual: return BuildInverted(EqualOpcode, LeftRegister, RightRegister); case FloatingPredicate::Count: return makeUnexpected(ErrorCode::Unsupported, "RISC-V floating comparison predicate is invalid"); } return makeUnexpected(ErrorCode::Unsupported, "RISC-V floating comparison predicate is invalid"); } [[nodiscard]] static Expected selectFloatToInteger(MachineInstr &Instruction, MachineFunction &Function, std::optional Type, bool Unsigned) { const std::optional Source = operandType(Function, Instruction, 1); if (!Type || !Source) return makeUnexpected(ErrorCode::VerificationError, "RISC-V floating-to-integer conversion has invalid types"); auto Rounding = appendImmediate(Instruction, 1); if (!Rounding) return std::unexpected(Rounding.error()); const bool IsWord = Type->scalarBitWidth() <= 32; if (Source->scalarBitWidth() == 32) { return Instruction.setOpcode(targetOpcode( IsWord ? (Unsigned ? InstructionOpcode::FCVT_WU_S : InstructionOpcode::FCVT_W_S) : (Unsigned ? InstructionOpcode::FCVT_LU_S : InstructionOpcode::FCVT_L_S))); } return Instruction.setOpcode(targetOpcode( IsWord ? (Unsigned ? InstructionOpcode::FCVT_WU_D : InstructionOpcode::FCVT_W_D) : (Unsigned ? InstructionOpcode::FCVT_LU_D : InstructionOpcode::FCVT_L_D))); } [[nodiscard]] static Expected selectIntegerToFloat(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target, std::optional Type, bool Unsigned) { const std::optional Source = operandType(Function, Instruction, 1); const MachineOperand *SourceOperand = Instruction.operandAt(1); const Register *SourceRegister = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); if (!Type || !Source || SourceRegister == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V integer-to-floating conversion has invalid types"); auto Normalized = normalizeIntegerBefore(Instruction, Function, Target, *SourceRegister, *Source, !Unsigned); if (!Normalized) return std::unexpected(Normalized.error()); auto SetSource = Instruction.operandAt(1)->setRegister(*Normalized); if (!SetSource) return std::unexpected(SetSource.error()); auto Rounding = appendImmediate(Instruction, 0); if (!Rounding) return std::unexpected(Rounding.error()); const bool IsWord = Source->scalarBitWidth() <= 32; if (Type->scalarBitWidth() == 32) { return Instruction.setOpcode(targetOpcode( IsWord ? (Unsigned ? InstructionOpcode::FCVT_S_WU : InstructionOpcode::FCVT_S_W) : (Unsigned ? InstructionOpcode::FCVT_S_LU : InstructionOpcode::FCVT_S_L))); } return Instruction.setOpcode(targetOpcode( IsWord ? (Unsigned ? InstructionOpcode::FCVT_D_WU : InstructionOpcode::FCVT_D_W) : (Unsigned ? InstructionOpcode::FCVT_D_LU : InstructionOpcode::FCVT_D_L))); } [[nodiscard]] static Expected selectFloatingMinMax(MachineInstr &Instruction, std::optional Type, InstructionOpcode Single, InstructionOpcode Double) { if (Type && Type->isFloat() && Type->scalarBitWidth() == 32) return Instruction.setOpcode(targetOpcode(Single)); if (Type && Type->isFloat() && Type->scalarBitWidth() == 64) return Instruction.setOpcode(targetOpcode(Double)); return makeUnexpected(ErrorCode::Unsupported, "RISC-V cannot select this floating min/max type"); } [[nodiscard]] static Expected selectFence(MachineInstr &Instruction) { const auto Ordering = signedImmediate(*Instruction.operandAt(0)); if (!Ordering) return std::unexpected(Ordering.error()); std::int64_t Pred = 0; std::int64_t Succ = 0; switch (*Ordering) { case 4: Pred = 1; Succ = 3; break; case 5: Pred = 3; Succ = 2; break; case 6: case 7: Pred = 3; Succ = 3; break; default: return makeUnexpected(ErrorCode::Unsupported, "RISC-V fence ordering is unsupported"); } auto SetPred = replaceImmediate(Instruction, 0, Pred); if (!SetPred) return std::unexpected(SetPred.error()); auto SetSucc = replaceImmediate(Instruction, 1, Succ); if (!SetSucc) return std::unexpected(SetSucc.error()); return Instruction.setOpcode(targetOpcode(InstructionOpcode::FENCE)); } [[nodiscard]] static Expected selectIndirectBranch(MachineInstr &Instruction) { auto Zero = physicalRegister(PhysicalRegisterID::X0); if (!Zero) return std::unexpected(Zero.error()); auto Inserted = insertPhysicalRegister(Instruction, 0, *Zero, RegisterOperandFlag::Define); if (!Inserted) return std::unexpected(Inserted.error()); auto Immediate = appendImmediate(Instruction, 0); if (!Immediate) return std::unexpected(Immediate.error()); return Instruction.setOpcode(targetOpcode(InstructionOpcode::JALR)); } [[nodiscard]] static Expected selectTrap(MachineInstr &Instruction) { while (Instruction.operandCount() != 0) { auto Erased = Instruction.eraseOperand(Instruction.operandCount() - 1U); if (!Erased) return std::unexpected(Erased.error()); } return Instruction.setOpcode(targetOpcode(InstructionOpcode::EBREAK)); } [[nodiscard]] static Expected selectBitcast(MachineInstr &Instruction, MachineFunction &Function) { const std::optional DestinationType = resultType(Function, Instruction); const std::optional SourceType = operandType(Function, Instruction, 1); if (!DestinationType || !SourceType) return makeUnexpected(ErrorCode::VerificationError, "RISC-V bitcast has invalid register types"); if (DestinationType->isInteger() && SourceType->isFloat()) { return Instruction.setOpcode(targetOpcode(SourceType->scalarBitWidth() == 32 ? InstructionOpcode::FMV_X_W : InstructionOpcode::FMV_X_D)); } if (DestinationType->isFloat() && SourceType->isInteger()) { return Instruction.setOpcode(targetOpcode(DestinationType->scalarBitWidth() == 32 ? InstructionOpcode::FMV_W_X : InstructionOpcode::FMV_D_X)); } return makeUnexpected(ErrorCode::Unsupported, "RISC-V bitcast pair is unsupported"); } [[nodiscard]] static Expected selectIntegerBinary(MachineInstr &Instruction, std::optional Type, InstructionOpcode DoubleWord, InstructionOpcode Word) { if (!Type || !isSupportedIntegerType(*Type)) return makeUnexpected(ErrorCode::Unsupported, "RISC-V integer instruction requires a supported scalar integer"); const bool IsWord = Type->isInteger() && Type->scalarBitWidth() <= 32; return Instruction.setOpcode(targetOpcode(IsWord ? Word : DoubleWord)); } [[nodiscard]] static Expected selectNormalizedIntegerBinary(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target, std::optional Type, InstructionOpcode DoubleWord, InstructionOpcode Word, bool Signed, bool NormalizeRight) { const MachineOperand *LeftOperand = Instruction.operandAt(1); const MachineOperand *RightOperand = Instruction.operandAt(2); const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf(); const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf(); if (!Type || !isSupportedIntegerScalar(*Type) || Left == nullptr || Right == nullptr) { return makeUnexpected(ErrorCode::VerificationError, "RISC-V normalized integer operation has invalid operands"); } auto NormalizedLeft = normalizeIntegerBefore(Instruction, Function, Target, *Left, *Type, Signed); if (!NormalizedLeft) return std::unexpected(NormalizedLeft.error()); Register SelectedRight = *Right; if (NormalizeRight) { auto NormalizedRight = normalizeIntegerBefore(Instruction, Function, Target, *Right, *Type, Signed); if (!NormalizedRight) return std::unexpected(NormalizedRight.error()); SelectedRight = *NormalizedRight; } auto SetLeft = Instruction.operandAt(1)->setRegister(*NormalizedLeft); if (!SetLeft) return std::unexpected(SetLeft.error()); auto SetRight = Instruction.operandAt(2)->setRegister(SelectedRight); if (!SetRight) return std::unexpected(SetRight.error()); const bool IsWord = Type->scalarBitWidth() <= 32; return Instruction.setOpcode(targetOpcode(IsWord ? Word : DoubleWord)); } [[nodiscard]] static Expected selectLoad(MachineInstr &Instruction, std::optional Type) { if (!Type) return makeUnexpected(ErrorCode::VerificationError, "load result has no virtual-register type"); const auto Bytes = memoryAccessBytes(Instruction); if (!Bytes) return makeUnexpected(ErrorCode::Unsupported, "RISC-V load size does not match its result type"); if (Instruction.memOperands().front()->isAtomic()) { if (!isSupportedIntegerType(*Type) || (*Bytes != 4 && *Bytes != 8)) return makeUnexpected(ErrorCode::Unsupported, "RISC-V atomic load requires a 32-bit or 64-bit GPR value"); return Instruction.setOpcode( targetOpcode(*Bytes == 4 ? InstructionOpcode::LR_W : InstructionOpcode::LR_D)); } auto AppendOffset = appendImmediate(Instruction, 0); if (!AppendOffset) return std::unexpected(AppendOffset.error()); if (isSupportedIntegerType(*Type)) { const InstructionOpcode Selected = *Bytes == 1 ? InstructionOpcode::LB : *Bytes == 2 ? InstructionOpcode::LH : *Bytes == 4 ? InstructionOpcode::LW : *Bytes == 8 ? InstructionOpcode::LD : InstructionOpcode::Invalid; if (Selected == InstructionOpcode::Invalid) return makeUnexpected(ErrorCode::Unsupported, "RISC-V integer load width is unsupported"); return Instruction.setOpcode(targetOpcode(Selected)); } if (Type->isFloat() && Type->scalarBitWidth() == 32) return Instruction.setOpcode(targetOpcode(InstructionOpcode::FLW)); if (Type->isFloat() && Type->scalarBitWidth() == 64) return Instruction.setOpcode(targetOpcode(InstructionOpcode::FLD)); return makeUnexpected(ErrorCode::Unsupported, "RISC-V cannot select this generic load type"); } [[nodiscard]] static Expected selectExtLoad(MachineInstr &Instruction, std::optional Type, bool Signed) { if (!Type || !isSupportedIntegerType(*Type)) return makeUnexpected(ErrorCode::Unsupported, "RISC-V extension load requires an integer result"); const auto Bytes = memoryAccessBytes(Instruction); if (!Bytes) return makeUnexpected(ErrorCode::Unsupported, "RISC-V extension load has an invalid width"); const InstructionOpcode Selected = *Bytes == 1 ? (Signed ? InstructionOpcode::LB : InstructionOpcode::LBU) : *Bytes == 2 ? (Signed ? InstructionOpcode::LH : InstructionOpcode::LHU) : *Bytes == 4 ? (Signed ? InstructionOpcode::LW : InstructionOpcode::LWU) : InstructionOpcode::Invalid; if (Selected == InstructionOpcode::Invalid) return makeUnexpected(ErrorCode::Unsupported, "RISC-V extension load width is unsupported"); auto Immediate = appendImmediate(Instruction, 0); if (!Immediate) return std::unexpected(Immediate.error()); return Instruction.setOpcode(targetOpcode(Selected)); } [[nodiscard]] static Expected selectFloatExtLoad(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target) { const MachineOperand *DestinationOperand = Instruction.operandAt(0); const Register *Destination = DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); if (Destination == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V floating extension load has no destination register"); const Register DestinationRegister = *Destination; auto F32 = LLT::makeFloat(FloatSemantics::IEEESingle); if (!F32) return std::unexpected(F32.error()); auto Temporary = createTemporary(Function, *F32, Target); if (!Temporary) return std::unexpected(Temporary.error()); auto SetDestination = Instruction.operandAt(0)->setRegister(*Temporary); if (!SetDestination) return std::unexpected(SetDestination.error()); auto Immediate = appendImmediate(Instruction, 0); if (!Immediate) return std::unexpected(Immediate.error()); auto SetOpcode = Instruction.setOpcode(targetOpcode(InstructionOpcode::FLW)); if (!SetOpcode) return std::unexpected(SetOpcode.error()); MachineIRBuilder Builder(Function, Target); auto Conversion = buildImmediateAfter(Builder, Instruction, InstructionOpcode::FCVT_D_S, DestinationRegister, *Temporary, 0); if (!Conversion) return std::unexpected(Conversion.error()); return {}; } [[nodiscard]] static Expected selectFloatTruncStore(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target) { const MachineOperand *ValueOperand = Instruction.operandAt(0); const Register *Value = ValueOperand == nullptr ? nullptr : ValueOperand->getIf(); if (Value == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V floating truncation store has no value register"); const Register ValueRegister = *Value; auto F32 = LLT::makeFloat(FloatSemantics::IEEESingle); if (!F32) return std::unexpected(F32.error()); auto Temporary = createTemporary(Function, *F32, Target); if (!Temporary) return std::unexpected(Temporary.error()); MachineIRBuilder Builder(Function, Target); auto Point = Builder.setInsertPointBefore(Instruction); if (!Point) return std::unexpected(Point.error()); auto Conversion = Builder.buildInstr(targetOpcode(InstructionOpcode::FCVT_S_D)); if (!Conversion) return std::unexpected(Conversion.error()); auto Def = Conversion->addDef(*Temporary); if (!Def) return std::unexpected(Def.error()); auto Use = Conversion->addUse(ValueRegister); if (!Use) return std::unexpected(Use.error()); auto Rounding = Conversion->addImmediate(0); if (!Rounding) return std::unexpected(Rounding.error()); auto Built = Conversion->build(); if (!Built) return std::unexpected(Built.error()); auto SetValue = Instruction.operandAt(0)->setRegister(*Temporary); if (!SetValue) return std::unexpected(SetValue.error()); auto Immediate = appendImmediate(Instruction, 0); if (!Immediate) return std::unexpected(Immediate.error()); return Instruction.setOpcode(targetOpcode(InstructionOpcode::FSW)); } [[nodiscard]] static Expected selectStore(MachineInstr &Instruction, MachineFunction &Function) { const MachineOperand *Value = Instruction.operandAt(0); const std::optional Type = Value == nullptr ? std::nullopt : registerType(Function, *Value); if (!Type) return makeUnexpected(ErrorCode::VerificationError, "store value has no virtual-register type"); const auto Bytes = memoryAccessBytes(Instruction); if (!Bytes) return makeUnexpected(ErrorCode::Unsupported, "RISC-V store size does not match its value type"); if (Instruction.memOperands().front()->isAtomic()) { if (!isSupportedIntegerType(*Type) || (*Bytes != 4 && *Bytes != 8)) return makeUnexpected(ErrorCode::Unsupported, "RISC-V atomic store requires a 32-bit or 64-bit GPR value"); auto Zero = physicalRegister(PhysicalRegisterID::X0); if (!Zero) return std::unexpected(Zero.error()); auto Destination = insertPhysicalRegister(Instruction, 0, *Zero, RegisterOperandFlag::Define); if (!Destination) return std::unexpected(Destination.error()); return Instruction.setOpcode( targetOpcode(*Bytes == 4 ? InstructionOpcode::AMOSWAP_W : InstructionOpcode::AMOSWAP_D)); } auto AppendOffset = appendImmediate(Instruction, 0); if (!AppendOffset) return std::unexpected(AppendOffset.error()); if (isSupportedIntegerType(*Type)) { const InstructionOpcode Selected = *Bytes == 1 ? InstructionOpcode::SB : *Bytes == 2 ? InstructionOpcode::SH : *Bytes == 4 ? InstructionOpcode::SW : *Bytes == 8 ? InstructionOpcode::SD : InstructionOpcode::Invalid; if (Selected == InstructionOpcode::Invalid) return makeUnexpected(ErrorCode::Unsupported, "RISC-V integer store width is unsupported"); return Instruction.setOpcode(targetOpcode(Selected)); } if (Type->isFloat() && Type->scalarBitWidth() == 32) return Instruction.setOpcode(targetOpcode(InstructionOpcode::FSW)); if (Type->isFloat() && Type->scalarBitWidth() == 64) return Instruction.setOpcode(targetOpcode(InstructionOpcode::FSD)); return makeUnexpected(ErrorCode::Unsupported, "RISC-V cannot select this generic store type"); } [[nodiscard]] static Expected selectBranch(MachineInstr &Instruction) { auto Zero = physicalRegister(PhysicalRegisterID::X0); if (!Zero) return std::unexpected(Zero.error()); auto Inserted = insertPhysicalRegister(Instruction, 0, *Zero, RegisterOperandFlag::Define); if (!Inserted) return std::unexpected(Inserted.error()); return Instruction.setOpcode(targetOpcode(InstructionOpcode::JAL)); } [[nodiscard]] static Expected selectConditionalBranch(MachineInstr &Instruction) { auto Zero = physicalRegister(PhysicalRegisterID::X0); if (!Zero) return std::unexpected(Zero.error()); auto Inserted = insertPhysicalRegister(Instruction, 1, *Zero, RegisterOperandFlag::Use); if (!Inserted) return std::unexpected(Inserted.error()); return Instruction.setOpcode(targetOpcode(InstructionOpcode::BNE)); } [[nodiscard]] static Expected selectIntegerConstant(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target, std::optional Type) { if (!Type || !isSupportedIntegerType(*Type)) { return makeUnexpected(ErrorCode::Unsupported, "RISC-V constant selection requires an i1-i64 or pointer result"); } const MachineOperand *ImmediateOperand = Instruction.operandAt(1); if (ImmediateOperand == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V generic constant has no immediate operand"); auto ImmediateValue = signedImmediate(*ImmediateOperand); if (!ImmediateValue) return std::unexpected(ImmediateValue.error()); const MachineOperand *DestinationOperand = Instruction.operandAt(0); const Register *DestinationValue = DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); if (DestinationValue == nullptr || !DestinationValue->isVirtual()) { return makeUnexpected(ErrorCode::VerificationError, "RISC-V generic constant must define a virtual register"); } const Register Destination = *DestinationValue; const std::int64_t Value = *ImmediateValue; const std::uint32_t ImmediateTargetFlags = ImmediateOperand->targetFlags(); const bool IsWord = Type->isInteger() && Type->scalarBitWidth() <= 32; const InstructionOpcode AddImmediate = IsWord ? InstructionOpcode::ADDIW : InstructionOpcode::ADDI; const InstructionOpcode ShiftLeftImmediate = IsWord ? InstructionOpcode::SLLIW : InstructionOpcode::SLLI; if (fitsSigned(Value, 12)) { auto Zero = physicalRegister(PhysicalRegisterID::X0); if (!Zero) return std::unexpected(Zero.error()); auto Source = insertPhysicalRegister(Instruction, 1, *Zero, RegisterOperandFlag::Use); if (!Source) return std::unexpected(Source.error()); auto EraseImmediate = Instruction.eraseOperand(2); if (!EraseImmediate) return std::unexpected(EraseImmediate.error()); auto NormalizeImmediate = Instruction.insertOperand( 2, MachineOperand::immediate(Value, ImmediateTargetFlags)); if (!NormalizeImmediate) return std::unexpected(NormalizeImmediate.error()); return Instruction.setOpcode(targetOpcode(AddImmediate)); } // Recursively decompose the value into signed 12-bit chunks. The // quotient is computed from an unsigned bit pattern with explicit sign // extension so both INT64_MIN and INT64_MAX remain well-defined. std::uint64_t Remaining = static_cast(Value); std::vector Digits; while (!fitsSigned(std::bit_cast(Remaining), 12)) { const std::uint64_t LowBits = Remaining & 0xFFFU; std::int64_t Digit = static_cast(LowBits); if (Digit >= 0x800) Digit -= 0x1000; Digits.push_back(Digit); const std::uint64_t Adjusted = Remaining - static_cast(Digit); std::uint64_t Quotient = Adjusted >> 12U; if ((Remaining & (std::uint64_t{1} << 63U)) != 0) Quotient |= ~std::uint64_t{0} << 52U; Remaining = Quotient; } const std::int64_t Base = std::bit_cast(Remaining); auto Temporary = createTemporary(Function, *Type, Target); if (!Temporary) return std::unexpected(Temporary.error()); auto *Parent = Instruction.parent(); if (Parent == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V constant selection received a detached instruction"); auto SetOpcode = Instruction.setOpcode(targetOpcode(AddImmediate)); if (!SetOpcode) return std::unexpected(SetOpcode.error()); auto SetDestination = Instruction.operandAt(0)->setRegister(*Temporary); if (!SetDestination) return std::unexpected(SetDestination.error()); auto EraseImmediate = Instruction.eraseOperand(1); if (!EraseImmediate) return std::unexpected(EraseImmediate.error()); auto Zero = physicalRegister(PhysicalRegisterID::X0); if (!Zero) return std::unexpected(Zero.error()); auto AddSource = insertPhysicalRegister(Instruction, 1, *Zero, RegisterOperandFlag::Use); if (!AddSource) return std::unexpected(AddSource.error()); auto AddBase = appendImmediate(Instruction, Base); if (!AddBase) return std::unexpected(AddBase.error()); MachineIRBuilder Builder(Function, Target); auto InsertPoint = Builder.setInsertPointAfter(Instruction); if (!InsertPoint) return std::unexpected(InsertPoint.error()); Register Current = *Temporary; for (std::size_t Reverse = Digits.size(); Reverse != 0; --Reverse) { const std::size_t Index = Reverse - 1U; auto ShiftResult = createTemporary(Function, *Type, Target); if (!ShiftResult) return std::unexpected(ShiftResult.error()); auto Shift = Builder.buildInstr(targetOpcode(ShiftLeftImmediate)); if (!Shift) return std::unexpected(Shift.error()); auto ShiftDef = Shift->addDef(*ShiftResult); if (!ShiftDef) return std::unexpected(ShiftDef.error()); auto ShiftUse = Shift->addUse(Current); if (!ShiftUse) return std::unexpected(ShiftUse.error()); auto ShiftAmount = Shift->addImmediate(12); if (!ShiftAmount) return std::unexpected(ShiftAmount.error()); auto BuiltShift = Shift->build(); if (!BuiltShift) return std::unexpected(BuiltShift.error()); const bool IsFinal = Index == 0; Register AddDestination; if (IsFinal) { AddDestination = Destination; } else { auto AddTemporary = createTemporary(Function, *Type, Target); if (!AddTemporary) return std::unexpected(AddTemporary.error()); AddDestination = *AddTemporary; } auto Add = Builder.setInsertPointAfter(**BuiltShift); if (!Add) return std::unexpected(Add.error()); auto AddInstruction = Builder.buildInstr(targetOpcode(AddImmediate)); if (!AddInstruction) return std::unexpected(AddInstruction.error()); auto AddDef = AddInstruction->addDef(AddDestination); if (!AddDef) return std::unexpected(AddDef.error()); auto AddUse = AddInstruction->addUse(*ShiftResult); if (!AddUse) return std::unexpected(AddUse.error()); auto AddImmediateResult = AddInstruction->addImmediate(Digits[Index]); if (!AddImmediateResult) return std::unexpected(AddImmediateResult.error()); auto BuiltAdd = AddInstruction->build(); if (!BuiltAdd) return std::unexpected(BuiltAdd.error()); Current = AddDestination; } return {}; } [[nodiscard]] static Expected selectFloatingConstant(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target, std::optional Type) { if (!Type || !isRVFloatType(*Type)) return makeUnexpected(ErrorCode::Unsupported, "RISC-V floating constant requires f32 or f64"); const MachineOperand *Immediate = Instruction.operandAt(1); const auto *Floating = Immediate == nullptr ? nullptr : Immediate->getIf(); const std::optional Semantics = Type->floatSemantics(); if (Floating == nullptr || !Semantics || Floating->Bits.semantics() != *Semantics) return makeUnexpected(ErrorCode::VerificationError, "RISC-V floating constant has mismatched bit semantics"); const MachineOperand *DestinationOperand = Instruction.operandAt(0); const Register *Destination = DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); if (Destination == nullptr || !Destination->isVirtual()) return makeUnexpected(ErrorCode::VerificationError, "RISC-V floating constant must define a virtual register"); const Register DestinationRegister = *Destination; const std::uint32_t BitWidth = Type->scalarBitWidth(); const std::uint64_t Bits = Floating->Bits.words().empty() ? 0 : Floating->Bits.words().front(); auto Integer = IntegerImmediate::fromUnsigned(64, Bits); if (!Integer) return std::unexpected(Integer.error()); auto IntegerType = LLT::makeInteger(64); if (!IntegerType) return std::unexpected(IntegerType.error()); auto Temporary = createTemporary(Function, *IntegerType, Target); if (!Temporary) return std::unexpected(Temporary.error()); auto SetOpcode = Instruction.setOpcode(opcode(GenericOpcode::G_CONSTANT)); if (!SetOpcode) return std::unexpected(SetOpcode.error()); auto SetDestination = Instruction.operandAt(0)->setRegister(*Temporary); if (!SetDestination) return std::unexpected(SetDestination.error()); auto EraseImmediate = Instruction.eraseOperand(1); if (!EraseImmediate) return std::unexpected(EraseImmediate.error()); auto AddInteger = Instruction.addOperand(MachineOperand::integerImmediate(*Integer)); if (!AddInteger) return std::unexpected(AddInteger.error()); auto IntegerSelection = selectIntegerConstant(Instruction, Function, Target, *IntegerType); if (!IntegerSelection) return std::unexpected(IntegerSelection.error()); MachineInstr *IntegerDefinition = nullptr; for (MachineOperand &DefOperand : Function.registerInfo().defOperands(*Temporary)) { IntegerDefinition = DefOperand.parent(); break; } if (IntegerDefinition == nullptr) return makeUnexpected(ErrorCode::InternalError, "RISC-V floating constant lost its integer materialization"); MachineIRBuilder Builder(Function, Target); auto InsertPoint = Builder.setInsertPointAfter(*IntegerDefinition); if (!InsertPoint) return std::unexpected(InsertPoint.error()); const InstructionOpcode Move = BitWidth == 32 ? InstructionOpcode::FMV_W_X : InstructionOpcode::FMV_D_X; auto MoveInstruction = Builder.buildInstr(targetOpcode(Move)); if (!MoveInstruction) return std::unexpected(MoveInstruction.error()); auto MoveDef = MoveInstruction->addDef(DestinationRegister); if (!MoveDef) return std::unexpected(MoveDef.error()); auto MoveUse = MoveInstruction->addUse(*Temporary); if (!MoveUse) return std::unexpected(MoveUse.error()); auto BuiltMove = MoveInstruction->build(); if (!BuiltMove) return std::unexpected(BuiltMove.error()); return {}; } [[nodiscard]] static Expected selectRoundingBinary(MachineInstr &Instruction, std::optional Type, InstructionOpcode Single, InstructionOpcode Double) { auto RoundingMode = appendImmediate(Instruction, 0); if (!RoundingMode) return std::unexpected(RoundingMode.error()); if (Type && Type->isFloat() && Type->scalarBitWidth() == 32) return Instruction.setOpcode(targetOpcode(Single)); if (Type && Type->isFloat() && Type->scalarBitWidth() == 64) return Instruction.setOpcode(targetOpcode(Double)); return makeUnexpected(ErrorCode::Unsupported, "RISC-V cannot select this floating binary type"); } [[nodiscard]] static Expected selectRoundingTernary(MachineInstr &Instruction, std::optional Type, InstructionOpcode Single, InstructionOpcode Double) { return selectRoundingBinary(Instruction, Type, Single, Double); } [[nodiscard]] static Expected selectRoundingUnary(MachineInstr &Instruction, std::optional Type, InstructionOpcode Single, InstructionOpcode Double) { return selectRoundingBinary(Instruction, Type, Single, Double); } [[nodiscard]] static Expected selectSignInjection(MachineInstr &Instruction, std::optional Type, InstructionOpcode Single, InstructionOpcode Double) { const MachineOperand *Source = Instruction.operandAt(1); const Register *SourceReg = Source == nullptr ? nullptr : Source->getIf(); if (SourceReg == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V floating sign operation requires a register source"); auto Duplicate = insertVirtualUse(Instruction, 2, *SourceReg); if (!Duplicate) return std::unexpected(Duplicate.error()); if (Type && Type->isFloat() && Type->scalarBitWidth() == 32) return Instruction.setOpcode(targetOpcode(Single)); if (Type && Type->isFloat() && Type->scalarBitWidth() == 64) return Instruction.setOpcode(targetOpcode(Double)); return makeUnexpected(ErrorCode::Unsupported, "RISC-V cannot select this floating unary type"); } }; class RISCVSSAInfo final : public TargetSSAInfo { public: [[nodiscard]] Expected isFallthroughEdge(const MachineBasicBlock &From, const MachineBasicBlock &To, const MachineFunction &, const TargetDescription &Target) const override { if (From.next() != &To || std::ranges::none_of(From.successors(), [&To](const MachineSuccessor &Successor) { return Successor.Block == &To; })) { return false; } bool CanFallThrough = true; for (const MachineInstr *Instruction = From.firstInstruction(); Instruction != nullptr; Instruction = Instruction->next()) { const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); if (Descriptor == nullptr) { return makeUnexpected(ErrorCode::VerificationError, "RISC-V fallthrough analysis encountered an unknown opcode"); } if (!Descriptor->Flags.has(InstructionFlag::Terminator)) continue; CanFallThrough = isConditionalBranch(Instruction->opcode()); } return CanFallThrough; } [[nodiscard]] Expected rewriteBranchTarget(MachineBasicBlock &Block, MachineBasicBlock &OldTarget, MachineBasicBlock &NewTarget, MachineFunction &Function, const TargetDescription &Target) const override { if (&Block.function() != &Function || &OldTarget.function() != &Function || &NewTarget.function() != &Function) { return makeUnexpected(ErrorCode::InvalidArgument, "RISC-V branch rewrite references another machine function"); } bool Changed = false; for (MachineInstr *Instruction = Block.firstInstruction(); Instruction != nullptr; Instruction = Instruction->next()) { const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); if (Descriptor == nullptr) { return makeUnexpected(ErrorCode::VerificationError, "RISC-V branch rewrite encountered an unknown opcode"); } if (!Descriptor->Flags.has(InstructionFlag::Branch) || Descriptor->Flags.has(InstructionFlag::IndirectBranch)) { continue; } for (std::size_t Index = 0; Index < Instruction->operandCount(); ++Index) { MachineOperand *Operand = Instruction->operandAt(Index); const BasicBlockOperand *TargetOperand = Operand == nullptr ? nullptr : Operand->getIf(); if (TargetOperand == nullptr || TargetOperand->Block != &OldTarget) continue; auto Rewritten = Operand->setBasicBlock(NewTarget); if (!Rewritten) return std::unexpected(Rewritten.error()); Changed = true; } } return Changed; } [[nodiscard]] Expected insertUnconditionalBranch(MachineBasicBlock &Block, MachineBasicBlock &TargetBlock, MachineFunction &Function, const TargetDescription &Target) const override { if (&Block.function() != &Function || &TargetBlock.function() != &Function) { return makeUnexpected(ErrorCode::InvalidArgument, "RISC-V branch insertion references another machine function"); } auto Zero = physicalRegister(PhysicalRegisterID::X0); if (!Zero) return std::unexpected(Zero.error()); MachineIRBuilder Builder(Function, Target); auto Point = Builder.setInsertPoint(Block); if (!Point) return std::unexpected(Point.error()); auto Branch = Builder.buildInstr(targetOpcode(InstructionOpcode::JAL)); if (!Branch) return std::unexpected(Branch.error()); auto Link = Branch->addDef(Register::fromPhysical(*Zero)); if (!Link) return std::unexpected(Link.error()); auto Destination = Branch->addBasicBlock(TargetBlock); if (!Destination) return std::unexpected(Destination.error()); auto Built = Branch->build(); if (!Built) return std::unexpected(Built.error()); return {}; } [[nodiscard]] Expected validateCopy(Register Destination, SubRegIndex DestinationSubReg, Register Source, SubRegIndex SourceSubReg, const MachineFunction &Function, const TargetDescription &Target) const override { if (Destination.isNoRegister() || Source.isNoRegister()) { return makeUnexpected(ErrorCode::VerificationError, "RISC-V COPY requires two nonzero registers"); } if (DestinationSubReg.value() != 0 || SourceSubReg.value() != 0) { return makeUnexpected(ErrorCode::Unsupported, "RV64GC does not model register subindices"); } auto DestinationShape = copyShape(Destination, Function, Target); if (!DestinationShape) return std::unexpected(DestinationShape.error()); auto SourceShape = copyShape(Source, Function, Target); if (!SourceShape) return std::unexpected(SourceShape.error()); if (DestinationShape->Bank == SourceShape->Bank) { if (DestinationShape->Width && SourceShape->Width && DestinationShape->Width != SourceShape->Width) { return makeUnexpected(ErrorCode::Unsupported, "RISC-V same-bank COPY has incompatible register widths"); } return {}; } const RegisterBankID GPR = RegisterBankID::fromValue(static_cast(RegisterBank::GPR)); const RegisterBankID FPR = RegisterBankID::fromValue(static_cast(RegisterBank::FPR)); const bool IsCrossBankMove = (DestinationShape->Bank == GPR && SourceShape->Bank == FPR) || (DestinationShape->Bank == FPR && SourceShape->Bank == GPR); if (!IsCrossBankMove) { return makeUnexpected(ErrorCode::Unsupported, "RISC-V COPY crosses unsupported register banks"); } auto SupportedWidth = [](std::optional Width) { return !Width || *Width == 32 || *Width == 64; }; if (!SupportedWidth(DestinationShape->Width) || !SupportedWidth(SourceShape->Width)) { return makeUnexpected(ErrorCode::Unsupported, "RISC-V GPR/FPR COPY requires 32-bit or 64-bit values"); } return {}; } private: struct CopyShape final { RegisterBankID Bank; std::optional Width; }; [[nodiscard]] static bool isConditionalBranch(Opcode Value) noexcept { return Value == targetOpcode(InstructionOpcode::BEQ) || Value == targetOpcode(InstructionOpcode::BNE) || Value == targetOpcode(InstructionOpcode::BLT) || Value == targetOpcode(InstructionOpcode::BGE) || Value == targetOpcode(InstructionOpcode::BLTU) || Value == targetOpcode(InstructionOpcode::BGEU); } [[nodiscard]] static Expected copyShape(Register Reg, const MachineFunction &Function, const TargetDescription &Target) { if (Reg.isVirtual()) { auto Info = Function.registerInfo().virtualRegisterInfo(Reg); if (!Info) return std::unexpected(Info.error()); const std::optional &Bank = Info->get().Bank; const std::optional &RegClass = Info->get().RegClass; if (!Bank || !RegClass) { return makeUnexpected(ErrorCode::VerificationError, "RISC-V COPY virtual register is not fully constrained"); } const RegisterClassDesc *Class = Target.registers().registerClass(RegClass.value()); if (Class == nullptr) { return makeUnexpected(ErrorCode::VerificationError, "RISC-V COPY references an unknown register class"); } return CopyShape{Bank.value(), Class->SizeInBits}; } const std::optional Physical = Reg.physicalRegister(); if (!Physical || !Target.registers().isKnownPhysicalRegister(*Physical)) { return makeUnexpected(ErrorCode::VerificationError, "RISC-V COPY references an unknown physical register"); } for (const RegisterBankDesc &Bank : Target.registers().registerBanks()) { if (Target.registers().bankContainsRegister(Bank.ID, *Physical)) return CopyShape{Bank.ID, std::nullopt}; } return makeUnexpected(ErrorCode::VerificationError, "RISC-V physical register is not assigned to a register bank"); } }; } // namespace const TargetLegalizerInfo &legalizerInfo() noexcept { static const RISCVLegalizerInfo Value(RISCVSubtargetFeatures::rv64gc()); return Value; } std::unique_ptr createLegalizerInfo(RISCVSubtargetFeatures Features) { return std::make_unique(Features); } const TargetRegisterBankInfo ®isterBankInfo() noexcept { static const RISCVRegisterBankInfo Value; return Value; } const TargetInstructionSelector &instructionSelector() noexcept { static const RISCVInstructionSelector Value; return Value; } const TargetSSAInfo &ssaInfo() noexcept { static const RISCVSSAInfo Value; return Value; } } // namespace rcc::riscv