// SPDX-License-Identifier: GPL-3.0-or-later #include "RISCVLegalizerInfo.h" #include "RISCVCallLowering.h" #include "RISCVImmediateMaterialization.h" #include "rcc/CodeGen/GenericOpcodes.h" #include "rcc/CodeGen/LowLevelType.h" #include "rcc/CodeGen/MachineBasicBlock.h" #include "rcc/CodeGen/MachineFunction.h" #include "rcc/CodeGen/MachineIRBuilder.h" #include "rcc/CodeGen/MachineInstr.h" #include "rcc/CodeGen/MachineMemOperand.h" #include "rcc/CodeGen/MachineOperand.h" #include "rcc/CodeGen/MachineRegisterInfo.h" #include "rcc/CodeGen/TargetDescription.h" #include "rcc/Target/RISCV/RISCVSubtarget.h" #include "rcc/Target/RISCV/RISCVTarget.h" #include #include #include #include #include #include #include #include #include #include namespace rcc::riscv { namespace { [[nodiscard]] Opcode targetOpcode(InstructionOpcode Value) noexcept { return Opcode::fromValue(static_cast(Value)); } [[nodiscard]] std::optional registerType(const MachineFunction &Function, const MachineOperand &Operand) { const Register *Reg = Operand.getIf(); if (Reg == nullptr || !Reg->isVirtual()) return std::nullopt; auto Info = Function.registerInfo().virtualRegisterInfo(*Reg); if (!Info) return std::nullopt; return Info->get().Type; } [[nodiscard]] std::optional resultType(const MachineFunction &Function, const MachineInstr &Instruction) { const MachineOperand *Result = Instruction.operandAt(0); return Result == nullptr ? std::nullopt : registerType(Function, *Result); } [[nodiscard]] std::optional operandType(const MachineFunction &Function, const MachineInstr &Instruction, std::size_t Index) { const MachineOperand *Operand = Instruction.operandAt(Index); return Operand == nullptr ? std::nullopt : registerType(Function, *Operand); } [[nodiscard]] bool isRVFloatType(LLT Type) noexcept { if (!Type.isFloat()) return false; const std::optional Semantics = Type.floatSemantics(); return Semantics && (*Semantics == FloatSemantics::IEEESingle || *Semantics == FloatSemantics::IEEEDouble); } [[nodiscard]] bool isSupportedIntegerType(LLT Type) noexcept { return ((Type.isInteger() || Type.isAnyScalar()) && Type.scalarBitWidth() != 0 && Type.scalarBitWidth() <= 64) || (Type.isPointer() && Type.scalarBitWidth() == 64); } [[nodiscard]] bool isSupportedIntegerScalar(LLT Type) noexcept { return (Type.isInteger() || Type.isAnyScalar()) && Type.scalarBitWidth() != 0 && Type.scalarBitWidth() <= 64; } [[nodiscard]] bool isNativeIntegerScalar(LLT Type) noexcept { return (Type.isInteger() || Type.isAnyScalar()) && (Type.scalarBitWidth() == 32 || Type.scalarBitWidth() == 64); } [[nodiscard]] bool isScalarGPRType(LLT Type) noexcept { return ((Type.isInteger() || Type.isAnyScalar()) && Type.scalarBitWidth() != 0 && Type.scalarBitWidth() <= 64) || (Type.isPointer() && Type.scalarBitWidth() == 64); } [[nodiscard]] bool isI1Type(LLT Type) noexcept { return (Type.isInteger() || Type.isAnyScalar()) && Type.scalarBitWidth() == 1; } [[nodiscard]] bool isScalarIntegerExtension(LLT Destination, LLT Source) noexcept { return (Destination.isInteger() || Destination.isAnyScalar()) && (Source.isInteger() || Source.isAnyScalar()) && Destination.scalarBitWidth() != 0 && Source.scalarBitWidth() != 0 && Source.scalarBitWidth() < Destination.scalarBitWidth() && Destination.scalarBitWidth() <= 64; } [[nodiscard]] Expected extendIntegerImmediate(const IntegerImmediate &Value, std::uint32_t DestinationWidth, bool SignExtend) { if (DestinationWidth == 0 || DestinationWidth > 64) return makeUnexpected(ErrorCode::Unsupported, "RISC-V constant extension requires a destination no wider than 64 bits"); std::uint64_t Raw = Value.words().empty() ? 0 : Value.words().front(); const std::uint32_t SourceWidth = Value.bitWidth(); if (SignExtend && Value.isNegative() && SourceWidth < 64) Raw |= ~std::uint64_t{0} << SourceWidth; if (DestinationWidth < 64) Raw &= (std::uint64_t{1} << DestinationWidth) - 1U; return IntegerImmediate::create(DestinationWidth, std::span(&Raw, 1)); } [[nodiscard]] Expected foldConstantExtension(MachineInstr &Instruction, MachineFunction &Function, GenericOpcode Generic) { const MachineOperand *SourceOperand = Instruction.operandAt(1); const MachineOperand *DestinationOperand = Instruction.operandAt(0); const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); const Register *Destination = DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); if (Source == nullptr || Destination == nullptr || !Source->isVirtual() || !Destination->isVirtual()) { return makeUnexpected(ErrorCode::VerificationError, "RISC-V constant extension requires virtual-register operands"); } const Register SourceRegister = *Source; const Register DestinationRegister = *Destination; const MachineOperand *SourceDefinition = Function.registerInfo().getUniqueVRegDef(SourceRegister); const MachineInstr *SourceDef = SourceDefinition == nullptr ? nullptr : SourceDefinition->parent(); if (SourceDef == nullptr || SourceDef->opcode() != opcode(GenericOpcode::G_CONSTANT)) return makeUnexpected(ErrorCode::Unsupported, "RISC-V constant extension requires a single constant definition"); const MachineOperand *ConstantOperand = SourceDef->operandAt(1); const auto *Integer = ConstantOperand == nullptr ? nullptr : ConstantOperand->getIf(); if (Integer == nullptr) return makeUnexpected(ErrorCode::Unsupported, "RISC-V constant extension requires an integer constant"); const auto DestinationInfo = Function.registerInfo().virtualRegisterInfo(*Destination); if (!DestinationInfo || !DestinationInfo->get().Type) return makeUnexpected(ErrorCode::VerificationError, "RISC-V constant extension destination has no type"); const std::uint32_t DestinationWidth = DestinationInfo->get().Type->scalarBitWidth(); const bool SignExtend = Generic != GenericOpcode::G_ZEXT; auto Extended = extendIntegerImmediate(Integer->Value, DestinationWidth, SignExtend); if (!Extended) return std::unexpected(Extended.error()); if (Function.registerInfo().hasOneUse(SourceRegister)) { MachineInstr *Def = const_cast(SourceDef); if (auto Replaced = Def->operandAt(0)->setRegister(DestinationRegister); !Replaced) return std::unexpected(Replaced.error()); auto Erased = Def->eraseOperand(1); if (!Erased) return std::unexpected(Erased.error()); auto Added = Def->addOperand(MachineOperand::integerImmediate(std::move(*Extended))); if (!Added) return std::unexpected(Added.error()); MachineBasicBlock *Parent = Instruction.parent(); if (Parent == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V constant extension is detached during folding"); return Parent->removeInstruction(Instruction); } auto SetOpcode = Instruction.setOpcode(opcode(GenericOpcode::G_CONSTANT)); if (!SetOpcode) return std::unexpected(SetOpcode.error()); while (Instruction.operandCount() > 1) { auto Erased = Instruction.eraseOperand(1); if (!Erased) return std::unexpected(Erased.error()); } auto Added = Instruction.addOperand(MachineOperand::integerImmediate(std::move(*Extended))); if (!Added) return std::unexpected(Added.error()); if (Function.registerInfo().useEmpty(SourceRegister)) { MachineInstr *Def = const_cast(SourceDef); MachineBasicBlock *Parent = Def->parent(); if (Parent == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V constant extension source definition is detached"); auto Removed = Parent->removeInstruction(*Def); if (!Removed) return std::unexpected(Removed.error()); // Keep the source register record as a tombstone-free historical vreg until // the legalizer worklist transaction has completed. } return {}; } struct WideConstantUnmergePlan final { MachineInstr *Extension = nullptr; MachineInstr *Unmerge = nullptr; std::vector Outputs; bool SignExtend = false; }; [[nodiscard]] MachineInstr *singleRegisterUse(const MachineFunction &Function, Register Reg) { MachineInstr *UseInstruction = nullptr; auto &RegisterInfo = const_cast(Function.registerInfo()); for (MachineOperand &Operand : RegisterInfo.useOperands(Reg)) { if (Operand.parent() == nullptr) continue; if (UseInstruction != nullptr) return nullptr; UseInstruction = Operand.parent(); } return UseInstruction; } [[nodiscard]] std::optional wideConstantUnmergePlan(const MachineInstr &Constant, const MachineFunction &Function) { const MachineOperand *ResultOperand = Constant.operandAt(0); const Register *Result = ResultOperand == nullptr ? nullptr : ResultOperand->getIf(); if (Result == nullptr || !Result->isVirtual()) return std::nullopt; MachineInstr *FirstUse = singleRegisterUse(Function, *Result); if (FirstUse == nullptr) return std::nullopt; MachineInstr *Extension = nullptr; MachineInstr *Unmerge = FirstUse; if (FirstUse->opcode() == opcode(GenericOpcode::G_ANYEXT) || FirstUse->opcode() == opcode(GenericOpcode::G_SEXT) || FirstUse->opcode() == opcode(GenericOpcode::G_ZEXT)) { Extension = FirstUse; const MachineOperand *ExtendedOperand = Extension->operandAt(0); const Register *Extended = ExtendedOperand == nullptr ? nullptr : ExtendedOperand->getIf(); if (Extended == nullptr || !Extended->isVirtual()) return std::nullopt; Unmerge = singleRegisterUse(Function, *Extended); if (Unmerge == nullptr) return std::nullopt; } if (Unmerge->opcode() != opcode(GenericOpcode::G_UNMERGE_VALUES) || Unmerge->operandCount() < 3) return std::nullopt; WideConstantUnmergePlan Plan; Plan.Extension = Extension; Plan.Unmerge = Unmerge; Plan.SignExtend = Extension != nullptr && Extension->opcode() != opcode(GenericOpcode::G_ZEXT); for (std::size_t Index = 0; Index + 1U < Unmerge->operandCount(); ++Index) { const MachineOperand *Operand = Unmerge->operandAt(Index); const Register *Reg = Operand == nullptr ? nullptr : Operand->getIf(); if (Reg == nullptr || !Reg->isVirtual() || !Operand->isDef()) return std::nullopt; const auto Info = Function.registerInfo().virtualRegisterInfo(*Reg); if (!Info || !Info->get().Type || Info->get().Type->scalarBitWidth() == 0 || Info->get().Type->scalarBitWidth() > 64 || (!Info->get().Type->isInteger() && !Info->get().Type->isAnyScalar())) return std::nullopt; Plan.Outputs.push_back(*Reg); } const MachineOperand *SourceOperand = Unmerge->operandAt(Unmerge->operandCount() - 1U); const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); const Register *ExpectedSource = Result; if (Extension != nullptr) { const MachineOperand *ExtendedOperand = Extension->operandAt(0); ExpectedSource = ExtendedOperand == nullptr ? nullptr : ExtendedOperand->getIf(); } if (Source == nullptr || ExpectedSource == nullptr || *Source != *ExpectedSource) return std::nullopt; return Plan; } [[nodiscard]] Expected wideConstantWord(const IntegerImmediate &Value, std::size_t WordIndex, std::uint32_t Width, bool SignExtend) { std::uint64_t Raw = 0; if (WordIndex < Value.words().size()) Raw = Value.words()[WordIndex]; else if (SignExtend && Value.isNegative()) Raw = std::numeric_limits::max(); const std::size_t WordBitOffset = WordIndex * 64U; if (SignExtend && Value.isNegative() && WordBitOffset < Value.bitWidth()) { const std::uint32_t Remaining = Value.bitWidth() - static_cast(WordBitOffset); if (Remaining < 64) Raw |= ~std::uint64_t{0} << Remaining; } if (Width < 64) Raw &= (std::uint64_t{1} << Width) - 1U; return IntegerImmediate::create(Width, std::span(&Raw, 1)); } [[nodiscard]] Expected foldWideConstant(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target) { const auto Plan = wideConstantUnmergePlan(Instruction, Function); if (!Plan) return makeUnexpected(ErrorCode::Unsupported, "RISC-V wide constant requires a G_UNMERGE_VALUES use"); const MachineOperand *ImmediateOperand = Instruction.operandAt(1); const auto *Integer = ImmediateOperand == nullptr ? nullptr : ImmediateOperand->getIf(); if (Integer == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide constant has no integer immediate"); std::vector Words; Words.reserve(Plan->Outputs.size()); for (std::size_t Index = 0; Index < Plan->Outputs.size(); ++Index) { const auto Info = Function.registerInfo().virtualRegisterInfo(Plan->Outputs[Index]); if (!Info || !Info->get().Type) return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide constant output has no virtual-register type"); auto Word = wideConstantWord(Integer->Value, Index, Info->get().Type->scalarBitWidth(), Plan->SignExtend); if (!Word) return std::unexpected(Word.error()); Words.push_back(std::move(*Word)); } std::vector Representatives(Plan->Outputs.size()); for (std::size_t Index = 0; Index < Representatives.size(); ++Index) { Representatives[Index] = Index; for (std::size_t Previous = 0; Previous < Index; ++Previous) { if (Words[Previous] == Words[Index]) { Representatives[Index] = Previous; break; } } } const Register FirstOutput = Plan->Outputs.front(); auto Retargeted = Instruction.operandAt(0)->setRegister(FirstOutput); if (!Retargeted) return std::unexpected(Retargeted.error()); auto SetOpcode = Instruction.setOpcode(opcode(GenericOpcode::G_CONSTANT)); if (!SetOpcode) return std::unexpected(SetOpcode.error()); while (Instruction.operandCount() > 1) { auto Erased = Instruction.eraseOperand(1); if (!Erased) return std::unexpected(Erased.error()); } auto Added = Instruction.addOperand(MachineOperand::integerImmediate(std::move(Words.front()))); if (!Added) return std::unexpected(Added.error()); MachineBasicBlock *Block = Instruction.parent(); if (Block == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide constant is detached during legalization"); if (Plan->Unmerge != &Instruction) { auto RemovedUnmerge = Plan->Unmerge->parent()->removeInstruction(*Plan->Unmerge); if (!RemovedUnmerge) return std::unexpected(RemovedUnmerge.error()); } if (Plan->Extension != nullptr) { auto RemovedExtension = Plan->Extension->parent()->removeInstruction(*Plan->Extension); if (!RemovedExtension) return std::unexpected(RemovedExtension.error()); } for (std::size_t Index = 1; Index < Plan->Outputs.size(); ++Index) { if (Representatives[Index] == Index) continue; auto Replaced = Function.registerInfo().replaceRegWith(Plan->Outputs[Index], Plan->Outputs[Representatives[Index]]); if (!Replaced) return std::unexpected(Replaced.error()); } MachineInstr *Last = &Instruction; for (std::size_t Index = 1; Index < Plan->Outputs.size(); ++Index) { if (Representatives[Index] != Index) continue; MachineIRBuilder Builder(Function, Target); auto Point = Builder.setInsertPointAfter(*Last); if (!Point) return std::unexpected(Point.error()); auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_CONSTANT)); if (!Pending) return std::unexpected(Pending.error()); auto Def = Pending->addDef(Plan->Outputs[Index]); if (!Def) return std::unexpected(Def.error()); auto Value = Pending->addIntegerImmediate(std::move(Words[Index])); if (!Value) return std::unexpected(Value.error()); auto Built = Pending->build(); if (!Built) return std::unexpected(Built.error()); Last = *Built; } return {}; } [[nodiscard]] Expected buildIntegerConstantBefore(MachineIRBuilder &Builder, MachineInstr &Anchor, std::uint32_t Width, std::uint64_t Value); [[nodiscard]] Expected buildBinaryBefore(MachineIRBuilder &Builder, MachineInstr &Anchor, GenericOpcode Generic, LLT Type, Register Left, Register Right); [[nodiscard]] Expected buildUnaryBefore(MachineIRBuilder &Builder, MachineInstr &Anchor, GenericOpcode Generic, LLT Type, Register Source); [[nodiscard]] Expected eraseInstruction(MachineInstr &Instruction); [[nodiscard]] Expected rewriteUnary(MachineInstr &Instruction, GenericOpcode Generic, Register Destination, Register Source); struct NarrowAnyExtBinaryPlan final { MachineInstr *Binary = nullptr; GenericOpcode Generic = GenericOpcode::Invalid; std::array WideInputs{}; std::vector Artifacts; std::vector Constants; std::uint32_t SourceWidth = 0; }; [[nodiscard]] std::optional narrowAnyExtBinaryPlan(const MachineInstr &Extension, const MachineFunction &Function) { const MachineOperand *DestinationOperand = Extension.operandAt(0); const MachineOperand *SourceOperand = Extension.operandAt(1); const Register *Destination = DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); if (Destination == nullptr || Source == nullptr || !Destination->isVirtual() || !Source->isVirtual() || singleRegisterUse(Function, *Source) != &Extension) return std::nullopt; const auto DestinationInfo = Function.registerInfo().virtualRegisterInfo(*Destination); const auto SourceInfo = Function.registerInfo().virtualRegisterInfo(*Source); if (!DestinationInfo || !SourceInfo || !DestinationInfo->get().Type || !SourceInfo->get().Type) return std::nullopt; const LLT DestinationType = *DestinationInfo->get().Type; const LLT SourceType = *SourceInfo->get().Type; if (!isScalarIntegerExtension(DestinationType, SourceType) || DestinationType.scalarBitWidth() != 64) return std::nullopt; const MachineOperand *SourceDefinition = Function.registerInfo().getUniqueVRegDef(*Source); MachineInstr *Binary = SourceDefinition == nullptr ? nullptr : const_cast(SourceDefinition->parent()); if (Binary == nullptr || Binary->operandCount() != 3) return std::nullopt; const GenericOpcode Generic = static_cast(Binary->opcode().value()); if (Generic != GenericOpcode::G_ADD && Generic != GenericOpcode::G_SUB && Generic != GenericOpcode::G_MUL && Generic != GenericOpcode::G_AND && Generic != GenericOpcode::G_OR && Generic != GenericOpcode::G_XOR && Generic != GenericOpcode::G_SHL) return std::nullopt; NarrowAnyExtBinaryPlan Plan; Plan.Binary = Binary; Plan.Generic = Generic; Plan.SourceWidth = SourceType.scalarBitWidth(); const auto AddArtifact = [&Plan](MachineInstr *Artifact) { if (std::ranges::find(Plan.Artifacts, Artifact) == Plan.Artifacts.end()) Plan.Artifacts.push_back(Artifact); }; const auto TruncatedWideSource = [&](MachineInstr *Truncation, MachineInstr *OnlyUse) -> std::optional { if (Truncation == nullptr || Truncation->opcode() != opcode(GenericOpcode::G_TRUNC) || Truncation->operandCount() != 2) return std::nullopt; const MachineOperand *NarrowOperand = Truncation->operandAt(0); const Register *Narrow = NarrowOperand == nullptr ? nullptr : NarrowOperand->getIf(); const MachineOperand *WideOperand = Truncation->operandAt(1); const Register *Wide = WideOperand == nullptr ? nullptr : WideOperand->getIf(); const std::optional WideType = WideOperand == nullptr ? std::nullopt : registerType(Function, *WideOperand); if (Narrow == nullptr || Wide == nullptr || !Wide->isVirtual() || !WideType || *WideType != DestinationType || singleRegisterUse(Function, *Narrow) != OnlyUse) return std::nullopt; return *Wide; }; for (std::size_t Index = 0; Index != Plan.WideInputs.size(); ++Index) { const MachineOperand *Operand = Binary->operandAt(Index + 1U); const Register *Reg = Operand == nullptr ? nullptr : Operand->getIf(); if (Reg == nullptr || !Reg->isVirtual()) return std::nullopt; const auto Info = Function.registerInfo().virtualRegisterInfo(*Reg); if (!Info || !Info->get().Type) return std::nullopt; if (*Info->get().Type == DestinationType) { Plan.WideInputs[Index] = *Reg; continue; } if (*Info->get().Type != SourceType) return std::nullopt; const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(*Reg); MachineInstr *Def = Definition == nullptr ? nullptr : const_cast(Definition->parent()); if (auto Wide = TruncatedWideSource(Def, Binary)) { Plan.WideInputs[Index] = *Wide; AddArtifact(Def); continue; } if (Def != nullptr && (Def->opcode() == opcode(GenericOpcode::COPY) || Def->opcode() == opcode(GenericOpcode::G_ANYEXT)) && Def->operandCount() == 2 && singleRegisterUse(Function, *Reg) == Binary) { const MachineOperand *NarrowOperand = Def->operandAt(1); const Register *Narrow = NarrowOperand == nullptr ? nullptr : NarrowOperand->getIf(); const MachineOperand *NarrowDefinition = Narrow == nullptr ? nullptr : Function.registerInfo().getUniqueVRegDef(*Narrow); MachineInstr *Truncation = NarrowDefinition == nullptr ? nullptr : const_cast(NarrowDefinition->parent()); if (auto Wide = TruncatedWideSource(Truncation, Def)) { Plan.WideInputs[Index] = *Wide; AddArtifact(Def); AddArtifact(Truncation); continue; } } if (Def != nullptr && Def->opcode() == opcode(GenericOpcode::G_CONSTANT) && singleRegisterUse(Function, *Reg) == Binary) { Plan.WideInputs[Index] = *Reg; Plan.Constants.push_back(Def); continue; } return std::nullopt; } return Plan; } [[nodiscard]] Expected widenConstantToRV64(MachineInstr &Instruction, MachineFunction &Function) { const MachineOperand *DestinationOperand = Instruction.operandAt(0); const Register *Destination = DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); const MachineOperand *ImmediateOperand = Instruction.operandAt(1); const auto *Immediate = ImmediateOperand == nullptr ? nullptr : ImmediateOperand->getIf(); if (Destination == nullptr || !Destination->isVirtual() || Immediate == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V narrow constant legalization has invalid operands"); auto Type = LLT::makeAnyScalar(64); if (!Type) return std::unexpected(Type.error()); auto SetType = Function.registerInfo().setType(*Destination, *Type); if (!SetType) return std::unexpected(SetType.error()); auto Extended = extendIntegerImmediate(Immediate->Value, 64, true); if (!Extended) return std::unexpected(Extended.error()); auto Erased = Instruction.eraseOperand(1); if (!Erased) return std::unexpected(Erased.error()); auto Added = Instruction.addOperand(MachineOperand::integerImmediate(std::move(*Extended))); if (!Added) return std::unexpected(Added.error()); return {}; } [[nodiscard]] Expected widenAtomicCompareExchangeSource(MachineInstr &Instruction, MachineFunction &Function, MachineIRBuilder &Builder, Register Source, LLT WideType) { const auto SourceInfo = Function.registerInfo().virtualRegisterInfo(Source); if (!SourceInfo || !SourceInfo->get().Type) return makeUnexpected(ErrorCode::VerificationError, "RISC-V compare-exchange source has no scalar type"); if (SourceInfo->get().Type->scalarBitWidth() == WideType.scalarBitWidth()) return Source; const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(Source); MachineInstr *Def = Definition == nullptr ? nullptr : const_cast(Definition->parent()); if (Def != nullptr && Def->opcode() == opcode(GenericOpcode::G_CONSTANT) && singleRegisterUse(Function, Source) == &Instruction) { auto Widened = widenConstantToRV64(*Def, Function); if (!Widened) return std::unexpected(Widened.error()); return Source; } return buildUnaryBefore(Builder, Instruction, GenericOpcode::G_ANYEXT, WideType, Source); } [[nodiscard]] Expected lowerAtomicCompareExchange(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target) { const std::optional OriginalType = resultType(Function, Instruction); const MachineOperand *DestinationOperand = Instruction.operandAt(0); const MachineOperand *CompareOperand = Instruction.operandAt(2); const MachineOperand *NewValueOperand = Instruction.operandAt(3); const Register *Destination = DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); const Register *Compare = CompareOperand == nullptr ? nullptr : CompareOperand->getIf(); const Register *NewValue = NewValueOperand == nullptr ? nullptr : NewValueOperand->getIf(); if (!OriginalType || Destination == nullptr || Compare == nullptr || NewValue == nullptr || OriginalType->scalarBitWidth() >= 64) return makeUnexpected(ErrorCode::VerificationError, "RISC-V narrow compare-exchange has invalid operands"); const Register OriginalDestination = *Destination; auto WideType = LLT::makeAnyScalar(64); if (!WideType) return std::unexpected(WideType.error()); MachineIRBuilder Builder(Function, Target); auto WideCompare = widenAtomicCompareExchangeSource(Instruction, Function, Builder, *Compare, *WideType); if (!WideCompare) return std::unexpected(WideCompare.error()); auto WideNewValue = widenAtomicCompareExchangeSource(Instruction, Function, Builder, *NewValue, *WideType); if (!WideNewValue) return std::unexpected(WideNewValue.error()); Register WideDestination; MachineInstr *FoldedExtension = singleRegisterUse(Function, OriginalDestination); if (FoldedExtension != nullptr && FoldedExtension->opcode() == opcode(GenericOpcode::G_ANYEXT) && operandType(Function, *FoldedExtension, 0) .transform([](LLT Type) { return Type.scalarBitWidth() == 64; }) .value_or(false)) { const MachineOperand *WideDestinationOperand = FoldedExtension->operandAt(0); const Register *FoldedDestination = WideDestinationOperand == nullptr ? nullptr : WideDestinationOperand->getIf(); if (FoldedDestination == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V compare-exchange extension has no destination"); WideDestination = *FoldedDestination; } else { auto Created = Function.registerInfo().createVirtualRegister(*WideType); if (!Created) return std::unexpected(Created.error()); WideDestination = *Created; FoldedExtension = nullptr; } if (auto Replaced = Instruction.operandAt(0)->setRegister(WideDestination); !Replaced) return std::unexpected(Replaced.error()); if (auto Replaced = Instruction.operandAt(2)->setRegister(*WideCompare); !Replaced) return std::unexpected(Replaced.error()); if (auto Replaced = Instruction.operandAt(3)->setRegister(*WideNewValue); !Replaced) return std::unexpected(Replaced.error()); if (FoldedExtension != nullptr) return eraseInstruction(*FoldedExtension); auto Point = Builder.setInsertPointAfter(Instruction); if (!Point) return std::unexpected(Point.error()); auto Truncation = Builder.buildInstr(opcode(GenericOpcode::G_TRUNC)); if (!Truncation) return std::unexpected(Truncation.error()); if (auto Added = Truncation->addDef(OriginalDestination); !Added) return std::unexpected(Added.error()); if (auto Added = Truncation->addUse(WideDestination); !Added) return std::unexpected(Added.error()); auto Built = Truncation->build(); if (!Built) return std::unexpected(Built.error()); return {}; } [[nodiscard]] Expected lowerAtomicCompareExchangeWithSuccess(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target) { const MachineOperand *OldValueOperand = Instruction.operandAt(0); const MachineOperand *SuccessOperand = Instruction.operandAt(1); const MachineOperand *AddressOperand = Instruction.operandAt(2); const MachineOperand *CompareOperand = Instruction.operandAt(3); const MachineOperand *NewValueOperand = Instruction.operandAt(4); const Register *OldValue = OldValueOperand == nullptr ? nullptr : OldValueOperand->getIf(); const Register *Success = SuccessOperand == nullptr ? nullptr : SuccessOperand->getIf(); const Register *Address = AddressOperand == nullptr ? nullptr : AddressOperand->getIf(); const Register *Compare = CompareOperand == nullptr ? nullptr : CompareOperand->getIf(); const Register *NewValue = NewValueOperand == nullptr ? nullptr : NewValueOperand->getIf(); const std::optional ValueType = resultType(Function, Instruction); if (OldValue == nullptr || Success == nullptr || Address == nullptr || Compare == nullptr || NewValue == nullptr || !ValueType || Instruction.memOperands().size() != 1) return makeUnexpected(ErrorCode::VerificationError, "RISC-V compare-exchange-with-success has invalid operands"); auto AtomicResult = Function.registerInfo().createVirtualRegister(*ValueType); if (!AtomicResult) return std::unexpected(AtomicResult.error()); MachineIRBuilder Builder(Function, Target); auto Point = Builder.setInsertPointBefore(Instruction); if (!Point) return std::unexpected(Point.error()); auto Atomic = Builder.buildInstr(opcode(GenericOpcode::G_ATOMIC_CMPXCHG)); if (!Atomic) return std::unexpected(Atomic.error()); if (auto Added = Atomic->addDef(*AtomicResult); !Added) return std::unexpected(Added.error()); if (auto Added = Atomic->addUse(*Address); !Added) return std::unexpected(Added.error()); if (auto Added = Atomic->addUse(*Compare); !Added) return std::unexpected(Added.error()); if (auto Added = Atomic->addUse(*NewValue); !Added) return std::unexpected(Added.error()); if (auto Added = Atomic->addMemOperand(*Instruction.memOperands().front()); !Added) return std::unexpected(Added.error()); auto BuiltAtomic = Atomic->build(); if (!BuiltAtomic) return std::unexpected(BuiltAtomic.error()); Point = Builder.setInsertPointBefore(Instruction); if (!Point) return std::unexpected(Point.error()); auto CompareResult = Builder.buildInstr(opcode(GenericOpcode::G_ICMP)); if (!CompareResult) return std::unexpected(CompareResult.error()); if (auto Added = CompareResult->addDef(*Success); !Added) return std::unexpected(Added.error()); if (auto Added = CompareResult->addIntegerPredicate(IntegerPredicate::Equal); !Added) return std::unexpected(Added.error()); if (auto Added = CompareResult->addUse(*AtomicResult); !Added) return std::unexpected(Added.error()); if (auto Added = CompareResult->addUse(*Compare); !Added) return std::unexpected(Added.error()); auto BuiltCompare = CompareResult->build(); if (!BuiltCompare) return std::unexpected(BuiltCompare.error()); while (!Instruction.memOperands().empty()) { auto Erased = Instruction.eraseMemOperand(0); if (!Erased) return std::unexpected(Erased.error()); } return rewriteUnary(Instruction, GenericOpcode::COPY, *OldValue, *AtomicResult); } [[nodiscard]] Expected widenIntegerCompareSource(MachineInstr &Instruction, MachineFunction &Function, MachineIRBuilder &Builder, Register Source, LLT WideType) { const auto SourceInfo = Function.registerInfo().virtualRegisterInfo(Source); if (!SourceInfo || !SourceInfo->get().Type) return makeUnexpected(ErrorCode::VerificationError, "RISC-V integer compare source has no scalar type"); const LLT SourceType = *SourceInfo->get().Type; if (SourceType.scalarBitWidth() == WideType.scalarBitWidth()) return Source; const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(Source); MachineInstr *Def = Definition == nullptr ? nullptr : const_cast(Definition->parent()); std::optional ExistingExtension; bool AmbiguousExtension = false; auto &RegisterInfo = Function.registerInfo(); for (MachineOperand &Use : RegisterInfo.useOperands(Source)) { MachineInstr *User = Use.parent(); if (User == nullptr || User == &Instruction || User->parent() == nullptr || (User->opcode() != opcode(GenericOpcode::G_ANYEXT) && User->opcode() != opcode(GenericOpcode::G_SEXT))) { continue; } const MachineOperand *ExtendedOperand = User->operandAt(0); const Register *Extended = ExtendedOperand == nullptr ? nullptr : ExtendedOperand->getIf(); const std::optional ExtendedType = ExtendedOperand == nullptr ? std::nullopt : registerType(Function, *ExtendedOperand); if (Extended == nullptr || !ExtendedType || *ExtendedType != WideType || (Def == nullptr || (Def->opcode() != opcode(GenericOpcode::G_CONSTANT) && User->opcode() != opcode(GenericOpcode::G_SEXT)))) { continue; } if (ExistingExtension) { AmbiguousExtension = true; break; } ExistingExtension = *Extended; } if (ExistingExtension && !AmbiguousExtension) return *ExistingExtension; if (Def != nullptr && Def->opcode() == opcode(GenericOpcode::G_CONSTANT) && singleRegisterUse(Function, Source) == &Instruction) { auto Widened = widenConstantToRV64(*Def, Function); if (!Widened) return std::unexpected(Widened.error()); return Source; } if (Def != nullptr && Def->opcode() == opcode(GenericOpcode::G_TRUNC) && Def->operandCount() == 2) { const MachineOperand *WideOperand = Def->operandAt(1); const Register *WideSource = WideOperand == nullptr ? nullptr : WideOperand->getIf(); const std::optional ExistingWideType = WideOperand == nullptr ? std::nullopt : registerType(Function, *WideOperand); if (WideSource != nullptr && ExistingWideType && *ExistingWideType == WideType) { auto Result = Function.registerInfo().createVirtualRegister(WideType); if (!Result) return std::unexpected(Result.error()); auto Point = Builder.setInsertPointBefore(Instruction); if (!Point) return std::unexpected(Point.error()); auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_SEXT_INREG)); if (!Pending) return std::unexpected(Pending.error()); if (auto Added = Pending->addDef(*Result); !Added) return std::unexpected(Added.error()); if (auto Added = Pending->addUse(*WideSource); !Added) return std::unexpected(Added.error()); if (auto Added = Pending->addImmediate(static_cast(SourceType.scalarBitWidth())); !Added) { return std::unexpected(Added.error()); } auto Built = Pending->build(); if (!Built) return std::unexpected(Built.error()); return *Result; } } return buildUnaryBefore(Builder, Instruction, GenericOpcode::G_SEXT, WideType, Source); } [[nodiscard]] Expected lowerIntegerCompare(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target) { const MachineOperand *DestinationOperand = Instruction.operandAt(0); const MachineOperand *LeftOperand = Instruction.operandAt(2); const MachineOperand *RightOperand = Instruction.operandAt(3); const Register *Destination = DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf(); const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf(); if (Destination == nullptr || Left == nullptr || Right == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V integer compare has invalid register operands"); const Register OriginalDestination = *Destination; const Register OriginalLeft = *Left; const Register OriginalRight = *Right; auto WideType = LLT::makeAnyScalar(64); if (!WideType) return std::unexpected(WideType.error()); MachineIRBuilder Builder(Function, Target); auto WideLeft = widenIntegerCompareSource(Instruction, Function, Builder, OriginalLeft, *WideType); if (!WideLeft) return std::unexpected(WideLeft.error()); auto WideRight = widenIntegerCompareSource(Instruction, Function, Builder, OriginalRight, *WideType); if (!WideRight) return std::unexpected(WideRight.error()); if (auto Replaced = Instruction.operandAt(2)->setRegister(*WideLeft); !Replaced) return std::unexpected(Replaced.error()); if (auto Replaced = Instruction.operandAt(3)->setRegister(*WideRight); !Replaced) return std::unexpected(Replaced.error()); const auto DestinationInfo = Function.registerInfo().virtualRegisterInfo(OriginalDestination); if (!DestinationInfo || !DestinationInfo->get().Type) return makeUnexpected(ErrorCode::VerificationError, "RISC-V integer compare destination has no scalar type"); if (DestinationInfo->get().Type->scalarBitWidth() == 64) return {}; MachineInstr *FoldedExtension = singleRegisterUse(Function, OriginalDestination); if (FoldedExtension != nullptr && (FoldedExtension->opcode() == opcode(GenericOpcode::G_ANYEXT) || FoldedExtension->opcode() == opcode(GenericOpcode::G_SEXT) || FoldedExtension->opcode() == opcode(GenericOpcode::G_ZEXT)) && operandType(Function, *FoldedExtension, 0) .transform([](LLT Type) { return Type.scalarBitWidth() == 64; }) .value_or(false)) { const MachineOperand *ExtendedOperand = FoldedExtension->operandAt(0); const Register *Extended = ExtendedOperand == nullptr ? nullptr : ExtendedOperand->getIf(); if (Extended == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V integer compare extension has no destination"); const Register WideDestination = *Extended; if (auto Replaced = Instruction.operandAt(0)->setRegister(WideDestination); !Replaced) return std::unexpected(Replaced.error()); return eraseInstruction(*FoldedExtension); } auto WideDestination = Function.registerInfo().createVirtualRegister(*WideType); if (!WideDestination) return std::unexpected(WideDestination.error()); if (auto Replaced = Instruction.operandAt(0)->setRegister(*WideDestination); !Replaced) return std::unexpected(Replaced.error()); if (Function.registerInfo().useEmpty(OriginalDestination)) return {}; auto Point = Builder.setInsertPointAfter(Instruction); if (!Point) return std::unexpected(Point.error()); auto Truncation = Builder.buildInstr(opcode(GenericOpcode::G_TRUNC)); if (!Truncation) return std::unexpected(Truncation.error()); if (auto Added = Truncation->addDef(OriginalDestination); !Added) return std::unexpected(Added.error()); if (auto Added = Truncation->addUse(*WideDestination); !Added) return std::unexpected(Added.error()); auto Built = Truncation->build(); if (!Built) return std::unexpected(Built.error()); return {}; } [[nodiscard]] Expected foldNarrowAnyExtBinary(MachineInstr &Extension, MachineFunction &Function, const TargetDescription &Target) { const auto Plan = narrowAnyExtBinaryPlan(Extension, Function); if (!Plan) return makeUnexpected(ErrorCode::Unsupported, "RISC-V narrow any-extension does not have a legal binary chain"); for (MachineInstr *Constant : Plan->Constants) { auto Widened = widenConstantToRV64(*Constant, Function); if (!Widened) return std::unexpected(Widened.error()); } for (std::size_t Index = 0; Index != Plan->WideInputs.size(); ++Index) { auto Replaced = Plan->Binary->operandAt(Index + 1U)->setRegister(Plan->WideInputs[Index]); if (!Replaced) return std::unexpected(Replaced.error()); } for (MachineInstr *Artifact : Plan->Artifacts) { if (Artifact->parent() == nullptr) continue; auto Removed = Artifact->parent()->removeInstruction(*Artifact); if (!Removed) return std::unexpected(Removed.error()); } if (Plan->Generic == GenericOpcode::G_SHL) { Register Amount = Plan->WideInputs[1]; if (Plan->SourceWidth < 32 || Plan->SourceWidth > 32) { const std::uint64_t Mask = Plan->SourceWidth == 64 ? std::numeric_limits::max() : (std::uint64_t{1} << Plan->SourceWidth) - 1U; auto WideType = LLT::makeAnyScalar(64); if (!WideType) return std::unexpected(WideType.error()); MachineIRBuilder Builder(Function, Target); auto MaskValue = buildIntegerConstantBefore(Builder, Extension, 64, Mask); if (!MaskValue) return std::unexpected(MaskValue.error()); auto MaskedAmount = buildBinaryBefore(Builder, Extension, GenericOpcode::G_AND, *WideType, Amount, *MaskValue); if (!MaskedAmount) return std::unexpected(MaskedAmount.error()); Amount = *MaskedAmount; } auto SetOpcode = Extension.setOpcode(Plan->SourceWidth == 32 ? targetOpcode(InstructionOpcode::G_SLLW) : opcode(GenericOpcode::G_SHL)); if (!SetOpcode) return std::unexpected(SetOpcode.error()); while (Extension.operandCount() > 1) { auto Erased = Extension.eraseOperand(1); if (!Erased) return std::unexpected(Erased.error()); } for (Register Input : {Plan->WideInputs[0], Amount}) { auto Operand = MachineOperand::registerOperand(Input, RegisterOperandFlags{RegisterOperandFlag::Use}); if (!Operand) return std::unexpected(Operand.error()); auto Added = Extension.addOperand(std::move(*Operand)); if (!Added) return std::unexpected(Added.error()); } MachineBasicBlock *BinaryBlock = Plan->Binary->parent(); if (BinaryBlock == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V narrow shift is detached during legalization"); return BinaryBlock->removeInstruction(*Plan->Binary); } if (Plan->SourceWidth == 32 && (Plan->Generic == GenericOpcode::G_ADD || Plan->Generic == GenericOpcode::G_SUB)) { auto WideType = LLT::makeAnyScalar(64); if (!WideType) return std::unexpected(WideType.error()); auto Temporary = Function.registerInfo().createVirtualRegister(*WideType); if (!Temporary) return std::unexpected(Temporary.error()); auto Retargeted = Plan->Binary->operandAt(0)->setRegister(*Temporary); if (!Retargeted) return std::unexpected(Retargeted.error()); auto SetOpcode = Extension.setOpcode(opcode(GenericOpcode::G_SEXT_INREG)); if (!SetOpcode) return std::unexpected(SetOpcode.error()); auto ReplacedSource = Extension.operandAt(1)->setRegister(*Temporary); if (!ReplacedSource) return std::unexpected(ReplacedSource.error()); auto Width = Extension.addOperand(MachineOperand::immediate(32)); if (!Width) return std::unexpected(Width.error()); return {}; } auto SetOpcode = Extension.setOpcode(opcode(Plan->Generic)); if (!SetOpcode) return std::unexpected(SetOpcode.error()); while (Extension.operandCount() > 1) { auto Erased = Extension.eraseOperand(1); if (!Erased) return std::unexpected(Erased.error()); } for (Register Input : Plan->WideInputs) { auto Operand = MachineOperand::registerOperand(Input, RegisterOperandFlags{RegisterOperandFlag::Use}); if (!Operand) return std::unexpected(Operand.error()); auto Added = Extension.addOperand(std::move(*Operand)); if (!Added) return std::unexpected(Added.error()); } MachineBasicBlock *BinaryBlock = Plan->Binary->parent(); if (BinaryBlock == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V narrow binary instruction is detached during legalization"); auto Removed = BinaryBlock->removeInstruction(*Plan->Binary); if (!Removed) return std::unexpected(Removed.error()); (void)Target; return {}; } [[nodiscard]] Expected lowerConstantToPool(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target) { const MachineOperand *DestinationOperand = Instruction.operandAt(0); const Register *Destination = DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); const MachineOperand *ImmediateOperand = Instruction.operandAt(1); const auto *Immediate = ImmediateOperand == nullptr ? nullptr : ImmediateOperand->getIf(); if (Destination == nullptr || !Destination->isVirtual() || Immediate == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V constant-pool legalization has invalid operands"); const auto DestinationInfo = Function.registerInfo().virtualRegisterInfo(*Destination); if (!DestinationInfo || !DestinationInfo->get().Type || DestinationInfo->get().Type->scalarBitWidth() != 64) return makeUnexpected(ErrorCode::VerificationError, "RISC-V constant-pool legalization requires a 64-bit destination"); const LLT DestinationType = *DestinationInfo->get().Type; auto Value = GenericConstantData::integer(Immediate->Value); if (!Value) return std::unexpected(Value.error()); auto Align = Alignment::create(8); if (!Align) return std::unexpected(Align.error()); MachineConstantPoolEntryDesc Description; Description.Value = std::move(*Value); Description.Size = 8; Description.Align = *Align; auto PoolIndex = Function.constantPool().addEntry(std::move(Description)); if (!PoolIndex) return std::unexpected(PoolIndex.error()); auto PointerType = LLT::makePointer(0, 64); if (!PointerType) return std::unexpected(PointerType.error()); auto Address = Function.registerInfo().createVirtualRegister(*PointerType); if (!Address) return std::unexpected(Address.error()); MachineIRBuilder Builder(Function, Target); auto Point = Builder.setInsertPointBefore(Instruction); if (!Point) return std::unexpected(Point.error()); auto AddressInstruction = Builder.buildInstr(opcode(GenericOpcode::G_CONSTANT_POOL)); if (!AddressInstruction) return std::unexpected(AddressInstruction.error()); auto AddressDef = AddressInstruction->addDef(*Address); if (!AddressDef) return std::unexpected(AddressDef.error()); auto AddressPool = AddressInstruction->addConstantPool(*PoolIndex); if (!AddressPool) return std::unexpected(AddressPool.error()); auto BuiltAddress = AddressInstruction->build(); if (!BuiltAddress) return std::unexpected(BuiltAddress.error()); MachineMemOperandDesc MemoryDescription; MemoryDescription.Accesses = {MemoryAccessFlag::Load}; MemoryDescription.Size = *MemorySize::fixed(8); MemoryDescription.MemoryType = DestinationType; MemoryDescription.Align = *Align; MemoryDescription.Base = MemoryBase::constantPoolPseudo(); auto Memory = Function.createMemOperand(std::move(MemoryDescription)); if (!Memory) return std::unexpected(Memory.error()); auto SetOpcode = Instruction.setOpcode(opcode(GenericOpcode::G_LOAD)); if (!SetOpcode) return std::unexpected(SetOpcode.error()); while (Instruction.operandCount() > 1) { auto Erased = Instruction.eraseOperand(1); if (!Erased) return std::unexpected(Erased.error()); } auto AddressUse = MachineOperand::registerOperand(*Address, RegisterOperandFlags{RegisterOperandFlag::Use}); if (!AddressUse) return std::unexpected(AddressUse.error()); auto AddedAddress = Instruction.addOperand(std::move(*AddressUse)); if (!AddedAddress) return std::unexpected(AddedAddress.error()); auto AddedMemory = Instruction.addMemOperand(**Memory); if (!AddedMemory) return std::unexpected(AddedMemory.error()); return {}; } [[nodiscard]] bool isSupportedNonAtomicMemoryAccess(const MachineInstr &Instruction) noexcept { if (Instruction.memOperands().size() != 1) return false; const MachineMemOperand &Memory = *Instruction.memOperands().front(); const MachineMemOperandDesc &Description = Memory.description(); return !Memory.isAtomic() && Description.AddressSpace == 0 && Description.Size.isKnown() && !Description.Size.isScalable() && Description.Size.minimumBytes() != 0; } [[nodiscard]] bool isSupportedAtomicMemoryAccess(const MachineInstr &Instruction, AtomicOperation Operation, std::uint64_t Bytes) noexcept { if (Instruction.memOperands().size() != 1 || (Bytes != 4 && Bytes != 8)) return false; const MachineMemOperand &Memory = *Instruction.memOperands().front(); const MachineMemOperandDesc &Description = Memory.description(); return Description.AtomicKind == Operation && Description.AddressSpace == 0 && Description.Size.isKnown() && !Description.Size.isScalable() && Description.Size.minimumBytes() == Bytes && Description.Align.bytes() >= Bytes && Description.SyncScope.value() == 0; } [[nodiscard]] bool isSupportedAtomicContainerAccess(const MachineInstr &Instruction, AtomicOperation Operation, std::uint64_t ContainerBytes) noexcept { if (Instruction.memOperands().size() != 1 || ContainerBytes == 0) return false; const MachineMemOperandDesc &Description = Instruction.memOperands().front()->description(); if (Description.AtomicKind != Operation || Description.AddressSpace != 0 || !Description.Size.isKnown() || Description.Size.isScalable() || Description.SyncScope.value() != 0) return false; const std::uint64_t MemoryBytes = Description.Size.minimumBytes(); return (MemoryBytes == 1 || MemoryBytes == 2 || MemoryBytes == 4 || MemoryBytes == 8) && MemoryBytes <= ContainerBytes && Description.Align.bytes() >= MemoryBytes; } [[nodiscard]] std::optional memoryAccessBytes(const MachineInstr &Instruction) noexcept { if (Instruction.memOperands().size() != 1) return std::nullopt; const MachineMemOperandDesc &Description = Instruction.memOperands().front()->description(); if (!Description.Size.isKnown() || Description.Size.isScalable()) return std::nullopt; return Description.Size.minimumBytes(); } [[nodiscard]] bool memoryMatchesType(const MachineInstr &Instruction, LLT Type) noexcept { const auto Bytes = memoryAccessBytes(Instruction); if (!Bytes || (!isSupportedIntegerType(Type) && !isRVFloatType(Type))) return false; if (isRVFloatType(Type)) return Type.scalarBitWidth() == 32 ? *Bytes == 4 : *Bytes == 8; const std::uint64_t ValueBytes = (Type.scalarBitWidth() + 7U) / 8U; return (*Bytes == 1 || *Bytes == 2 || *Bytes == 4 || *Bytes == 8) && *Bytes <= ValueBytes; } [[nodiscard]] Expected signedImmediate(const MachineOperand &Operand) { if (const std::int64_t *Value = Operand.getIf()) return *Value; const auto *Integer = Operand.getIf(); if (Integer == nullptr || Integer->Value.bitWidth() == 0 || Integer->Value.bitWidth() > 64) return makeUnexpected(ErrorCode::Unsupported, "RISC-V constant selection requires an immediate no wider than 64 bits"); std::uint64_t Raw = Integer->Value.words().empty() ? 0 : Integer->Value.words().front(); const std::uint32_t Width = Integer->Value.bitWidth(); if (Width < 64 && (Raw & (std::uint64_t{1} << (Width - 1U))) != 0) Raw |= ~std::uint64_t{0} << Width; return std::bit_cast(Raw); } [[nodiscard]] Expected replaceRegisterUses(MachineFunction &Function, Register OldReg, Register NewReg) { std::vector Uses; for (MachineOperand &Operand : Function.registerInfo().useOperands(OldReg)) Uses.push_back(&Operand); for (MachineOperand *Operand : Uses) { auto Replaced = Operand->setRegister(NewReg); if (!Replaced) return std::unexpected(Replaced.error()); } return {}; } [[nodiscard]] Expected eraseInstruction(MachineInstr &Instruction) { MachineBasicBlock *Block = Instruction.parent(); if (Block == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V legalizer cannot erase a detached instruction"); return Block->removeInstruction(Instruction); } [[nodiscard]] bool allDefsUnused(const MachineInstr &Instruction, const MachineFunction &Function) { for (const MachineOperand &Operand : Instruction.operands()) { const Register *Reg = Operand.getIf(); if (Reg != nullptr && Operand.isDef() && Reg->isVirtual() && !Function.registerInfo().useEmpty(*Reg)) return false; } return true; } void eraseDeadArtifacts(std::span Candidates, const MachineFunction &Function) { bool Changed = false; do { Changed = false; for (auto It = Candidates.rbegin(); It != Candidates.rend(); ++It) { MachineInstr *Instruction = *It; if (Instruction == nullptr || Instruction->parent() == nullptr || !allDefsUnused(*Instruction, Function)) continue; const Opcode Value = Instruction->opcode(); if (Value != opcode(GenericOpcode::G_TRUNC) && Value != opcode(GenericOpcode::G_ANYEXT) && Value != opcode(GenericOpcode::G_SEXT) && Value != opcode(GenericOpcode::G_ZEXT) && Value != opcode(GenericOpcode::G_MERGE_VALUES) && Value != opcode(GenericOpcode::G_UNMERGE_VALUES) && Value != opcode(GenericOpcode::COPY)) continue; if (eraseInstruction(*Instruction)) Changed = true; } } while (Changed); } [[nodiscard]] Expected buildIntegerConstantBefore(MachineIRBuilder &Builder, MachineInstr &Anchor, std::uint32_t Width, std::uint64_t Value) { auto Type = LLT::makeAnyScalar(Width); if (!Type) return std::unexpected(Type.error()); auto Result = Builder.function().registerInfo().createVirtualRegister(*Type); if (!Result) return std::unexpected(Result.error()); auto Immediate = IntegerImmediate::fromUnsigned(Width, Value); if (!Immediate) return std::unexpected(Immediate.error()); auto Point = Builder.setInsertPointBefore(Anchor); if (!Point) return std::unexpected(Point.error()); auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_CONSTANT)); if (!Pending) return std::unexpected(Pending.error()); if (auto Added = Pending->addDef(*Result); !Added) return std::unexpected(Added.error()); if (auto Added = Pending->addIntegerImmediate(std::move(*Immediate)); !Added) return std::unexpected(Added.error()); auto Built = Pending->build(); if (!Built) return std::unexpected(Built.error()); return *Result; } [[nodiscard]] Expected buildUnaryBefore(MachineIRBuilder &Builder, MachineInstr &Anchor, GenericOpcode Generic, LLT Type, Register Source) { auto Result = Builder.function().registerInfo().createVirtualRegister(Type); if (!Result) return std::unexpected(Result.error()); auto Point = Builder.setInsertPointBefore(Anchor); if (!Point) return std::unexpected(Point.error()); auto Pending = Builder.buildInstr(opcode(Generic)); if (!Pending) return std::unexpected(Pending.error()); if (auto Added = Pending->addDef(*Result); !Added) return std::unexpected(Added.error()); if (auto Added = Pending->addUse(Source); !Added) return std::unexpected(Added.error()); auto Built = Pending->build(); if (!Built) return std::unexpected(Built.error()); return *Result; } [[nodiscard]] Expected buildBinaryBefore(MachineIRBuilder &Builder, MachineInstr &Anchor, GenericOpcode Generic, LLT Type, Register Left, Register Right) { auto Result = Builder.function().registerInfo().createVirtualRegister(Type); if (!Result) return std::unexpected(Result.error()); auto Point = Builder.setInsertPointBefore(Anchor); if (!Point) return std::unexpected(Point.error()); auto Pending = Builder.buildInstr(opcode(Generic)); if (!Pending) return std::unexpected(Pending.error()); if (auto Added = Pending->addDef(*Result); !Added) return std::unexpected(Added.error()); if (auto Added = Pending->addUse(Left); !Added) return std::unexpected(Added.error()); if (auto Added = Pending->addUse(Right); !Added) return std::unexpected(Added.error()); auto Built = Pending->build(); if (!Built) return std::unexpected(Built.error()); return *Result; } [[nodiscard]] Expected rewriteUnary(MachineInstr &Instruction, GenericOpcode Generic, Register Destination, Register Source) { auto SetOpcode = Instruction.setOpcode(opcode(Generic)); if (!SetOpcode) return std::unexpected(SetOpcode.error()); while (Instruction.operandCount() != 0) { auto Erased = Instruction.eraseOperand(Instruction.operandCount() - 1U); if (!Erased) return std::unexpected(Erased.error()); } auto Def = MachineOperand::registerOperand(Destination, RegisterOperandFlags{RegisterOperandFlag::Define}); if (!Def) return std::unexpected(Def.error()); if (auto Added = Instruction.addOperand(std::move(*Def)); !Added) return std::unexpected(Added.error()); auto Use = MachineOperand::registerOperand(Source, RegisterOperandFlags{RegisterOperandFlag::Use}); if (!Use) return std::unexpected(Use.error()); return Instruction.addOperand(std::move(*Use)); } [[nodiscard]] Expected widenNarrowMultiply(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target) { const std::optional Type = resultType(Function, Instruction); const MachineOperand *DestinationOperand = Instruction.operandAt(0); const MachineOperand *LeftOperand = Instruction.operandAt(1); const MachineOperand *RightOperand = Instruction.operandAt(2); const Register *Destination = DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf(); const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf(); if (!Type || Destination == nullptr || Left == nullptr || Right == nullptr || Type->scalarBitWidth() >= 64) { return makeUnexpected(ErrorCode::VerificationError, "RISC-V narrow multiply has invalid operands"); } MachineInstr *Extension = singleRegisterUse(Function, *Destination); if (Extension != nullptr && Extension->opcode() == opcode(GenericOpcode::G_ANYEXT)) { const auto Plan = narrowAnyExtBinaryPlan(*Extension, Function); if (Plan && Plan->Binary == &Instruction) return foldNarrowAnyExtBinary(*Extension, Function, Target); } auto WideType = LLT::makeAnyScalar(64); if (!WideType) return std::unexpected(WideType.error()); MachineIRBuilder Builder(Function, Target); auto WideLeft = buildUnaryBefore(Builder, Instruction, GenericOpcode::G_ANYEXT, *WideType, *Left); if (!WideLeft) return std::unexpected(WideLeft.error()); auto WideRight = buildUnaryBefore(Builder, Instruction, GenericOpcode::G_ANYEXT, *WideType, *Right); if (!WideRight) return std::unexpected(WideRight.error()); auto Product = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_MUL, *WideType, *WideLeft, *WideRight); if (!Product) return std::unexpected(Product.error()); return rewriteUnary(Instruction, GenericOpcode::G_TRUNC, *Destination, *Product); } [[nodiscard]] Expected lowerMultiplyLibcall(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target, CallingConvention Convention, const TargetCallLowering &CallLowering) { const std::optional Type = resultType(Function, Instruction); const MachineOperand *DestinationOperand = Instruction.operandAt(0); const MachineOperand *LeftOperand = Instruction.operandAt(1); const MachineOperand *RightOperand = Instruction.operandAt(2); const Register *Destination = DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf(); const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf(); if (!Type || Destination == nullptr || Left == nullptr || Right == nullptr || Type->scalarBitWidth() != 64) { return makeUnexpected(ErrorCode::VerificationError, "RISC-V multiply libcall requires three s64 registers"); } auto ABIAlignment = Alignment::create(8); auto Callee = MachineCallTarget::externalSymbol("__muldi3"); if (!ABIAlignment || !Callee) return makeUnexpected(ErrorCode::InternalError, "RISC-V multiply libcall ABI facts are unavailable"); const auto Value = [Type = *Type, Alignment = *ABIAlignment](Register Reg, std::uint32_t Number) { return ABIValue{ .Value = Reg, .Type = Type, .OriginalValue = Number, .FragmentIndex = 0, .FragmentCount = 1, .FragmentOffsetBytes = 0, .OriginalAlignment = Alignment, .Extension = ABIExtensionKind::None, .Flags = {}, .ByValSizeBytes = 0, .ByValAlignment = Alignment, }; }; CallLoweringRequest Request; Request.Convention = callingConventionID(Convention); Request.Callee = std::move(*Callee); Request.Arguments = {Value(*Left, 0), Value(*Right, 1)}; Request.Results = {Value(*Destination, 0)}; Request.FixedArgumentCount = Request.Arguments.size(); MachineIRBuilder Builder(Function, Target); auto Point = Builder.setInsertPointBefore(Instruction); if (!Point) return std::unexpected(Point.error()); auto Lowered = CallLowering.lowerCall(Builder, Request); if (!Lowered) return std::unexpected(Lowered.error()); return eraseInstruction(Instruction); } [[nodiscard]] Expected rewriteBinary(MachineInstr &Instruction, GenericOpcode Generic, Register Destination, Register Left, Register Right) { auto Rewritten = rewriteUnary(Instruction, Generic, Destination, Left); if (!Rewritten) return std::unexpected(Rewritten.error()); auto Use = MachineOperand::registerOperand(Right, RegisterOperandFlags{RegisterOperandFlag::Use}); if (!Use) return std::unexpected(Use.error()); return Instruction.addOperand(std::move(*Use)); } [[nodiscard]] bool isKnownZeroExtended(const MachineFunction &Function, Register Reg, std::uint32_t ActiveBits, unsigned Depth = 0) { if (Depth > 8 || !Reg.isVirtual()) return false; const auto Info = Function.registerInfo().virtualRegisterInfo(Reg); if (!Info || !Info->get().Type) return false; const std::uint32_t Width = Info->get().Type->scalarBitWidth(); if (Width <= ActiveBits) return true; const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(Reg); const MachineInstr *Def = Definition == nullptr ? nullptr : Definition->parent(); if (Def == nullptr) return false; if (Def->opcode() == opcode(GenericOpcode::G_ASSERT_ZEXT)) { const MachineOperand *BitsOperand = Def->operandAt(2); auto Bits = BitsOperand == nullptr ? Expected(std::unexpected( Error(ErrorCode::InvalidArgument, "missing asserted extension width"))) : signedImmediate(*BitsOperand); return Bits && *Bits >= 0 && static_cast(*Bits) <= ActiveBits; } if (Def->opcode() == opcode(GenericOpcode::G_ZEXT)) { const MachineOperand *SourceOperand = Def->operandAt(1); const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); if (Source == nullptr) return false; const auto SourceInfo = Function.registerInfo().virtualRegisterInfo(*Source); return SourceInfo && SourceInfo->get().Type && SourceInfo->get().Type->scalarBitWidth() <= ActiveBits; } if (Def->opcode() == opcode(GenericOpcode::G_TRUNC)) { const MachineOperand *SourceOperand = Def->operandAt(1); const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); return Source != nullptr && isKnownZeroExtended(Function, *Source, ActiveBits, Depth + 1U); } return false; } [[nodiscard]] Expected zeroExtendToRV64(MachineIRBuilder &Builder, MachineInstr &Anchor, MachineFunction &Function, Register Source, std::vector &DeadCandidates) { const auto Info = Function.registerInfo().virtualRegisterInfo(Source); if (!Info || !Info->get().Type) return makeUnexpected(ErrorCode::VerificationError, "RISC-V merge source has no scalar type"); const LLT SourceType = *Info->get().Type; if (SourceType.scalarBitWidth() == 64) return Source; const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(Source); MachineInstr *Def = Definition == nullptr ? nullptr : const_cast(Definition->parent()); if (Def != nullptr && Def->opcode() == opcode(GenericOpcode::G_TRUNC)) { const MachineOperand *WideOperand = Def->operandAt(1); const Register *Wide = WideOperand == nullptr ? nullptr : WideOperand->getIf(); if (Wide != nullptr) { const auto WideInfo = Function.registerInfo().virtualRegisterInfo(*Wide); if (WideInfo && WideInfo->get().Type && WideInfo->get().Type->scalarBitWidth() == 64 && isKnownZeroExtended(Function, *Wide, SourceType.scalarBitWidth())) { DeadCandidates.push_back(Def); return *Wide; } } } auto WideType = LLT::makeAnyScalar(64); if (!WideType) return std::unexpected(WideType.error()); return buildUnaryBefore(Builder, Anchor, GenericOpcode::G_ZEXT, *WideType, Source); } [[nodiscard]] MachineInstr *singleUseInstruction(const MachineFunction &Function, Register Reg) { MachineInstr *Result = nullptr; auto &RegisterInfo = const_cast(Function.registerInfo()); for (MachineOperand &Operand : RegisterInfo.useOperands(Reg)) { MachineInstr *Instruction = Operand.parent(); if (Instruction == nullptr || Instruction->parent() == nullptr) continue; if (Result != nullptr && Result != Instruction) return nullptr; Result = Instruction; } return Result; } [[nodiscard]] Expected lowerWideMergeValues(MachineInstr &Instruction, MachineFunction &Function) { const MachineOperand *DestinationOperand = Instruction.operandAt(0); const Register *Destination = DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); const MachineOperand *LowOperand = Instruction.operandAt(1); const Register *Low = LowOperand == nullptr ? nullptr : LowOperand->getIf(); if (Destination == nullptr || Low == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide merge has invalid register operands"); std::vector Truncations; auto &RegisterInfo = Function.registerInfo(); std::vector Uses; for (MachineOperand &Use : RegisterInfo.useOperands(*Destination)) Uses.push_back(&Use); for (MachineOperand *Use : Uses) { MachineInstr *Trunc = Use->parent(); if (Trunc == nullptr || Trunc->parent() == nullptr || Trunc->opcode() != opcode(GenericOpcode::G_TRUNC) || Trunc->operandAt(1) != Use) return makeUnexpected(ErrorCode::Unsupported, "RISC-V wide merge is only representable through low-part truncation"); const MachineOperand *TruncResultOperand = Trunc->operandAt(0); const Register *TruncResult = TruncResultOperand == nullptr ? nullptr : TruncResultOperand->getIf(); const auto LowInfo = RegisterInfo.virtualRegisterInfo(*Low); const auto ResultInfo = TruncResult == nullptr ? Expected>( std::unexpected(Error(ErrorCode::InvalidArgument, "missing truncation result"))) : RegisterInfo.virtualRegisterInfo(*TruncResult); if (!LowInfo || !LowInfo->get().Type || !ResultInfo || !ResultInfo->get().Type || ResultInfo->get().Type->scalarBitWidth() > LowInfo->get().Type->scalarBitWidth()) return makeUnexpected(ErrorCode::Unsupported, "RISC-V wide merge truncation spans more than its low source"); if (*ResultInfo->get().Type == *LowInfo->get().Type) { auto Replaced = replaceRegisterUses(Function, *TruncResult, *Low); if (!Replaced) return std::unexpected(Replaced.error()); Truncations.push_back(Trunc); } else { auto Replaced = Use->setRegister(*Low); if (!Replaced) return std::unexpected(Replaced.error()); } } for (MachineInstr *Trunc : Truncations) { auto Erased = eraseInstruction(*Trunc); if (!Erased) return std::unexpected(Erased.error()); } return eraseInstruction(Instruction); } [[nodiscard]] Expected lowerMergeValues(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target) { const std::optional DestinationType = resultType(Function, Instruction); if (!DestinationType || DestinationType->scalarBitWidth() == 0) return makeUnexpected(ErrorCode::VerificationError, "RISC-V merge destination has no scalar type"); if (DestinationType->scalarBitWidth() > 64) return lowerWideMergeValues(Instruction, Function); const MachineOperand *DestinationOperand = Instruction.operandAt(0); const Register *Destination = DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); if (Destination == nullptr || Instruction.operandCount() < 3) return makeUnexpected(ErrorCode::VerificationError, "RISC-V merge has invalid register operands"); Register Result = *Destination; MachineInstr *FoldedExtension = nullptr; if (DestinationType->scalarBitWidth() < 64) { MachineInstr *Use = singleUseInstruction(Function, *Destination); if (Use != nullptr && Use->opcode() == opcode(GenericOpcode::G_ZEXT) && operandType(Function, *Use, 0) .transform([](LLT Type) { return Type.scalarBitWidth() == 64; }) .value_or(false)) { const MachineOperand *ExtendedOperand = Use->operandAt(0); const Register *Extended = ExtendedOperand == nullptr ? nullptr : ExtendedOperand->getIf(); if (Extended != nullptr) { Result = *Extended; FoldedExtension = Use; } } } auto WideType = LLT::makeAnyScalar(64); if (!WideType) return std::unexpected(WideType.error()); MachineIRBuilder Builder(Function, Target); std::vector DeadCandidates; std::vector Terms; Terms.reserve(Instruction.operandCount() - 1U); for (std::size_t Index = 1; Index < Instruction.operandCount(); ++Index) { const MachineOperand *Operand = Instruction.operandAt(Index); const Register *Source = Operand == nullptr ? nullptr : Operand->getIf(); if (Source == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V merge source is not a register"); auto WideSource = zeroExtendToRV64(Builder, Instruction, Function, *Source, DeadCandidates); if (!WideSource) return std::unexpected(WideSource.error()); if (Index == 1) { Terms.push_back(*WideSource); continue; } const std::optional SourceType = registerType(Function, *Operand); if (!SourceType) return makeUnexpected(ErrorCode::VerificationError, "RISC-V merge source has no scalar type"); const std::uint64_t Shift = static_cast(SourceType->scalarBitWidth()) * (Index - 1U); auto Amount = buildIntegerConstantBefore(Builder, Instruction, 64, Shift); if (!Amount) return std::unexpected(Amount.error()); auto Shifted = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_SHL, *WideType, *WideSource, *Amount); if (!Shifted) return std::unexpected(Shifted.error()); Terms.push_back(*Shifted); } Register Accumulator = Terms.front(); for (std::size_t Index = 1; Index + 1U < Terms.size(); ++Index) { auto Combined = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_OR, *WideType, Accumulator, Terms[Index]); if (!Combined) return std::unexpected(Combined.error()); Accumulator = *Combined; } if (DestinationType->scalarBitWidth() == 64 || FoldedExtension != nullptr) { auto Rewritten = rewriteBinary(Instruction, GenericOpcode::G_OR, Result, Accumulator, Terms.back()); if (!Rewritten) return std::unexpected(Rewritten.error()); } else { auto Combined = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_OR, *WideType, Accumulator, Terms.back()); if (!Combined) return std::unexpected(Combined.error()); auto Rewritten = rewriteUnary(Instruction, GenericOpcode::G_TRUNC, *Destination, *Combined); if (!Rewritten) return std::unexpected(Rewritten.error()); } if (FoldedExtension != nullptr) { auto Erased = eraseInstruction(*FoldedExtension); if (!Erased) return std::unexpected(Erased.error()); } eraseDeadArtifacts(DeadCandidates, Function); return {}; } [[nodiscard]] Expected foldWideExtendedUnmerge(MachineInstr &Instruction, MachineFunction &Function) { const std::size_t OutputCount = Instruction.operandCount() - 1U; const MachineOperand *SourceOperand = Instruction.operandAt(OutputCount); const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); const MachineOperand *Definition = Source == nullptr ? nullptr : Function.registerInfo().getUniqueVRegDef(*Source); MachineInstr *Extension = Definition == nullptr ? nullptr : const_cast(Definition->parent()); if (Extension == nullptr || (Extension->opcode() != opcode(GenericOpcode::G_ZEXT) && Extension->opcode() != opcode(GenericOpcode::G_ANYEXT) && Extension->opcode() != opcode(GenericOpcode::G_SEXT))) return makeUnexpected(ErrorCode::Unsupported, "RISC-V wide unmerge requires an extension-defined source"); const MachineOperand *NarrowOperand = Extension->operandAt(1); const Register *Narrow = NarrowOperand == nullptr ? nullptr : NarrowOperand->getIf(); if (Narrow == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide unmerge extension has no register source"); const auto NarrowInfo = Function.registerInfo().virtualRegisterInfo(*Narrow); if (!NarrowInfo || !NarrowInfo->get().Type) return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide unmerge extension source has no type"); const std::uint32_t NarrowWidth = NarrowInfo->get().Type->scalarBitWidth(); std::vector DeadUsers; for (std::size_t Index = 0; Index < OutputCount; ++Index) { const MachineOperand *OutputOperand = Instruction.operandAt(Index); const Register *Output = OutputOperand == nullptr ? nullptr : OutputOperand->getIf(); if (Output == nullptr || Function.registerInfo().useEmpty(*Output)) continue; const auto OutputInfo = Function.registerInfo().virtualRegisterInfo(*Output); if (!OutputInfo || !OutputInfo->get().Type) return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide unmerge output has no type"); const std::uint64_t Offset = static_cast(Index) * OutputInfo->get().Type->scalarBitWidth(); if (Offset != 0) return makeUnexpected(ErrorCode::Unsupported, "RISC-V wide unmerge cannot materialize a used high wide part"); if (OutputInfo->get().Type->scalarBitWidth() == NarrowWidth) { auto Replaced = replaceRegisterUses(Function, *Output, *Narrow); if (!Replaced) return std::unexpected(Replaced.error()); continue; } if (OutputInfo->get().Type->scalarBitWidth() < NarrowWidth) return makeUnexpected(ErrorCode::Unsupported, "RISC-V wide unmerge output unexpectedly narrows its source"); std::vector Uses; for (MachineOperand &Use : Function.registerInfo().useOperands(*Output)) Uses.push_back(&Use); for (MachineOperand *Use : Uses) { MachineInstr *Trunc = Use->parent(); if (Trunc == nullptr || Trunc->parent() == nullptr || Trunc->opcode() != opcode(GenericOpcode::G_TRUNC) || Trunc->operandAt(1) != Use) return makeUnexpected( ErrorCode::Unsupported, "RISC-V wide unmerge result is only representable through truncation"); const MachineOperand *ResultOperand = Trunc->operandAt(0); const Register *Result = ResultOperand == nullptr ? nullptr : ResultOperand->getIf(); const auto ResultInfo = Result == nullptr ? Expected>( std::unexpected(Error(ErrorCode::InvalidArgument, "missing truncation result"))) : Function.registerInfo().virtualRegisterInfo(*Result); if (!ResultInfo || !ResultInfo->get().Type || ResultInfo->get().Type->scalarBitWidth() > NarrowWidth) return makeUnexpected(ErrorCode::Unsupported, "RISC-V wide unmerge truncation exceeds its source width"); if (ResultInfo->get().Type->scalarBitWidth() == NarrowWidth) { auto Replaced = replaceRegisterUses(Function, *Result, *Narrow); if (!Replaced) return std::unexpected(Replaced.error()); DeadUsers.push_back(Trunc); } else { auto Replaced = Use->setRegister(*Narrow); if (!Replaced) return std::unexpected(Replaced.error()); } } } for (MachineInstr *User : DeadUsers) { auto Erased = eraseInstruction(*User); if (!Erased) return std::unexpected(Erased.error()); } auto Erased = eraseInstruction(Instruction); if (!Erased) return std::unexpected(Erased.error()); if (Function.registerInfo().useEmpty(*Source)) return eraseInstruction(*Extension); return {}; } [[nodiscard]] Expected lowerUnmergeValues(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target) { if (Instruction.operandCount() < 3) return makeUnexpected(ErrorCode::VerificationError, "RISC-V unmerge has invalid register operands"); const std::size_t OutputCount = Instruction.operandCount() - 1U; const std::optional OutputType = operandType(Function, Instruction, 0); const std::optional SourceType = operandType(Function, Instruction, OutputCount); if (!OutputType || !SourceType) return makeUnexpected(ErrorCode::VerificationError, "RISC-V unmerge operands have no scalar types"); if (SourceType->scalarBitWidth() > 64 || OutputType->scalarBitWidth() > 64) return foldWideExtendedUnmerge(Instruction, Function); const MachineOperand *SourceOperand = Instruction.operandAt(OutputCount); const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); if (Source == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V unmerge source is not a register"); auto WideType = LLT::makeAnyScalar(64); if (!WideType) return std::unexpected(WideType.error()); MachineIRBuilder Builder(Function, Target); std::vector FoldedExtensions; for (std::size_t Index = 0; Index < OutputCount; ++Index) { const MachineOperand *OutputOperand = Instruction.operandAt(Index); const Register *Output = OutputOperand == nullptr ? nullptr : OutputOperand->getIf(); if (Output == nullptr || Function.registerInfo().useEmpty(*Output)) continue; Register Extracted = *Source; const std::uint64_t Offset = static_cast(Index) * OutputType->scalarBitWidth(); if (Offset != 0) { auto Amount = buildIntegerConstantBefore(Builder, Instruction, 64, Offset); if (!Amount) return std::unexpected(Amount.error()); auto Shifted = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_LSHR, *WideType, *Source, *Amount); if (!Shifted) return std::unexpected(Shifted.error()); Extracted = *Shifted; } MachineInstr *Extension = singleUseInstruction(Function, *Output); const bool FoldZExt = Extension != nullptr && Extension->opcode() == opcode(GenericOpcode::G_ZEXT) && operandType(Function, *Extension, 0) .transform([](LLT Type) { return Type.scalarBitWidth() == 64; }) .value_or(false); if (FoldZExt) { const MachineOperand *ExtendedOperand = Extension->operandAt(0); const Register *Extended = ExtendedOperand == nullptr ? nullptr : ExtendedOperand->getIf(); if (Extended == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V unmerge extension has no destination"); const std::uint64_t Mask = OutputType->scalarBitWidth() == 64 ? std::numeric_limits::max() : (std::uint64_t{1} << OutputType->scalarBitWidth()) - 1U; auto MaskReg = buildIntegerConstantBefore(Builder, Instruction, 64, Mask); if (!MaskReg) return std::unexpected(MaskReg.error()); auto Masked = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_AND, *WideType, Extracted, *MaskReg); if (!Masked) return std::unexpected(Masked.error()); auto Replaced = replaceRegisterUses(Function, *Extended, *Masked); if (!Replaced) return std::unexpected(Replaced.error()); FoldedExtensions.push_back(Extension); continue; } auto Truncated = buildUnaryBefore(Builder, Instruction, GenericOpcode::G_TRUNC, *OutputType, Extracted); if (!Truncated) return std::unexpected(Truncated.error()); auto Replaced = replaceRegisterUses(Function, *Output, *Truncated); if (!Replaced) return std::unexpected(Replaced.error()); } for (MachineInstr *Extension : FoldedExtensions) { auto Erased = eraseInstruction(*Extension); if (!Erased) return std::unexpected(Erased.error()); } return eraseInstruction(Instruction); } struct WideValueParts final { std::uint32_t BitWidth = 0; std::vector Parts; std::vector Artifacts; }; [[nodiscard]] Expected collectWideValueParts(Register Value, MachineFunction &Function, unsigned Depth = 0) { if (Depth > 16) return makeUnexpected(ErrorCode::Unsupported, "RISC-V wide-value artifact chain is too deep"); const auto Info = Function.registerInfo().virtualRegisterInfo(Value); if (!Info || !Info->get().Type) return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide value has no scalar type"); const LLT Type = *Info->get().Type; if ((!Type.isInteger() && !Type.isAnyScalar()) || Type.scalarBitWidth() == 0) return makeUnexpected(ErrorCode::Unsupported, "RISC-V wide-value decomposition requires an integer scalar"); if (Type.scalarBitWidth() <= 64) return WideValueParts{.BitWidth = Type.scalarBitWidth(), .Parts = {Value}, .Artifacts = {}}; const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(Value); MachineInstr *Def = Definition == nullptr ? nullptr : const_cast(Definition->parent()); if (Def == nullptr) return makeUnexpected(ErrorCode::Unsupported, "RISC-V wide value has no decomposable definition"); if (Def->opcode() == opcode(GenericOpcode::G_MERGE_VALUES)) { WideValueParts Result; Result.BitWidth = Type.scalarBitWidth(); for (std::size_t Index = 1; Index < Def->operandCount(); ++Index) { const MachineOperand *Operand = Def->operandAt(Index); const Register *Part = Operand == nullptr ? nullptr : Operand->getIf(); const std::optional PartType = Operand == nullptr ? std::nullopt : registerType(Function, *Operand); if (Part == nullptr || !PartType || PartType->scalarBitWidth() != 64) return makeUnexpected(ErrorCode::Unsupported, "RISC-V wide merge must already use 64-bit limbs"); Result.Parts.push_back(*Part); } Result.Artifacts.push_back(Def); return Result; } if (Def->opcode() == opcode(GenericOpcode::G_TRUNC)) { const MachineOperand *SourceOperand = Def->operandAt(1); const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); if (Source == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide truncation has no source register"); auto Result = collectWideValueParts(*Source, Function, Depth + 1U); if (!Result) return std::unexpected(Result.error()); Result->BitWidth = Type.scalarBitWidth(); const std::size_t RequiredParts = (Type.scalarBitWidth() + 63U) / 64U; if (Result->Parts.size() < RequiredParts) return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide truncation source has too few limbs"); Result->Parts.resize(RequiredParts); Result->Artifacts.push_back(Def); return Result; } return makeUnexpected(ErrorCode::Unsupported, "RISC-V wide value is not defined by merge/trunc artifacts"); } struct WideResultPlan final { std::vector Outputs; MachineInstr *Extension = nullptr; MachineInstr *Unmerge = nullptr; }; [[nodiscard]] Expected wideResultPlan(Register Result, MachineFunction &Function, std::size_t RequiredParts) { MachineInstr *Use = singleUseInstruction(Function, Result); WideResultPlan Plan; if (Use != nullptr && (Use->opcode() == opcode(GenericOpcode::G_ANYEXT) || Use->opcode() == opcode(GenericOpcode::G_SEXT) || Use->opcode() == opcode(GenericOpcode::G_ZEXT))) { Plan.Extension = Use; const MachineOperand *ExtendedOperand = Use->operandAt(0); const Register *Extended = ExtendedOperand == nullptr ? nullptr : ExtendedOperand->getIf(); Use = Extended == nullptr ? nullptr : singleUseInstruction(Function, *Extended); } if (Use == nullptr || Use->opcode() != opcode(GenericOpcode::G_UNMERGE_VALUES)) return makeUnexpected(ErrorCode::Unsupported, "RISC-V wide result must feed an unmerge artifact"); Plan.Unmerge = Use; const std::size_t OutputCount = Use->operandCount() - 1U; if (OutputCount < RequiredParts) return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide result unmerge has too few outputs"); Plan.Outputs.reserve(RequiredParts); for (std::size_t Index = 0; Index < RequiredParts; ++Index) { const MachineOperand *Operand = Use->operandAt(Index); const Register *Output = Operand == nullptr ? nullptr : Operand->getIf(); const std::optional Type = Operand == nullptr ? std::nullopt : registerType(Function, *Operand); if (Output == nullptr || !Type || Type->scalarBitWidth() != 64) return makeUnexpected(ErrorCode::Unsupported, "RISC-V wide result must unmerge to 64-bit limbs"); Plan.Outputs.push_back(*Output); } return Plan; } [[nodiscard]] Expected buildIntegerCompareBefore(MachineIRBuilder &Builder, MachineInstr &Anchor, IntegerPredicate Predicate, Register Left, Register Right) { auto Type = LLT::makeAnyScalar(64); if (!Type) return std::unexpected(Type.error()); auto Result = Builder.function().registerInfo().createVirtualRegister(*Type); if (!Result) return std::unexpected(Result.error()); auto Point = Builder.setInsertPointBefore(Anchor); if (!Point) return std::unexpected(Point.error()); auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_ICMP)); if (!Pending) return std::unexpected(Pending.error()); if (auto Added = Pending->addDef(*Result); !Added) return std::unexpected(Added.error()); if (auto Added = Pending->addIntegerPredicate(Predicate); !Added) return std::unexpected(Added.error()); if (auto Added = Pending->addUse(Left); !Added) return std::unexpected(Added.error()); if (auto Added = Pending->addUse(Right); !Added) return std::unexpected(Added.error()); auto Built = Pending->build(); if (!Built) return std::unexpected(Built.error()); return *Result; } [[nodiscard]] Expected buildSelectBefore(MachineIRBuilder &Builder, MachineInstr &Anchor, LLT Type, Register Condition, Register TrueValue, Register FalseValue) { auto Result = Builder.function().registerInfo().createVirtualRegister(Type); if (!Result) return std::unexpected(Result.error()); auto Point = Builder.setInsertPointBefore(Anchor); if (!Point) return std::unexpected(Point.error()); auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_SELECT)); if (!Pending) return std::unexpected(Pending.error()); if (auto Added = Pending->addDef(*Result); !Added) return std::unexpected(Added.error()); if (auto Added = Pending->addUse(Condition); !Added) return std::unexpected(Added.error()); if (auto Added = Pending->addUse(TrueValue); !Added) return std::unexpected(Added.error()); if (auto Added = Pending->addUse(FalseValue); !Added) return std::unexpected(Added.error()); auto Built = Pending->build(); if (!Built) return std::unexpected(Built.error()); return *Result; } struct TwoLimbShiftResult final { Register Low; Register High; }; [[nodiscard]] Expected emitTwoLimbLeftShiftBody(MachineIRBuilder &Builder, MachineInstr &Anchor, Register Low, Register High, Register Amount, Register AmountExcess, Register AmountLack, Register Zero, Register IsShort, Register IsZero) { auto Type = LLT::makeAnyScalar(64); if (!Type) return std::unexpected(Type.error()); auto LowShort = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SHL, *Type, Low, Amount); if (!LowShort) return std::unexpected(LowShort.error()); auto CrossLow = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_LSHR, *Type, Low, AmountLack); if (!CrossLow) return std::unexpected(CrossLow.error()); auto HighShort = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SHL, *Type, High, Amount); if (!HighShort) return std::unexpected(HighShort.error()); auto ShortHigh = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_OR, *Type, *CrossLow, *HighShort); if (!ShortHigh) return std::unexpected(ShortHigh.error()); auto LongHigh = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SHL, *Type, Low, AmountExcess); if (!LongHigh) return std::unexpected(LongHigh.error()); auto ResultLow = buildSelectBefore(Builder, Anchor, *Type, IsShort, *LowShort, Zero); if (!ResultLow) return std::unexpected(ResultLow.error()); auto ShiftedHigh = buildSelectBefore(Builder, Anchor, *Type, IsShort, *ShortHigh, *LongHigh); if (!ShiftedHigh) return std::unexpected(ShiftedHigh.error()); auto ResultHigh = buildSelectBefore(Builder, Anchor, *Type, IsZero, High, *ShiftedHigh); if (!ResultHigh) return std::unexpected(ResultHigh.error()); return TwoLimbShiftResult{.Low = *ResultLow, .High = *ResultHigh}; } [[nodiscard]] Expected emitTwoLimbLeftShift(MachineIRBuilder &Builder, MachineInstr &Anchor, Register Low, Register High, Register Amount) { auto Type = LLT::makeAnyScalar(64); if (!Type) return std::unexpected(Type.error()); auto Boundary = buildIntegerConstantBefore(Builder, Anchor, 64, 64); if (!Boundary) return std::unexpected(Boundary.error()); auto AmountExcess = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, Amount, *Boundary); if (!AmountExcess) return std::unexpected(AmountExcess.error()); auto AmountLack = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, *Boundary, Amount); if (!AmountLack) return std::unexpected(AmountLack.error()); auto Zero = buildIntegerConstantBefore(Builder, Anchor, 64, 0); if (!Zero) return std::unexpected(Zero.error()); auto IsShort = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan, Amount, *Boundary); if (!IsShort) return std::unexpected(IsShort.error()); auto IsZero = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::Equal, Amount, *Zero); if (!IsZero) return std::unexpected(IsZero.error()); return emitTwoLimbLeftShiftBody(Builder, Anchor, Low, High, Amount, *AmountExcess, *AmountLack, *Zero, *IsShort, *IsZero); } [[nodiscard]] Expected emitLowLimbOfTwoLimbLogicalRightShift(MachineIRBuilder &Builder, MachineInstr &Anchor, Register Low, Register High, Register Amount, Register Boundary, Register Zero) { auto Type = LLT::makeAnyScalar(64); if (!Type) return std::unexpected(Type.error()); auto AmountExcess = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, Amount, Boundary); if (!AmountExcess) return std::unexpected(AmountExcess.error()); auto AmountLack = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, Boundary, Amount); if (!AmountLack) return std::unexpected(AmountLack.error()); auto IsShort = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan, Amount, Boundary); if (!IsShort) return std::unexpected(IsShort.error()); auto IsZero = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::Equal, Amount, Zero); if (!IsZero) return std::unexpected(IsZero.error()); auto LowShort = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_LSHR, *Type, Low, Amount); if (!LowShort) return std::unexpected(LowShort.error()); auto HighShort = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SHL, *Type, High, *AmountLack); if (!HighShort) return std::unexpected(HighShort.error()); auto ShortValue = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_OR, *Type, *LowShort, *HighShort); if (!ShortValue) return std::unexpected(ShortValue.error()); auto LongValue = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_LSHR, *Type, High, *AmountExcess); if (!LongValue) return std::unexpected(LongValue.error()); auto Shifted = buildSelectBefore(Builder, Anchor, *Type, *IsShort, *ShortValue, *LongValue); if (!Shifted) return std::unexpected(Shifted.error()); return buildSelectBefore(Builder, Anchor, *Type, *IsZero, Low, *Shifted); } [[nodiscard]] Expected> emitThreeLimbLeftShift(MachineIRBuilder &Builder, MachineInstr &Anchor, std::span Parts, Register Amount) { if (Parts.size() != 3) return makeUnexpected(ErrorCode::VerificationError, "RISC-V three-limb shift requires exactly three limbs"); auto Type = LLT::makeAnyScalar(64); if (!Type) return std::unexpected(Type.error()); auto OuterBoundary = buildIntegerConstantBefore(Builder, Anchor, 64, 128); if (!OuterBoundary) return std::unexpected(OuterBoundary.error()); auto OuterExcess = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, Amount, *OuterBoundary); if (!OuterExcess) return std::unexpected(OuterExcess.error()); auto OuterLack = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, *OuterBoundary, Amount); if (!OuterLack) return std::unexpected(OuterLack.error()); auto Zero = buildIntegerConstantBefore(Builder, Anchor, 64, 0); if (!Zero) return std::unexpected(Zero.error()); auto OuterIsShort = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan, Amount, *OuterBoundary); if (!OuterIsShort) return std::unexpected(OuterIsShort.error()); auto OuterIsZero = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::Equal, Amount, *Zero); if (!OuterIsZero) return std::unexpected(OuterIsZero.error()); auto InnerBoundary = buildIntegerConstantBefore(Builder, Anchor, 64, 64); if (!InnerBoundary) return std::unexpected(InnerBoundary.error()); auto InnerExcess = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, Amount, *InnerBoundary); if (!InnerExcess) return std::unexpected(InnerExcess.error()); auto InnerLack = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, *InnerBoundary, Amount); if (!InnerLack) return std::unexpected(InnerLack.error()); auto InnerIsShort = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan, Amount, *InnerBoundary); if (!InnerIsShort) return std::unexpected(InnerIsShort.error()); auto InnerIsZero = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::Equal, Amount, *Zero); if (!InnerIsZero) return std::unexpected(InnerIsZero.error()); auto Lower = emitTwoLimbLeftShiftBody(Builder, Anchor, Parts[0], Parts[1], Amount, *InnerExcess, *InnerLack, *Zero, *InnerIsShort, *InnerIsZero); if (!Lower) return std::unexpected(Lower.error()); auto Cross = emitLowLimbOfTwoLimbLogicalRightShift(Builder, Anchor, Parts[0], Parts[1], *OuterLack, *InnerBoundary, *Zero); if (!Cross) return std::unexpected(Cross.error()); auto HighIsShort = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan, Amount, *InnerBoundary); if (!HighIsShort) return std::unexpected(HighIsShort.error()); auto HighShort = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SHL, *Type, Parts[2], Amount); if (!HighShort) return std::unexpected(HighShort.error()); auto BoundedHigh = buildSelectBefore(Builder, Anchor, *Type, *HighIsShort, *HighShort, *Zero); if (!BoundedHigh) return std::unexpected(BoundedHigh.error()); auto ShortHigh = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_OR, *Type, *Cross, *BoundedHigh); if (!ShortHigh) return std::unexpected(ShortHigh.error()); auto LongIsShort = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan, *OuterExcess, *InnerBoundary); if (!LongIsShort) return std::unexpected(LongIsShort.error()); auto LongShift = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SHL, *Type, Parts[0], *OuterExcess); if (!LongShift) return std::unexpected(LongShift.error()); auto LongHigh = buildSelectBefore(Builder, Anchor, *Type, *LongIsShort, *LongShift, *Zero); if (!LongHigh) return std::unexpected(LongHigh.error()); auto ResultLow = buildSelectBefore(Builder, Anchor, *Type, *OuterIsShort, Lower->Low, *Zero); if (!ResultLow) return std::unexpected(ResultLow.error()); auto ResultMiddle = buildSelectBefore(Builder, Anchor, *Type, *OuterIsShort, Lower->High, *Zero); if (!ResultMiddle) return std::unexpected(ResultMiddle.error()); auto ShiftedHigh = buildSelectBefore(Builder, Anchor, *Type, *OuterIsShort, *ShortHigh, *LongHigh); if (!ShiftedHigh) return std::unexpected(ShiftedHigh.error()); auto ResultHigh = buildSelectBefore(Builder, Anchor, *Type, *OuterIsZero, Parts[2], *ShiftedHigh); if (!ResultHigh) return std::unexpected(ResultHigh.error()); return std::vector{*ResultLow, *ResultMiddle, *ResultHigh}; } [[nodiscard]] Expected lowerWideShiftLeft(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target) { const MachineOperand *DestinationOperand = Instruction.operandAt(0); const MachineOperand *ValueOperand = Instruction.operandAt(1); const MachineOperand *AmountOperand = Instruction.operandAt(2); const Register *Destination = DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); const Register *Value = ValueOperand == nullptr ? nullptr : ValueOperand->getIf(); const Register *Amount = AmountOperand == nullptr ? nullptr : AmountOperand->getIf(); if (Destination == nullptr || Value == nullptr || Amount == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide shift has invalid register operands"); auto ValueParts = collectWideValueParts(*Value, Function); if (!ValueParts) return std::unexpected(ValueParts.error()); auto AmountParts = collectWideValueParts(*Amount, Function); if (!AmountParts) return std::unexpected(AmountParts.error()); if (ValueParts->Parts.size() < 2 || AmountParts->Parts.empty()) return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide shift has an invalid limb decomposition"); auto ResultPlan = wideResultPlan(*Destination, Function, ValueParts->Parts.size()); if (!ResultPlan) return std::unexpected(ResultPlan.error()); MachineIRBuilder Builder(Function, Target); Expected> Results = ValueParts->Parts.size() == 2 ? emitTwoLimbLeftShift(Builder, Instruction, ValueParts->Parts[0], ValueParts->Parts[1], AmountParts->Parts[0]) .transform([](TwoLimbShiftResult Result) { return std::vector{Result.Low, Result.High}; }) : ValueParts->Parts.size() == 3 ? emitThreeLimbLeftShift(Builder, Instruction, ValueParts->Parts, AmountParts->Parts[0]) : Expected>(std::unexpected( Error(ErrorCode::Unsupported, "RISC-V wide shift currently requires two or three 64-bit limbs"))); if (!Results) return std::unexpected(Results.error()); for (std::size_t Index = 0; Index < Results->size(); ++Index) { auto Replaced = replaceRegisterUses(Function, ResultPlan->Outputs[Index], (*Results)[Index]); if (!Replaced) return std::unexpected(Replaced.error()); } if (auto Erased = eraseInstruction(*ResultPlan->Unmerge); !Erased) return std::unexpected(Erased.error()); if (ResultPlan->Extension != nullptr) { if (auto Erased = eraseInstruction(*ResultPlan->Extension); !Erased) return std::unexpected(Erased.error()); } if (auto Erased = eraseInstruction(Instruction); !Erased) return std::unexpected(Erased.error()); for (std::size_t Index = 1; Index < AmountParts->Parts.size(); ++Index) { const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(AmountParts->Parts[Index]); if (Definition != nullptr && Definition->parent() != nullptr) AmountParts->Artifacts.push_back(const_cast(Definition->parent())); } ValueParts->Artifacts.insert(ValueParts->Artifacts.end(), AmountParts->Artifacts.begin(), AmountParts->Artifacts.end()); eraseDeadArtifacts(ValueParts->Artifacts, Function); return {}; } [[nodiscard]] Expected> emitWideAdd(MachineIRBuilder &Builder, MachineInstr &Anchor, std::span Left, std::span Right) { if (Left.size() != Right.size() || Left.size() < 2) return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide add requires matching multi-limb operands"); auto Type = LLT::makeAnyScalar(64); if (!Type) return std::unexpected(Type.error()); std::vector Results; Results.reserve(Left.size()); auto Low = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_ADD, *Type, Left[0], Right[0]); if (!Low) return std::unexpected(Low.error()); auto Carry = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan, *Low, Right[0]); if (!Carry) return std::unexpected(Carry.error()); auto LowCopy = buildUnaryBefore(Builder, Anchor, GenericOpcode::COPY, *Type, *Low); if (!LowCopy) return std::unexpected(LowCopy.error()); Results.push_back(*LowCopy); for (std::size_t Index = 1; Index < Left.size(); ++Index) { auto Base = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_ADD, *Type, Left[Index], Right[Index]); if (!Base) return std::unexpected(Base.error()); std::optional BaseCarry; if (Index + 1U < Left.size()) { auto Compared = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan, *Base, Left[Index]); if (!Compared) return std::unexpected(Compared.error()); BaseCarry = *Compared; } auto Sum = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_ADD, *Type, *Base, *Carry); if (!Sum) return std::unexpected(Sum.error()); if (Index + 1U < Left.size()) { auto Zero = buildIntegerConstantBefore(Builder, Anchor, 64, 0); if (!Zero) return std::unexpected(Zero.error()); auto IsZero = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::Equal, *Sum, *Zero); if (!IsZero) return std::unexpected(IsZero.error()); auto CarryFromIncoming = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_AND, *Type, *IsZero, *Carry); if (!CarryFromIncoming) return std::unexpected(CarryFromIncoming.error()); auto NextCarry = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_OR, *Type, *BaseCarry, *CarryFromIncoming); if (!NextCarry) return std::unexpected(NextCarry.error()); Carry = *NextCarry; } auto Copy = buildUnaryBefore(Builder, Anchor, GenericOpcode::COPY, *Type, *Sum); if (!Copy) return std::unexpected(Copy.error()); Results.push_back(*Copy); } return Results; } [[nodiscard]] Expected> emitWideSub(MachineIRBuilder &Builder, MachineInstr &Anchor, std::span Left, std::span Right) { if (Left.size() != Right.size() || Left.size() < 2) return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide sub requires matching multi-limb operands"); auto Type = LLT::makeAnyScalar(64); if (!Type) return std::unexpected(Type.error()); std::vector Results; Results.reserve(Left.size()); auto Low = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, Left[0], Right[0]); if (!Low) return std::unexpected(Low.error()); Results.push_back(*Low); auto Borrow = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan, Left[0], Right[0]); if (!Borrow) return std::unexpected(Borrow.error()); for (std::size_t Index = 1; Index < Left.size(); ++Index) { auto Base = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, Left[Index], Right[Index]); if (!Base) return std::unexpected(Base.error()); std::optional BaseBorrow; if (Index + 1U < Left.size()) { auto Compared = buildIntegerCompareBefore( Builder, Anchor, IntegerPredicate::UnsignedGreaterThan, *Base, Left[Index]); if (!Compared) return std::unexpected(Compared.error()); BaseBorrow = *Compared; } auto Difference = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, *Base, *Borrow); if (!Difference) return std::unexpected(Difference.error()); Results.push_back(*Difference); if (Index + 1U < Left.size()) { auto Zero = buildIntegerConstantBefore(Builder, Anchor, 64, 0); if (!Zero) return std::unexpected(Zero.error()); auto IsZero = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::Equal, *Base, *Zero); if (!IsZero) return std::unexpected(IsZero.error()); auto BorrowFromIncoming = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_AND, *Type, *IsZero, *Borrow); if (!BorrowFromIncoming) return std::unexpected(BorrowFromIncoming.error()); auto NextBorrow = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_OR, *Type, *BaseBorrow, *BorrowFromIncoming); if (!NextBorrow) return std::unexpected(NextBorrow.error()); Borrow = *NextBorrow; } } return Results; } [[nodiscard]] Expected> emitWideBitwise(MachineIRBuilder &Builder, MachineInstr &Anchor, GenericOpcode Generic, std::span Left, std::span Right) { if (Left.size() != Right.size() || Left.size() < 2) return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide bitwise operation requires matching multi-limb operands"); auto Type = LLT::makeAnyScalar(64); if (!Type) return std::unexpected(Type.error()); std::vector Results; Results.reserve(Left.size()); for (std::size_t Index = 0; Index < Left.size(); ++Index) { auto Part = buildBinaryBefore(Builder, Anchor, Generic, *Type, Left[Index], Right[Index]); if (!Part) return std::unexpected(Part.error()); Results.push_back(*Part); } return Results; } [[nodiscard]] Expected lowerWideBinary(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target, GenericOpcode Generic) { const MachineOperand *DestinationOperand = Instruction.operandAt(0); const MachineOperand *LeftOperand = Instruction.operandAt(1); const MachineOperand *RightOperand = Instruction.operandAt(2); const Register *Destination = DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf(); const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf(); if (Destination == nullptr || Left == nullptr || Right == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide binary instruction has invalid operands"); auto LeftParts = collectWideValueParts(*Left, Function); if (!LeftParts) return std::unexpected(LeftParts.error()); auto RightParts = collectWideValueParts(*Right, Function); if (!RightParts) return std::unexpected(RightParts.error()); if (LeftParts->Parts.size() != RightParts->Parts.size()) return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide binary operands have different limb counts"); auto ResultPlan = wideResultPlan(*Destination, Function, LeftParts->Parts.size()); if (!ResultPlan) return std::unexpected(ResultPlan.error()); MachineIRBuilder Builder(Function, Target); Expected> Results = Generic == GenericOpcode::G_ADD ? emitWideAdd(Builder, Instruction, LeftParts->Parts, RightParts->Parts) : Generic == GenericOpcode::G_SUB ? emitWideSub(Builder, Instruction, LeftParts->Parts, RightParts->Parts) : emitWideBitwise(Builder, Instruction, Generic, LeftParts->Parts, RightParts->Parts); if (!Results) return std::unexpected(Results.error()); for (std::size_t Index = 0; Index < Results->size(); ++Index) { auto Replaced = replaceRegisterUses(Function, ResultPlan->Outputs[Index], (*Results)[Index]); if (!Replaced) return std::unexpected(Replaced.error()); } if (auto Erased = eraseInstruction(*ResultPlan->Unmerge); !Erased) return std::unexpected(Erased.error()); if (ResultPlan->Extension != nullptr) { if (auto Erased = eraseInstruction(*ResultPlan->Extension); !Erased) return std::unexpected(Erased.error()); } if (auto Erased = eraseInstruction(Instruction); !Erased) return std::unexpected(Erased.error()); LeftParts->Artifacts.insert(LeftParts->Artifacts.end(), RightParts->Artifacts.begin(), RightParts->Artifacts.end()); eraseDeadArtifacts(LeftParts->Artifacts, Function); return {}; } [[nodiscard]] bool canLowerNarrowBranchCondition(const MachineInstr &Instruction, const MachineFunction &Function) { if (Instruction.opcode() != opcode(GenericOpcode::G_BRCOND) || Instruction.operandCount() < 2) return false; const MachineOperand *ConditionOperand = Instruction.operandAt(0); const Register *Condition = ConditionOperand == nullptr ? nullptr : ConditionOperand->getIf(); if (Condition == nullptr || !Condition->isVirtual() || singleRegisterUse(Function, *Condition) != &Instruction) return false; const auto ConditionInfo = Function.registerInfo().virtualRegisterInfo(*Condition); const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(*Condition); const MachineInstr *Truncation = Definition == nullptr ? nullptr : Definition->parent(); if (!ConditionInfo || !ConditionInfo->get().Type || ConditionInfo->get().Type->scalarBitWidth() != 1 || Truncation == nullptr || Truncation->opcode() != opcode(GenericOpcode::G_TRUNC) || Truncation->operandCount() != 2) return false; return operandType(Function, *Truncation, 1) .transform([](LLT Type) { return isScalarGPRType(Type) && Type.scalarBitWidth() == 64; }) .value_or(false); } [[nodiscard]] Expected lowerNarrowBranchCondition(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target) { if (!canLowerNarrowBranchCondition(Instruction, Function)) return makeUnexpected(ErrorCode::Unsupported, "RISC-V branch condition is not a narrow truncation"); const MachineOperand *ConditionOperand = Instruction.operandAt(0); const Register Condition = *ConditionOperand->getIf(); const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(Condition); MachineInstr *Truncation = const_cast(Definition->parent()); const Register Source = *Truncation->operandAt(1)->getIf(); auto Type = LLT::makeAnyScalar(64); if (!Type) return std::unexpected(Type.error()); MachineIRBuilder Builder(Function, Target); auto One = buildIntegerConstantBefore(Builder, Instruction, 64, 1); if (!One) return std::unexpected(One.error()); auto Masked = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_AND, *Type, Source, *One); if (!Masked) return std::unexpected(Masked.error()); auto Replaced = Instruction.operandAt(0)->setRegister(*Masked); if (!Replaced) return std::unexpected(Replaced.error()); if (Function.registerInfo().useEmpty(Condition)) return eraseInstruction(*Truncation); return {}; } struct NarrowPhiIncoming final { Register Value; MachineBasicBlock *Block = nullptr; MachineInstr *ReusableTruncation = nullptr; }; [[nodiscard]] MachineInstr *firstInstructionAfterLeadingCopies(MachineBasicBlock &Block) { MachineInstr *Instruction = Block.firstInstruction(); while (Instruction != nullptr && Instruction->opcode() == opcode(GenericOpcode::COPY)) Instruction = Instruction->next(); return Instruction; } [[nodiscard]] Expected buildIncomingConversion(MachineFunction &Function, const TargetDescription &Target, MachineBasicBlock &Block, MachineInstr *Anchor, GenericOpcode Generic, LLT Type, Register Source) { auto Result = Function.registerInfo().createVirtualRegister(Type); if (!Result) return std::unexpected(Result.error()); MachineIRBuilder Builder(Function, Target); auto Point = Anchor == nullptr ? Builder.setInsertPoint(Block) : Builder.setInsertPointBefore(*Anchor); if (!Point) return std::unexpected(Point.error()); auto Pending = Builder.buildInstr(opcode(Generic)); if (!Pending) return std::unexpected(Pending.error()); if (auto Added = Pending->addDef(*Result); !Added) return std::unexpected(Added.error()); if (auto Added = Pending->addUse(Source); !Added) return std::unexpected(Added.error()); auto Built = Pending->build(); if (!Built) return std::unexpected(Built.error()); return *Result; } [[nodiscard]] Expected lowerNarrowPhi(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target) { const std::optional OriginalType = resultType(Function, Instruction); const MachineOperand *DestinationOperand = Instruction.operandAt(0); const Register *Destination = DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); if (!OriginalType || Destination == nullptr || OriginalType->scalarBitWidth() >= 32 || Instruction.operandCount() < 3 || (Instruction.operandCount() % 2U) == 0) return makeUnexpected(ErrorCode::VerificationError, "RISC-V narrow PHI has invalid operands"); auto LegalType = LLT::makeAnyScalar(32); if (!LegalType) return std::unexpected(LegalType.error()); std::vector Incoming; Incoming.reserve((Instruction.operandCount() - 1U) / 2U); for (std::size_t Index = 1; Index < Instruction.operandCount(); Index += 2) { const MachineOperand *ValueOperand = Instruction.operandAt(Index); const Register *Value = ValueOperand == nullptr ? nullptr : ValueOperand->getIf(); const MachineOperand *BlockOperand = Instruction.operandAt(Index + 1U); const auto *Block = BlockOperand == nullptr ? nullptr : BlockOperand->getIf(); if (Value == nullptr || Block == nullptr || Block->Block == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V narrow PHI has an invalid incoming edge"); MachineInstr *Reusable = nullptr; const MachineOperand *Definition = Value->isVirtual() ? Function.registerInfo().getUniqueVRegDef(*Value) : nullptr; MachineInstr *Def = Definition == nullptr ? nullptr : const_cast(Definition->parent()); if (Def != nullptr && Def->opcode() == opcode(GenericOpcode::G_TRUNC) && Def->operandCount() == 2 && singleRegisterUse(Function, *Value) == &Instruction && operandType(Function, *Def, 1) .transform([](LLT Type) { return Type.scalarBitWidth() >= 32; }) .value_or(false)) { Reusable = Def; } Incoming.push_back( NarrowPhiIncoming{.Value = *Value, .Block = Block->Block, .ReusableTruncation = Reusable}); } for (NarrowPhiIncoming &Entry : Incoming) { if (Entry.ReusableTruncation == nullptr) continue; const std::size_t Count = static_cast(std::ranges::count( Incoming, Entry.ReusableTruncation, &NarrowPhiIncoming::ReusableTruncation)); if (Count != 1) Entry.ReusableTruncation = nullptr; } for (NarrowPhiIncoming &Entry : Incoming) { if (Entry.ReusableTruncation == nullptr) continue; MachineBasicBlock *OriginalBlock = Entry.ReusableTruncation->parent(); if (OriginalBlock == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V narrow PHI input truncation is detached"); auto Removed = OriginalBlock->removeInstruction(*Entry.ReusableTruncation); if (!Removed) return std::unexpected(Removed.error()); } std::vector> Anchors; for (const NarrowPhiIncoming &Entry : Incoming) { if (std::ranges::find(Anchors, Entry.Block, &decltype(Anchors)::value_type::first) == Anchors.end()) { Anchors.emplace_back(Entry.Block, firstInstructionAfterLeadingCopies(*Entry.Block)); } } std::vector LegalIncoming; LegalIncoming.reserve(Incoming.size()); for (NarrowPhiIncoming &Entry : Incoming) { const auto AnchorPosition = std::ranges::find(Anchors, Entry.Block, &decltype(Anchors)::value_type::first); MachineInstr *Anchor = AnchorPosition->second; if (Entry.ReusableTruncation != nullptr) { auto SetType = Function.registerInfo().setType(Entry.Value, *LegalType); if (!SetType) return std::unexpected(SetType.error()); auto Inserted = Anchor == nullptr ? Entry.Block->appendInstruction(*Entry.ReusableTruncation) : Entry.Block->insertInstructionBefore(*Anchor, *Entry.ReusableTruncation); if (!Inserted) return std::unexpected(Inserted.error()); LegalIncoming.push_back(Entry.Value); continue; } const auto ValueInfo = Function.registerInfo().virtualRegisterInfo(Entry.Value); if (!ValueInfo || !ValueInfo->get().Type) return makeUnexpected(ErrorCode::VerificationError, "RISC-V narrow PHI input has no scalar type"); const std::uint32_t Width = ValueInfo->get().Type->scalarBitWidth(); const GenericOpcode Conversion = Width < 32 ? GenericOpcode::G_ANYEXT : Width > 32 ? GenericOpcode::G_TRUNC : GenericOpcode::COPY; auto Converted = buildIncomingConversion(Function, Target, *Entry.Block, Anchor, Conversion, *LegalType, Entry.Value); if (!Converted) return std::unexpected(Converted.error()); LegalIncoming.push_back(*Converted); } auto LegalResult = Function.registerInfo().createVirtualRegister(*LegalType); if (!LegalResult) return std::unexpected(LegalResult.error()); MachineIRBuilder Builder(Function, Target); auto Point = Builder.setInsertPointBefore(Instruction); if (!Point) return std::unexpected(Point.error()); auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_PHI)); if (!Pending) return std::unexpected(Pending.error()); if (auto Added = Pending->addDef(*LegalResult); !Added) return std::unexpected(Added.error()); for (std::size_t Index = 0; Index < Incoming.size(); ++Index) { if (auto Added = Pending->addUse(LegalIncoming[Index]); !Added) return std::unexpected(Added.error()); if (auto Added = Pending->addBasicBlock(*Incoming[Index].Block); !Added) return std::unexpected(Added.error()); } auto Built = Pending->build(); if (!Built) return std::unexpected(Built.error()); auto Replaced = replaceRegisterUses(Function, *Destination, *LegalResult); if (!Replaced) return std::unexpected(Replaced.error()); return eraseInstruction(Instruction); } [[nodiscard]] Expected lowerWidePhi(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target) { const MachineOperand *DestinationOperand = Instruction.operandAt(0); const Register *Destination = DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); if (Destination == nullptr || Instruction.operandCount() < 5 || (Instruction.operandCount() % 2U) == 0) return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide PHI has invalid operands"); std::vector IncomingParts; std::vector IncomingBlocks; for (std::size_t Index = 1; Index < Instruction.operandCount(); Index += 2) { const MachineOperand *ValueOperand = Instruction.operandAt(Index); const Register *Value = ValueOperand == nullptr ? nullptr : ValueOperand->getIf(); const MachineOperand *BlockOperand = Instruction.operandAt(Index + 1U); const auto *Block = BlockOperand == nullptr ? nullptr : BlockOperand->getIf(); if (Value == nullptr || Block == nullptr || Block->Block == nullptr) return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide PHI has an invalid incoming edge"); auto Parts = collectWideValueParts(*Value, Function); if (!Parts) return std::unexpected(Parts.error()); if (!IncomingParts.empty() && Parts->Parts.size() != IncomingParts.front().Parts.size()) return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide PHI incoming values have different limb counts"); IncomingParts.push_back(std::move(*Parts)); IncomingBlocks.push_back(Block->Block); } const std::size_t PartCount = IncomingParts.front().Parts.size(); auto ResultPlan = wideResultPlan(*Destination, Function, PartCount); if (!ResultPlan) return std::unexpected(ResultPlan.error()); auto Type = LLT::makeAnyScalar(64); if (!Type) return std::unexpected(Type.error()); MachineIRBuilder Builder(Function, Target); std::vector Results; Results.reserve(PartCount); for (std::size_t PartIndex = 0; PartIndex < PartCount; ++PartIndex) { auto Result = Function.registerInfo().createVirtualRegister(*Type); if (!Result) return std::unexpected(Result.error()); auto Point = Builder.setInsertPointBefore(Instruction); if (!Point) return std::unexpected(Point.error()); auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_PHI)); if (!Pending) return std::unexpected(Pending.error()); if (auto Added = Pending->addDef(*Result); !Added) return std::unexpected(Added.error()); for (std::size_t IncomingIndex = 0; IncomingIndex < IncomingParts.size(); ++IncomingIndex) { if (auto Added = Pending->addUse(IncomingParts[IncomingIndex].Parts[PartIndex]); !Added) return std::unexpected(Added.error()); if (auto Added = Pending->addBasicBlock(*IncomingBlocks[IncomingIndex]); !Added) return std::unexpected(Added.error()); } auto Built = Pending->build(); if (!Built) return std::unexpected(Built.error()); Results.push_back(*Result); } for (std::size_t Index = 0; Index < PartCount; ++Index) { auto Replaced = replaceRegisterUses(Function, ResultPlan->Outputs[Index], Results[Index]); if (!Replaced) return std::unexpected(Replaced.error()); } if (auto Erased = eraseInstruction(*ResultPlan->Unmerge); !Erased) return std::unexpected(Erased.error()); if (ResultPlan->Extension != nullptr) { if (auto Erased = eraseInstruction(*ResultPlan->Extension); !Erased) return std::unexpected(Erased.error()); } if (auto Erased = eraseInstruction(Instruction); !Erased) return std::unexpected(Erased.error()); std::vector Artifacts; for (WideValueParts &Parts : IncomingParts) Artifacts.insert(Artifacts.end(), Parts.Artifacts.begin(), Parts.Artifacts.end()); eraseDeadArtifacts(Artifacts, Function); return {}; } [[nodiscard]] bool canFoldWideMergeImmediately(const MachineInstr &Instruction, const MachineFunction &Function) { const MachineOperand *LowOperand = Instruction.operandAt(1); const Register *Low = LowOperand == nullptr ? nullptr : LowOperand->getIf(); if (Low == nullptr) return false; const auto LowInfo = Function.registerInfo().virtualRegisterInfo(*Low); if (!LowInfo || !LowInfo->get().Type) return false; const MachineOperand *DestinationOperand = Instruction.operandAt(0); const Register *Destination = DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); if (Destination == nullptr) return false; auto &RegisterInfo = const_cast(Function.registerInfo()); for (MachineOperand &Use : RegisterInfo.useOperands(*Destination)) { MachineInstr *Trunc = Use.parent(); if (Trunc == nullptr || Trunc->opcode() != opcode(GenericOpcode::G_TRUNC) || Trunc->operandAt(1) != &Use) return false; const std::optional Result = resultType(Function, *Trunc); if (!Result || Result->scalarBitWidth() > LowInfo->get().Type->scalarBitWidth()) return false; } return true; } struct RV64ScalarLayout final { std::uint64_t StoreSize = 0; std::uint64_t AllocationSize = 0; std::uint64_t ABIAlignment = 0; }; [[nodiscard]] std::optional rv64ScalarLayout(LLT Type) noexcept { if (Type.isPointer()) { return Type.addressSpace() == 0 && Type.scalarBitWidth() == 64 ? std::optional{{8, 8, 8}} : std::nullopt; } if ((!isSupportedIntegerType(Type) && !isRVFloatType(Type)) || Type.scalarBitWidth() == 0) { return std::nullopt; } const std::uint64_t StoreBytes = (Type.scalarBitWidth() + 7U) / 8U; const std::uint64_t AllocationBytes = std::bit_ceil(StoreBytes); return RV64ScalarLayout{StoreBytes, AllocationBytes, std::min(AllocationBytes, 8)}; } [[nodiscard]] Expected createMemoryAccess(MachineFunction &Function, MemoryAccessFlag Access, LLT Type, std::uint64_t SizeBytes, std::uint64_t AlignmentBytes) { auto Size = MemorySize::fixed(SizeBytes); auto Align = Alignment::create(AlignmentBytes); if (!Size || !Align) return makeUnexpected(ErrorCode::InternalError, "RISC-V memory facts are invalid"); MachineMemOperandDesc Description; Description.Accesses = {Access}; Description.Size = *Size; Description.MemoryType = Type; Description.Align = *Align; return Function.createMemOperand(std::move(Description)); } struct VAArgPlan final { Register Destination; Register ListPointer; LLT DestinationType; LLT PointerType; std::uint64_t Alignment = 0; RV64ScalarLayout ValueLayout; }; [[nodiscard]] std::optional vaargPlan(const MachineInstr &Instruction, const MachineFunction &Function) { if (Instruction.opcode() != opcode(GenericOpcode::G_VAARG) || Instruction.operandCount() != 3 || !Instruction.memOperands().empty()) { return std::nullopt; } const MachineOperand *DestinationOperand = Instruction.operandAt(0); const MachineOperand *ListOperand = Instruction.operandAt(1); const MachineOperand *AlignmentOperand = Instruction.operandAt(2); const Register *Destination = DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); const Register *ListPointer = ListOperand == nullptr ? nullptr : ListOperand->getIf(); const std::optional DestinationType = resultType(Function, Instruction); const std::optional PointerType = operandType(Function, Instruction, 1); const auto AlignmentValue = AlignmentOperand == nullptr ? Expected(std::unexpected( Error(ErrorCode::InvalidArgument, "missing alignment"))) : signedImmediate(*AlignmentOperand); const std::optional ValueLayout = DestinationType ? rv64ScalarLayout(*DestinationType) : std::nullopt; if (Destination == nullptr || ListPointer == nullptr || !Destination->isVirtual() || !ListPointer->isVirtual() || !DestinationOperand->isDef() || !ListOperand->isUse() || !DestinationType || !PointerType || !PointerType->isPointer() || PointerType->addressSpace() != 0 || PointerType->scalarBitWidth() != 64 || !AlignmentValue || *AlignmentValue <= 0 || !std::has_single_bit(static_cast(*AlignmentValue)) || !ValueLayout) { return std::nullopt; } return VAArgPlan{*Destination, *ListPointer, *DestinationType, *PointerType, static_cast(*AlignmentValue), *ValueLayout}; } [[nodiscard]] Expected lowerVAArg(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target) { const auto Plan = vaargPlan(Instruction, Function); if (!Plan) { return makeUnexpected(ErrorCode::VerificationError, "RISC-V va_arg legalization requires a scalar result, p0 list, and " "power-of-two alignment"); } auto ListLoadMemory = createMemoryAccess(Function, MemoryAccessFlag::Load, Plan->PointerType, 8, 8); if (!ListLoadMemory) return std::unexpected(ListLoadMemory.error()); auto ListStoreMemory = createMemoryAccess(Function, MemoryAccessFlag::Store, Plan->PointerType, 8, 8); if (!ListStoreMemory) return std::unexpected(ListStoreMemory.error()); auto CurrentList = Function.registerInfo().createVirtualRegister(Plan->PointerType); if (!CurrentList) return std::unexpected(CurrentList.error()); MachineIRBuilder Builder(Function, Target); if (auto Point = Builder.setInsertPointBefore(Instruction); !Point) return std::unexpected(Point.error()); auto ListLoad = Builder.buildInstr(opcode(GenericOpcode::G_LOAD)); if (!ListLoad) return std::unexpected(ListLoad.error()); if (auto Added = ListLoad->addDef(*CurrentList); !Added) return std::unexpected(Added.error()); if (auto Added = ListLoad->addUse(Plan->ListPointer); !Added) return std::unexpected(Added.error()); if (auto Added = ListLoad->addMemOperand(**ListLoadMemory); !Added) return std::unexpected(Added.error()); if (auto Built = ListLoad->build(); !Built) return std::unexpected(Built.error()); Register ValueAddress = *CurrentList; if (Plan->Alignment > 1) { auto AlignmentOffset = buildIntegerConstantBefore(Builder, Instruction, 64, Plan->Alignment - 1U); if (!AlignmentOffset) return std::unexpected(AlignmentOffset.error()); auto BiasedAddress = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_PTR_ADD, Plan->PointerType, ValueAddress, *AlignmentOffset); if (!BiasedAddress) return std::unexpected(BiasedAddress.error()); auto AlignmentMask = buildIntegerConstantBefore(Builder, Instruction, 64, ~(Plan->Alignment - std::uint64_t{1})); if (!AlignmentMask) return std::unexpected(AlignmentMask.error()); auto AlignedAddress = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_PTRMASK, Plan->PointerType, *BiasedAddress, *AlignmentMask); if (!AlignedAddress) return std::unexpected(AlignedAddress.error()); ValueAddress = *AlignedAddress; } auto Increment = buildIntegerConstantBefore(Builder, Instruction, 64, Plan->ValueLayout.AllocationSize); if (!Increment) return std::unexpected(Increment.error()); auto NextList = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_PTR_ADD, Plan->PointerType, ValueAddress, *Increment); if (!NextList) return std::unexpected(NextList.error()); auto ListStore = Builder.buildInstr(opcode(GenericOpcode::G_STORE)); if (!ListStore) return std::unexpected(ListStore.error()); if (auto Added = ListStore->addUse(*NextList); !Added) return std::unexpected(Added.error()); if (auto Added = ListStore->addUse(Plan->ListPointer); !Added) return std::unexpected(Added.error()); if (auto Added = ListStore->addMemOperand(**ListStoreMemory); !Added) return std::unexpected(Added.error()); if (auto Built = ListStore->build(); !Built) return std::unexpected(Built.error()); if (!Function.registerInfo().useEmpty(Plan->Destination)) { auto ValueLoadMemory = createMemoryAccess(Function, MemoryAccessFlag::Load, Plan->DestinationType, Plan->ValueLayout.StoreSize, Plan->ValueLayout.ABIAlignment); if (!ValueLoadMemory) return std::unexpected(ValueLoadMemory.error()); auto ValueLoad = Builder.buildInstr(opcode(GenericOpcode::G_LOAD)); if (!ValueLoad) return std::unexpected(ValueLoad.error()); if (auto Added = ValueLoad->addDef(Plan->Destination); !Added) return std::unexpected(Added.error()); if (auto Added = ValueLoad->addUse(ValueAddress); !Added) return std::unexpected(Added.error()); if (auto Added = ValueLoad->addMemOperand(**ValueLoadMemory); !Added) return std::unexpected(Added.error()); if (auto Built = ValueLoad->build(); !Built) return std::unexpected(Built.error()); } return eraseInstruction(Instruction); } struct VACopyPlan final { Register Destination; Register Source; LLT PointerType; }; [[nodiscard]] std::optional vacopyPlan(const MachineInstr &Instruction, const MachineFunction &Function) { if (Instruction.opcode() != opcode(GenericOpcode::G_INTRINSIC_W_SIDE_EFFECTS) || Instruction.operandCount() != 3 || !Instruction.memOperands().empty()) { return std::nullopt; } const MachineOperand *IntrinsicIDOperand = Instruction.operandAt(0); const MachineOperand *DestinationOperand = Instruction.operandAt(1); const MachineOperand *SourceOperand = Instruction.operandAt(2); const auto *Intrinsic = IntrinsicIDOperand == nullptr ? nullptr : IntrinsicIDOperand->getIf(); const Register *Destination = DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); const std::optional DestinationType = operandType(Function, Instruction, 1); const std::optional SourceType = operandType(Function, Instruction, 2); if (Intrinsic == nullptr || Intrinsic->ID != intrinsicID(GenericIntrinsic::VACopy) || Destination == nullptr || Source == nullptr || !Destination->isVirtual() || !Source->isVirtual() || !DestinationOperand->isUse() || !SourceOperand->isUse() || !DestinationType || !SourceType || *DestinationType != *SourceType || !DestinationType->isPointer() || DestinationType->addressSpace() != 0 || DestinationType->scalarBitWidth() != 64) { return std::nullopt; } return VACopyPlan{*Destination, *Source, *DestinationType}; } [[nodiscard]] Expected lowerVACopy(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target) { const auto Plan = vacopyPlan(Instruction, Function); if (!Plan) { return makeUnexpected(ErrorCode::VerificationError, "RISC-V va_copy legalization requires two p0 operands"); } auto Size = MemorySize::fixed(8); auto Align = Alignment::create(8); if (!Size || !Align) return makeUnexpected(ErrorCode::InternalError, "RISC-V pointer memory facts are invalid"); MachineMemOperandDesc LoadDescription; LoadDescription.Accesses = {MemoryAccessFlag::Load}; LoadDescription.Size = *Size; LoadDescription.MemoryType = Plan->PointerType; LoadDescription.Align = *Align; auto LoadMemory = Function.createMemOperand(std::move(LoadDescription)); if (!LoadMemory) return std::unexpected(LoadMemory.error()); MachineMemOperandDesc StoreDescription; StoreDescription.Accesses = {MemoryAccessFlag::Store}; StoreDescription.Size = *Size; StoreDescription.MemoryType = Plan->PointerType; StoreDescription.Align = *Align; auto StoreMemory = Function.createMemOperand(std::move(StoreDescription)); if (!StoreMemory) return std::unexpected(StoreMemory.error()); auto Value = Function.registerInfo().createVirtualRegister(Plan->PointerType); if (!Value) return std::unexpected(Value.error()); MachineIRBuilder Builder(Function, Target); if (auto Point = Builder.setInsertPointBefore(Instruction); !Point) return std::unexpected(Point.error()); auto Load = Builder.buildInstr(opcode(GenericOpcode::G_LOAD)); if (!Load) return std::unexpected(Load.error()); if (auto Added = Load->addDef(*Value); !Added) return std::unexpected(Added.error()); if (auto Added = Load->addUse(Plan->Source); !Added) return std::unexpected(Added.error()); if (auto Added = Load->addMemOperand(**LoadMemory); !Added) return std::unexpected(Added.error()); if (auto Built = Load->build(); !Built) return std::unexpected(Built.error()); auto Store = Builder.buildInstr(opcode(GenericOpcode::G_STORE)); if (!Store) return std::unexpected(Store.error()); if (auto Added = Store->addUse(*Value); !Added) return std::unexpected(Added.error()); if (auto Added = Store->addUse(Plan->Destination); !Added) return std::unexpected(Added.error()); if (auto Added = Store->addMemOperand(**StoreMemory); !Added) return std::unexpected(Added.error()); if (auto Built = Store->build(); !Built) return std::unexpected(Built.error()); return eraseInstruction(Instruction); } } // namespace Expected RISCVLegalizerInfo::classify(const MachineInstr &Instruction, const MachineFunction &Function, const TargetDescription &) const { if (!Subtarget.features().has(RISCVExtension::I)) { return makeUnexpected(ErrorCode::InternalError, "RISC-V legalizer requires the base I extension"); } if (!isGenericOpcode(Instruction.opcode())) return LegalizationAction::Legal; const GenericOpcode Generic = static_cast(Instruction.opcode().value()); const std::optional Type = resultType(Function, Instruction); switch (Generic) { case GenericOpcode::G_INTRINSIC_W_SIDE_EFFECTS: return vacopyPlan(Instruction, Function) ? LegalizationAction::Custom : LegalizationAction::Unsupported; case GenericOpcode::G_VAARG: return vaargPlan(Instruction, Function) ? LegalizationAction::Custom : LegalizationAction::Unsupported; case GenericOpcode::G_FRAME_INDEX: return Type && Type->isPointer() && Type->addressSpace() == 0 && Type->scalarBitWidth() == 64 ? LegalizationAction::Legal : LegalizationAction::Unsupported; case GenericOpcode::G_BR: return LegalizationAction::Legal; case GenericOpcode::G_BRCOND: { const std::optional Condition = operandType(Function, Instruction, 0); if (!Condition || !isScalarGPRType(*Condition)) return LegalizationAction::Unsupported; return canLowerNarrowBranchCondition(Instruction, Function) ? LegalizationAction::Custom : LegalizationAction::Legal; } case GenericOpcode::G_PHI: { if (!Type) return LegalizationAction::Unsupported; for (std::size_t Index = 1; Index < Instruction.operandCount(); Index += 2) { const std::optional Incoming = operandType(Function, Instruction, Index); if (!Incoming || *Incoming != *Type) return LegalizationAction::Unsupported; } if ((Type->isInteger() || Type->isAnyScalar()) && Type->scalarBitWidth() > 64) return LegalizationAction::Custom; if ((Type->isInteger() || Type->isAnyScalar()) && Type->scalarBitWidth() < 32) return LegalizationAction::Custom; if (!isSupportedIntegerType(*Type) && !isRVFloatType(*Type)) return LegalizationAction::Unsupported; return LegalizationAction::Legal; } case GenericOpcode::G_IMPLICIT_DEF: return Type && (isSupportedIntegerType(*Type) || isRVFloatType(*Type)) ? LegalizationAction::Legal : LegalizationAction::Unsupported; case GenericOpcode::G_STORE: if (Instruction.operandCount() == 0) return LegalizationAction::Unsupported; if (const std::optional ValueType = registerType(Function, *Instruction.operandAt(0)); ValueType) { const std::optional AddressType = operandType(Function, Instruction, 1); if (!AddressType || !AddressType->isPointer() || AddressType->addressSpace() != 0 || AddressType->scalarBitWidth() != 64) { return LegalizationAction::Unsupported; } if (Instruction.memOperands().size() == 1 && Instruction.memOperands().front()->isAtomic()) { const std::uint64_t Bytes = ValueType->scalarBitWidth() / 8U; return isSupportedIntegerType(*ValueType) && isSupportedAtomicMemoryAccess(Instruction, AtomicOperation::Store, Bytes) ? LegalizationAction::Legal : LegalizationAction::Unsupported; } if ((isSupportedIntegerType(*ValueType) || isRVFloatType(*ValueType)) && isSupportedNonAtomicMemoryAccess(Instruction) && memoryMatchesType(Instruction, *ValueType)) { return LegalizationAction::Legal; } } return LegalizationAction::Unsupported; case GenericOpcode::G_SEXTLOAD: case GenericOpcode::G_ZEXTLOAD: { const auto Bytes = memoryAccessBytes(Instruction); const std::optional AddressType = operandType(Function, Instruction, 1); return Type && Bytes && AddressType && isSupportedIntegerType(*Type) && AddressType->isPointer() && AddressType->addressSpace() == 0 && AddressType->scalarBitWidth() == 64 && *Bytes < ((Type->scalarBitWidth() + 7U) / 8U) && (*Bytes == 1 || *Bytes == 2 || *Bytes == 4) && isSupportedNonAtomicMemoryAccess(Instruction) ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_FPEXTLOAD: { const auto Bytes = memoryAccessBytes(Instruction); const std::optional AddressType = operandType(Function, Instruction, 1); return Type && Bytes && AddressType && Type->isFloat() && Type->scalarBitWidth() == 64 && AddressType->isPointer() && AddressType->addressSpace() == 0 && AddressType->scalarBitWidth() == 64 && *Bytes == 4 && isSupportedNonAtomicMemoryAccess(Instruction) ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_FPTRUNCSTORE: { const std::optional ValueType = operandType(Function, Instruction, 0); const std::optional AddressType = operandType(Function, Instruction, 1); const auto Bytes = memoryAccessBytes(Instruction); return ValueType && AddressType && Bytes && ValueType->isFloat() && ValueType->scalarBitWidth() == 64 && AddressType->isPointer() && AddressType->addressSpace() == 0 && AddressType->scalarBitWidth() == 64 && *Bytes == 4 && isSupportedNonAtomicMemoryAccess(Instruction) ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_LOAD: if (Type) { const std::optional AddressType = operandType(Function, Instruction, 1); if (!AddressType || !AddressType->isPointer() || AddressType->addressSpace() != 0 || AddressType->scalarBitWidth() != 64) { return LegalizationAction::Unsupported; } if (Instruction.memOperands().size() == 1 && Instruction.memOperands().front()->isAtomic()) { const std::uint64_t Bytes = Type->scalarBitWidth() / 8U; return isSupportedIntegerType(*Type) && isSupportedAtomicMemoryAccess(Instruction, AtomicOperation::Load, Bytes) ? LegalizationAction::Legal : LegalizationAction::Unsupported; } if ((isSupportedIntegerType(*Type) || isRVFloatType(*Type)) && isSupportedNonAtomicMemoryAccess(Instruction) && memoryMatchesType(Instruction, *Type)) { return LegalizationAction::Legal; } } return LegalizationAction::Unsupported; case GenericOpcode::G_FADD: case GenericOpcode::G_FSUB: case GenericOpcode::G_FMUL: case GenericOpcode::G_FDIV: { const std::optional Left = operandType(Function, Instruction, 1); const std::optional Right = operandType(Function, Instruction, 2); return Type && Left && Right && isRVFloatType(*Type) && *Left == *Type && *Right == *Type ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_FMA: { const std::optional First = operandType(Function, Instruction, 1); const std::optional Second = operandType(Function, Instruction, 2); const std::optional Third = operandType(Function, Instruction, 3); return Type && First && Second && Third && isRVFloatType(*Type) && *First == *Type && *Second == *Type && *Third == *Type ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_FNEG: case GenericOpcode::G_FABS: case GenericOpcode::G_FSQRT: { const std::optional Source = operandType(Function, Instruction, 1); return Type && Source && isRVFloatType(*Type) && *Source == *Type ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_FCOPYSIGN: { const std::optional Left = operandType(Function, Instruction, 1); const std::optional Right = operandType(Function, Instruction, 2); return Type && Left && Right && isRVFloatType(*Type) && *Left == *Type && *Right == *Type ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_FCONSTANT: { const MachineOperand *Immediate = Instruction.operandAt(1); const auto *Floating = Immediate == nullptr ? nullptr : Immediate->getIf(); const std::optional Semantics = Type ? Type->floatSemantics() : std::nullopt; return Type && isRVFloatType(*Type) && Floating != nullptr && Semantics && Floating->Bits.semantics() == *Semantics ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_ASSERT_SEXT: case GenericOpcode::G_ASSERT_ZEXT: case GenericOpcode::G_ASSERT_ALIGN: { const std::optional Source = operandType(Function, Instruction, 1); return Type && Source && *Type == *Source && isScalarGPRType(*Type) && Instruction.operandCount() == 3 ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_FREEZE: case GenericOpcode::G_CONSTANT_FOLD_BARRIER: { const std::optional Source = operandType(Function, Instruction, 1); return Type && Source && *Type == *Source && (isSupportedIntegerType(*Type) || isRVFloatType(*Type)) ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_PTRTOINT: { const std::optional Source = operandType(Function, Instruction, 1); return Type && Source && Type->isInteger() && Type->scalarBitWidth() == 64 && Source->isPointer() && Source->scalarBitWidth() == 64 ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_INTTOPTR: { const std::optional Source = operandType(Function, Instruction, 1); return Type && Source && Type->isPointer() && Type->scalarBitWidth() == 64 && Source->isInteger() && Source->scalarBitWidth() == 64 ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_BITCAST: { const std::optional Source = operandType(Function, Instruction, 1); const bool IntegerFloatPair = Type && Source && Type->scalarBitWidth() == Source->scalarBitWidth() && ((Type->isFloat() && Source->isInteger()) || (Type->isInteger() && Source->isFloat())) && (Type->scalarBitWidth() == 32 || Type->scalarBitWidth() == 64); return IntegerFloatPair ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_ANYEXT: case GenericOpcode::G_SEXT: case GenericOpcode::G_ZEXT: { const std::optional Source = operandType(Function, Instruction, 1); if (!Type || !Source) return LegalizationAction::Unsupported; if (!isScalarIntegerExtension(*Type, *Source)) { const MachineOperand *DestinationOperand = Instruction.operandAt(0); const Register *Destination = DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); MachineInstr *OnlyUse = Destination == nullptr ? nullptr : singleUseInstruction(Function, *Destination); return (Type->isInteger() || Type->isAnyScalar()) && (Source->isInteger() || Source->isAnyScalar()) && Type->scalarBitWidth() > 64 && Source->scalarBitWidth() < Type->scalarBitWidth() && OnlyUse != nullptr && OnlyUse->opcode() == opcode(GenericOpcode::G_UNMERGE_VALUES) ? LegalizationAction::Legal : LegalizationAction::Unsupported; } const MachineOperand *SourceOperand = Instruction.operandAt(1); const Register *SourceRegister = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); const MachineOperand *SourceDefinition = SourceRegister == nullptr || !SourceRegister->isVirtual() ? nullptr : Function.registerInfo().getUniqueVRegDef(*SourceRegister); const MachineInstr *SourceDef = SourceDefinition == nullptr ? nullptr : SourceDefinition->parent(); if (SourceDef != nullptr && SourceDef->opcode() == opcode(GenericOpcode::G_CONSTANT)) return LegalizationAction::Custom; if (Generic == GenericOpcode::G_ANYEXT && narrowAnyExtBinaryPlan(Instruction, Function).has_value()) return LegalizationAction::Custom; return LegalizationAction::Legal; } case GenericOpcode::G_TRUNC: { const std::optional Source = operandType(Function, Instruction, 1); if (Type && Source && (Type->isInteger() || Type->isAnyScalar()) && (Source->isInteger() || Source->isAnyScalar()) && Type->scalarBitWidth() > 64 && Source->scalarBitWidth() > Type->scalarBitWidth()) return LegalizationAction::Legal; return Type && Source && isSupportedIntegerScalar(*Type) && isSupportedIntegerScalar(*Source) && Type->scalarBitWidth() != 0 && Source->scalarBitWidth() >= Type->scalarBitWidth() ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_MERGE_VALUES: case GenericOpcode::G_UNMERGE_VALUES: { if (Instruction.operandCount() < 3) return LegalizationAction::Unsupported; const std::size_t SmallTypeIndex = Generic == GenericOpcode::G_MERGE_VALUES ? 1 : 0; const std::size_t BigTypeIndex = Generic == GenericOpcode::G_MERGE_VALUES ? 0 : Instruction.operandCount() - 1U; const std::optional SmallType = operandType(Function, Instruction, SmallTypeIndex); const std::optional BigType = operandType(Function, Instruction, BigTypeIndex); if (SmallType && BigType && Generic == GenericOpcode::G_MERGE_VALUES && BigType->scalarBitWidth() > 64 && !canFoldWideMergeImmediately(Instruction, Function)) return LegalizationAction::Legal; return SmallType && BigType && (SmallType->isInteger() || SmallType->isAnyScalar()) && (BigType->isInteger() || BigType->isAnyScalar()) && SmallType->scalarBitWidth() != 0 && BigType->scalarBitWidth() != 0 ? LegalizationAction::Custom : LegalizationAction::Unsupported; } case GenericOpcode::G_SEXT_INREG: { const std::optional Source = operandType(Function, Instruction, 1); const MachineOperand *WidthOperand = Instruction.operandAt(2); const auto Width = WidthOperand == nullptr ? Expected(std::unexpected( Error(ErrorCode::InvalidArgument, "missing in-register extension width"))) : signedImmediate(*WidthOperand); return Type && Source && *Type == *Source && isScalarGPRType(*Type) && Width && *Width > 0 && static_cast(*Width) <= Type->scalarBitWidth() ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_ICMP: { const MachineOperand *Predicate = Instruction.operandAt(1); const auto *IntegerPredicate = Predicate == nullptr ? nullptr : Predicate->getIf(); const std::optional Left = operandType(Function, Instruction, 2); const std::optional Right = operandType(Function, Instruction, 3); if (!Type || !isSupportedIntegerScalar(*Type) || IntegerPredicate == nullptr || IntegerPredicate->integerPredicate() == nullptr || !Left || !Right || *Left != *Right || !isSupportedIntegerType(*Left)) { return LegalizationAction::Unsupported; } return Type->scalarBitWidth() == 64 && Left->scalarBitWidth() == 64 ? LegalizationAction::Legal : LegalizationAction::Custom; } case GenericOpcode::G_FCMP: { const MachineOperand *Predicate = Instruction.operandAt(1); const auto *FloatingPredicate = Predicate == nullptr ? nullptr : Predicate->getIf(); const std::optional Left = operandType(Function, Instruction, 2); const std::optional Right = operandType(Function, Instruction, 3); return Type && isI1Type(*Type) && FloatingPredicate != nullptr && FloatingPredicate->floatingPredicate() != nullptr && Left && Right && *Left == *Right && isRVFloatType(*Left) ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_FPEXT: case GenericOpcode::G_FPTRUNC: { const std::optional Source = operandType(Function, Instruction, 1); const bool IsExtension = Generic == GenericOpcode::G_FPEXT; return Type && Source && isRVFloatType(*Type) && isRVFloatType(*Source) && ((IsExtension && Source->scalarBitWidth() == 32 && Type->scalarBitWidth() == 64) || (!IsExtension && Source->scalarBitWidth() == 64 && Type->scalarBitWidth() == 32)) ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_FPTOSI: case GenericOpcode::G_FPTOUI: { const std::optional Source = operandType(Function, Instruction, 1); return Type && Source && isSupportedIntegerType(*Type) && isRVFloatType(*Source) ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_SITOFP: case GenericOpcode::G_UITOFP: { const std::optional Source = operandType(Function, Instruction, 1); return Type && Source && isRVFloatType(*Type) && isSupportedIntegerType(*Source) ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_FMINNUM: case GenericOpcode::G_FMAXNUM: { const std::optional Left = operandType(Function, Instruction, 1); const std::optional Right = operandType(Function, Instruction, 2); return Type && Left && Right && isRVFloatType(*Type) && *Left == *Type && *Right == *Type ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_SELECT: { const std::optional Condition = operandType(Function, Instruction, 1); const std::optional TrueValue = operandType(Function, Instruction, 2); const std::optional FalseValue = operandType(Function, Instruction, 3); return Type && Condition && TrueValue && FalseValue && isScalarGPRType(*Condition) && *TrueValue == *Type && *FalseValue == *Type && (isSupportedIntegerType(*Type) || isRVFloatType(*Type)) ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_PTRMASK: { const std::optional Pointer = operandType(Function, Instruction, 1); const std::optional Mask = operandType(Function, Instruction, 2); return Type && Pointer && Mask && Type->isPointer() && Type->addressSpace() == 0 && Type->scalarBitWidth() == 64 && *Pointer == *Type && isSupportedIntegerScalar(*Mask) && Mask->scalarBitWidth() == 64 ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_ABS: { const std::optional Source = operandType(Function, Instruction, 1); return Type && Source && *Source == *Type && isSupportedIntegerScalar(*Type) ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_SDIVREM: case GenericOpcode::G_UDIVREM: { const std::optional Remainder = operandType(Function, Instruction, 1); const std::optional Left = operandType(Function, Instruction, 2); const std::optional Right = operandType(Function, Instruction, 3); return Type && Remainder && Left && Right && isSupportedIntegerScalar(*Type) && *Remainder == *Type && *Left == *Type && *Right == *Type ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_SMULH: case GenericOpcode::G_UMULH: { const std::optional Left = operandType(Function, Instruction, 1); const std::optional Right = operandType(Function, Instruction, 2); return Type && Left && Right && isNativeIntegerScalar(*Type) && *Left == *Type && *Right == *Type ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_SMIN: case GenericOpcode::G_SMAX: case GenericOpcode::G_UMIN: case GenericOpcode::G_UMAX: { const std::optional Left = operandType(Function, Instruction, 1); const std::optional Right = operandType(Function, Instruction, 2); return Type && Left && Right && isSupportedIntegerScalar(*Type) && *Left == *Type && *Right == *Type ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_UADDO: case GenericOpcode::G_USUBO: case GenericOpcode::G_SADDO: case GenericOpcode::G_SSUBO: case GenericOpcode::G_UMULO: case GenericOpcode::G_SMULO: { const std::optional Overflow = operandType(Function, Instruction, 1); const std::optional Left = operandType(Function, Instruction, 2); const std::optional Right = operandType(Function, Instruction, 3); return Type && Overflow && Left && Right && isNativeIntegerScalar(*Type) && isI1Type(*Overflow) && *Left == *Type && *Right == *Type ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_UADDE: case GenericOpcode::G_USUBE: case GenericOpcode::G_SADDE: case GenericOpcode::G_SSUBE: { const std::optional CarryOut = operandType(Function, Instruction, 1); const std::optional Left = operandType(Function, Instruction, 2); const std::optional Right = operandType(Function, Instruction, 3); const std::optional CarryIn = operandType(Function, Instruction, 4); return Type && CarryOut && Left && Right && CarryIn && isNativeIntegerScalar(*Type) && isI1Type(*CarryOut) && isI1Type(*CarryIn) && *Left == *Type && *Right == *Type ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_ATOMICRMW_XCHG: case GenericOpcode::G_ATOMICRMW_ADD: case GenericOpcode::G_ATOMICRMW_SUB: case GenericOpcode::G_ATOMICRMW_AND: case GenericOpcode::G_ATOMICRMW_OR: case GenericOpcode::G_ATOMICRMW_XOR: case GenericOpcode::G_ATOMICRMW_MAX: case GenericOpcode::G_ATOMICRMW_MIN: case GenericOpcode::G_ATOMICRMW_UMAX: case GenericOpcode::G_ATOMICRMW_UMIN: { const std::optional Address = operandType(Function, Instruction, 1); const std::optional Value = operandType(Function, Instruction, 2); const std::uint64_t Bytes = Type ? Type->scalarBitWidth() / 8U : 0; return Type && Address && Value && isNativeIntegerScalar(*Type) && *Value == *Type && Address->isPointer() && Address->addressSpace() == 0 && Address->scalarBitWidth() == 64 && isSupportedAtomicMemoryAccess(Instruction, AtomicOperation::ReadModifyWrite, Bytes) ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_ATOMIC_CMPXCHG: { const std::optional Address = operandType(Function, Instruction, 1); const std::optional Compare = operandType(Function, Instruction, 2); const std::optional NewValue = operandType(Function, Instruction, 3); if (!Type || !Address || !Compare || !NewValue || *Compare != *Type || *NewValue != *Type || (!Type->isInteger() && !Type->isAnyScalar()) || (Type->scalarBitWidth() != 8 && Type->scalarBitWidth() != 16 && Type->scalarBitWidth() != 32 && Type->scalarBitWidth() != 64) || !Address->isPointer() || Address->addressSpace() != 0 || Address->scalarBitWidth() != 64 || !isSupportedAtomicContainerAccess(Instruction, AtomicOperation::CompareExchange, (Type->scalarBitWidth() + 7U) / 8U)) { return LegalizationAction::Unsupported; } return Type->scalarBitWidth() == 64 ? LegalizationAction::Legal : LegalizationAction::Custom; } case GenericOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: { const std::optional Success = operandType(Function, Instruction, 1); const std::optional Address = operandType(Function, Instruction, 2); const std::optional Compare = operandType(Function, Instruction, 3); const std::optional NewValue = operandType(Function, Instruction, 4); return Type && Success && Address && Compare && NewValue && isI1Type(*Success) && *Compare == *Type && *NewValue == *Type && (Type->isInteger() || Type->isAnyScalar()) && (Type->scalarBitWidth() == 8 || Type->scalarBitWidth() == 16 || Type->scalarBitWidth() == 32 || Type->scalarBitWidth() == 64) && Address->isPointer() && Address->addressSpace() == 0 && Address->scalarBitWidth() == 64 && isSupportedAtomicContainerAccess(Instruction, AtomicOperation::CompareExchange, (Type->scalarBitWidth() + 7U) / 8U) ? LegalizationAction::Custom : LegalizationAction::Unsupported; } case GenericOpcode::G_FENCE: { const MachineOperand *OrderingOperand = Instruction.operandAt(0); const MachineOperand *ScopeOperand = Instruction.operandAt(1); const auto Ordering = OrderingOperand == nullptr ? Expected(std::unexpected( Error(ErrorCode::InvalidArgument, "missing fence ordering"))) : signedImmediate(*OrderingOperand); const auto Scope = ScopeOperand == nullptr ? Expected(std::unexpected( Error(ErrorCode::InvalidArgument, "missing fence scope"))) : signedImmediate(*ScopeOperand); return Ordering && Scope && *Ordering >= 4 && *Ordering <= 7 && *Scope == 0 ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_CONSTANT_POOL: { const MachineOperand *PoolOperand = Instruction.operandAt(1); const auto *Pool = PoolOperand == nullptr ? nullptr : PoolOperand->getIf(); return Type && Type->isPointer() && Type->addressSpace() == 0 && Type->scalarBitWidth() == 64 && Pool != nullptr && Function.constantPool().entry(Pool->Index) != nullptr ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_GLOBAL_VALUE: { const MachineOperand *GlobalOperand = Instruction.operandAt(1); const auto *Global = GlobalOperand == nullptr ? nullptr : GlobalOperand->getIf(); return Type && Type->isPointer() && Type->addressSpace() == 0 && Type->scalarBitWidth() == 64 && Global != nullptr && !Global->Name.empty() ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_BRINDIRECT: return operandType(Function, Instruction, 0) .transform([](LLT Value) { return Value.isPointer() && Value.addressSpace() == 0 && Value.scalarBitWidth() == 64; }) .value_or(false) ? LegalizationAction::Legal : LegalizationAction::Unsupported; case GenericOpcode::G_TRAP: case GenericOpcode::G_DEBUGTRAP: return Instruction.operandCount() == 0 ? LegalizationAction::Legal : LegalizationAction::Unsupported; case GenericOpcode::G_UBSANTRAP: return Instruction.operandCount() == 1 ? LegalizationAction::Legal : LegalizationAction::Unsupported; case GenericOpcode::G_CONSTANT: { const MachineOperand *Immediate = Instruction.operandAt(1); if (!Type || Immediate == nullptr || (Immediate->kind() != MachineOperand::Kind::Immediate && Immediate->kind() != MachineOperand::Kind::IntegerImmediate)) return LegalizationAction::Unsupported; if (Type->scalarBitWidth() > 64) return wideConstantUnmergePlan(Instruction, Function).has_value() ? LegalizationAction::Custom : LegalizationAction::Unsupported; if ((Type->isInteger() || Type->isAnyScalar()) && Type->scalarBitWidth() == 64) { auto Value = signedImmediate(*Immediate); if (Value && shouldUseRV64IntegerConstantPool(*Value)) return LegalizationAction::Custom; } return isSupportedIntegerType(*Type) ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_ADD: case GenericOpcode::G_SUB: case GenericOpcode::G_MUL: case GenericOpcode::G_SDIV: case GenericOpcode::G_UDIV: case GenericOpcode::G_SREM: case GenericOpcode::G_UREM: case GenericOpcode::G_AND: case GenericOpcode::G_OR: case GenericOpcode::G_XOR: case GenericOpcode::G_SHL: case GenericOpcode::G_LSHR: case GenericOpcode::G_ASHR: { const std::optional Left = operandType(Function, Instruction, 1); const std::optional Right = operandType(Function, Instruction, 2); if (!Type || !Left || !Right || *Left != *Type || *Right != *Type || (!Type->isInteger() && !Type->isAnyScalar())) return LegalizationAction::Unsupported; if (Type->scalarBitWidth() > 64) { return Generic == GenericOpcode::G_ADD || Generic == GenericOpcode::G_SUB || Generic == GenericOpcode::G_AND || Generic == GenericOpcode::G_OR || Generic == GenericOpcode::G_XOR || Generic == GenericOpcode::G_SHL ? LegalizationAction::Custom : LegalizationAction::Unsupported; } if (Generic == GenericOpcode::G_MUL && !Subtarget.features().has(RISCVExtension::M)) { if (Type->scalarBitWidth() == 64) return LegalizationAction::Libcall; const MachineOperand *DestinationOperand = Instruction.operandAt(0); const Register *Destination = DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); MachineInstr *Extension = Destination == nullptr ? nullptr : singleRegisterUse(Function, *Destination); const std::optional ExtensionType = Extension == nullptr ? std::nullopt : resultType(Function, *Extension); if (Extension != nullptr && Extension->opcode() == opcode(GenericOpcode::G_ANYEXT) && ExtensionType && ExtensionType->scalarBitWidth() == 64) { return LegalizationAction::Legal; } return LegalizationAction::Custom; } return isSupportedIntegerScalar(*Type) ? LegalizationAction::Legal : LegalizationAction::Unsupported; } case GenericOpcode::G_PTR_ADD: { const std::optional Pointer = operandType(Function, Instruction, 1); const std::optional Offset = operandType(Function, Instruction, 2); return Type && Pointer && Offset && Type->isPointer() && Type->addressSpace() == 0 && Type->scalarBitWidth() == 64 && *Pointer == *Type && isSupportedIntegerScalar(*Offset) && Offset->scalarBitWidth() == 64 ? LegalizationAction::Legal : LegalizationAction::Unsupported; } default: break; } // Keep the unsupported policy mechanically synchronized with the complete // target-independent opcode inventory. Supported cases return above. switch (Generic) { #define RCC_CORE_OPCODE(Name) #define RCC_GENERIC_OPCODE(Name) case GenericOpcode::Name: #include "rcc/CodeGen/GenericOpcode.def" #undef RCC_GENERIC_OPCODE #undef RCC_CORE_OPCODE return LegalizationAction::Unsupported; case GenericOpcode::Invalid: case GenericOpcode::CORE_PSEUDO_END: case GenericOpcode::G_GENERIC_END: default: break; } return makeUnexpected(ErrorCode::InternalError, "RISC-V legalizer received an invalid generic opcode value"); } Expected RISCVLegalizerInfo::lower(MachineInstr &Instruction, MachineFunction &Function, const TargetDescription &Target, LegalizationAction Action) const { if (Action == LegalizationAction::Custom && Instruction.opcode() == opcode(GenericOpcode::G_INTRINSIC_W_SIDE_EFFECTS)) { return lowerVACopy(Instruction, Function, Target); } if (Action == LegalizationAction::Custom && Instruction.opcode() == opcode(GenericOpcode::G_VAARG)) { return lowerVAArg(Instruction, Function, Target); } if (Instruction.opcode() == opcode(GenericOpcode::G_MUL) && !Subtarget.features().has(RISCVExtension::M)) { if (Action == LegalizationAction::Custom) return widenNarrowMultiply(Instruction, Function, Target); if (Action == LegalizationAction::Libcall) return lowerMultiplyLibcall(Instruction, Function, Target, Subtarget.features().has(RISCVExtension::D) ? CallingConvention::LP64D : CallingConvention::LP64, *Subtarget.getCallLowering()); } if (Action == LegalizationAction::Custom && resultType(Function, Instruction) .transform([](LLT Type) { return Type.scalarBitWidth() > 64; }) .value_or(false)) { const Opcode Value = Instruction.opcode(); if (Value == opcode(GenericOpcode::G_ADD) || Value == opcode(GenericOpcode::G_SUB) || Value == opcode(GenericOpcode::G_AND) || Value == opcode(GenericOpcode::G_OR) || Value == opcode(GenericOpcode::G_XOR)) { return lowerWideBinary(Instruction, Function, Target, static_cast(Value.value())); } if (Value == opcode(GenericOpcode::G_SHL)) return lowerWideShiftLeft(Instruction, Function, Target); if (Value == opcode(GenericOpcode::G_PHI)) return lowerWidePhi(Instruction, Function, Target); } if (Action == LegalizationAction::Custom && Instruction.opcode() == opcode(GenericOpcode::G_MERGE_VALUES)) return lowerMergeValues(Instruction, Function, Target); if (Action == LegalizationAction::Custom && Instruction.opcode() == opcode(GenericOpcode::G_UNMERGE_VALUES)) return lowerUnmergeValues(Instruction, Function, Target); if (Action == LegalizationAction::Custom && Instruction.opcode() == opcode(GenericOpcode::G_BRCOND)) return lowerNarrowBranchCondition(Instruction, Function, Target); if (Action == LegalizationAction::Custom && Instruction.opcode() == opcode(GenericOpcode::G_PHI)) return lowerNarrowPhi(Instruction, Function, Target); if (Action == LegalizationAction::Custom && Instruction.opcode() == opcode(GenericOpcode::G_ICMP)) return lowerIntegerCompare(Instruction, Function, Target); if (Action == LegalizationAction::Custom && Instruction.opcode() == opcode(GenericOpcode::G_ATOMIC_CMPXCHG)) return lowerAtomicCompareExchange(Instruction, Function, Target); if (Action == LegalizationAction::Custom && Instruction.opcode() == opcode(GenericOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS)) return lowerAtomicCompareExchangeWithSuccess(Instruction, Function, Target); if (Action == LegalizationAction::Custom && (Instruction.opcode() == opcode(GenericOpcode::G_ANYEXT) || Instruction.opcode() == opcode(GenericOpcode::G_SEXT) || Instruction.opcode() == opcode(GenericOpcode::G_ZEXT))) { if (Instruction.opcode() == opcode(GenericOpcode::G_ANYEXT) && narrowAnyExtBinaryPlan(Instruction, Function).has_value()) return foldNarrowAnyExtBinary(Instruction, Function, Target); return foldConstantExtension(Instruction, Function, static_cast(Instruction.opcode().value())); } if (Action == LegalizationAction::Custom && Instruction.opcode() == opcode(GenericOpcode::G_CONSTANT)) { const MachineOperand *Immediate = Instruction.operandAt(1); const std::optional Type = resultType(Function, Instruction); if (Immediate != nullptr && Type && (Type->isInteger() || Type->isAnyScalar()) && Type->scalarBitWidth() == 64) { auto Value = signedImmediate(*Immediate); if (Value && shouldUseRV64IntegerConstantPool(*Value)) return lowerConstantToPool(Instruction, Function, Target); } return foldWideConstant(Instruction, Function, Target); } return makeUnexpected(ErrorCode::Unsupported, "RISC-V legalizer has no lowering rule for this instruction"); } } // namespace rcc::riscv