diff options
Diffstat (limited to 'backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp')
| -rw-r--r-- | backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp | 3818 |
1 files changed, 3818 insertions, 0 deletions
diff --git a/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp b/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp new file mode 100644 index 0000000..e9d72fd --- /dev/null +++ b/backend/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp @@ -0,0 +1,3818 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "RISCVLegalizerInfo.h" + +#include "RISCVCallLowering.h" +#include "RISCVImmediateMaterialization.h" +#include "rcc/CodeGen/GenericOpcodes.h" +#include "rcc/CodeGen/LowLevelType.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineIRBuilder.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineMemOperand.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/Target/RISCV/RISCVSubtarget.h" +#include "rcc/Target/RISCV/RISCVTarget.h" + +#include <algorithm> +#include <array> +#include <bit> +#include <cstdint> +#include <limits> +#include <optional> +#include <ranges> +#include <string> +#include <utility> +#include <vector> + +namespace rcc::riscv { +namespace { + +[[nodiscard]] Opcode targetOpcode(InstructionOpcode Value) noexcept { + return Opcode::fromValue(static_cast<std::uint32_t>(Value)); +} + +[[nodiscard]] std::optional<LLT> registerType(const MachineFunction &Function, + const MachineOperand &Operand) { + const Register *Reg = Operand.getIf<Register>(); + if (Reg == nullptr || !Reg->isVirtual()) + return std::nullopt; + auto Info = Function.registerInfo().virtualRegisterInfo(*Reg); + if (!Info) + return std::nullopt; + return Info->get().Type; +} + +[[nodiscard]] std::optional<LLT> resultType(const MachineFunction &Function, + const MachineInstr &Instruction) { + const MachineOperand *Result = Instruction.operandAt(0); + return Result == nullptr ? std::nullopt : registerType(Function, *Result); +} + +[[nodiscard]] std::optional<LLT> operandType(const MachineFunction &Function, + const MachineInstr &Instruction, std::size_t Index) { + const MachineOperand *Operand = Instruction.operandAt(Index); + return Operand == nullptr ? std::nullopt : registerType(Function, *Operand); +} + +[[nodiscard]] bool isRVFloatType(LLT Type) noexcept { + if (!Type.isFloat()) + return false; + const std::optional<FloatSemantics> Semantics = Type.floatSemantics(); + return Semantics && + (*Semantics == FloatSemantics::IEEESingle || *Semantics == FloatSemantics::IEEEDouble); +} + +[[nodiscard]] bool isSupportedIntegerType(LLT Type) noexcept { + return ((Type.isInteger() || Type.isAnyScalar()) && Type.scalarBitWidth() != 0 && + Type.scalarBitWidth() <= 64) || + (Type.isPointer() && Type.scalarBitWidth() == 64); +} + +[[nodiscard]] bool isSupportedIntegerScalar(LLT Type) noexcept { + return (Type.isInteger() || Type.isAnyScalar()) && Type.scalarBitWidth() != 0 && + Type.scalarBitWidth() <= 64; +} + +[[nodiscard]] bool isNativeIntegerScalar(LLT Type) noexcept { + return (Type.isInteger() || Type.isAnyScalar()) && + (Type.scalarBitWidth() == 32 || Type.scalarBitWidth() == 64); +} + +[[nodiscard]] bool isScalarGPRType(LLT Type) noexcept { + return ((Type.isInteger() || Type.isAnyScalar()) && Type.scalarBitWidth() != 0 && + Type.scalarBitWidth() <= 64) || + (Type.isPointer() && Type.scalarBitWidth() == 64); +} + +[[nodiscard]] bool isI1Type(LLT Type) noexcept { + return (Type.isInteger() || Type.isAnyScalar()) && Type.scalarBitWidth() == 1; +} + +[[nodiscard]] bool isScalarIntegerExtension(LLT Destination, LLT Source) noexcept { + return (Destination.isInteger() || Destination.isAnyScalar()) && + (Source.isInteger() || Source.isAnyScalar()) && Destination.scalarBitWidth() != 0 && + Source.scalarBitWidth() != 0 && Source.scalarBitWidth() < Destination.scalarBitWidth() && + Destination.scalarBitWidth() <= 64; +} + +[[nodiscard]] Expected<IntegerImmediate> extendIntegerImmediate(const IntegerImmediate &Value, + std::uint32_t DestinationWidth, + bool SignExtend) { + if (DestinationWidth == 0 || DestinationWidth > 64) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V constant extension requires a destination no wider than 64 bits"); + + std::uint64_t Raw = Value.words().empty() ? 0 : Value.words().front(); + const std::uint32_t SourceWidth = Value.bitWidth(); + if (SignExtend && Value.isNegative() && SourceWidth < 64) + Raw |= ~std::uint64_t{0} << SourceWidth; + if (DestinationWidth < 64) + Raw &= (std::uint64_t{1} << DestinationWidth) - 1U; + return IntegerImmediate::create(DestinationWidth, std::span<const std::uint64_t>(&Raw, 1)); +} + +[[nodiscard]] Expected<void> +foldConstantExtension(MachineInstr &Instruction, MachineFunction &Function, GenericOpcode Generic) { + const MachineOperand *SourceOperand = Instruction.operandAt(1); + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>(); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); + if (Source == nullptr || Destination == nullptr || !Source->isVirtual() || + !Destination->isVirtual()) { + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V constant extension requires virtual-register operands"); + } + const Register SourceRegister = *Source; + const Register DestinationRegister = *Destination; + const MachineOperand *SourceDefinition = Function.registerInfo().getUniqueVRegDef(SourceRegister); + const MachineInstr *SourceDef = + SourceDefinition == nullptr ? nullptr : SourceDefinition->parent(); + if (SourceDef == nullptr || SourceDef->opcode() != opcode(GenericOpcode::G_CONSTANT)) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V constant extension requires a single constant definition"); + const MachineOperand *ConstantOperand = SourceDef->operandAt(1); + const auto *Integer = + ConstantOperand == nullptr ? nullptr : ConstantOperand->getIf<IntegerImmediateOperand>(); + if (Integer == nullptr) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V constant extension requires an integer constant"); + const auto DestinationInfo = Function.registerInfo().virtualRegisterInfo(*Destination); + if (!DestinationInfo || !DestinationInfo->get().Type) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V constant extension destination has no type"); + const std::uint32_t DestinationWidth = DestinationInfo->get().Type->scalarBitWidth(); + const bool SignExtend = Generic != GenericOpcode::G_ZEXT; + auto Extended = extendIntegerImmediate(Integer->Value, DestinationWidth, SignExtend); + if (!Extended) + return std::unexpected(Extended.error()); + + if (Function.registerInfo().hasOneUse(SourceRegister)) { + MachineInstr *Def = const_cast<MachineInstr *>(SourceDef); + if (auto Replaced = Def->operandAt(0)->setRegister(DestinationRegister); !Replaced) + return std::unexpected(Replaced.error()); + auto Erased = Def->eraseOperand(1); + if (!Erased) + return std::unexpected(Erased.error()); + auto Added = Def->addOperand(MachineOperand::integerImmediate(std::move(*Extended))); + if (!Added) + return std::unexpected(Added.error()); + MachineBasicBlock *Parent = Instruction.parent(); + if (Parent == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V constant extension is detached during folding"); + return Parent->removeInstruction(Instruction); + } + + auto SetOpcode = Instruction.setOpcode(opcode(GenericOpcode::G_CONSTANT)); + if (!SetOpcode) + return std::unexpected(SetOpcode.error()); + while (Instruction.operandCount() > 1) { + auto Erased = Instruction.eraseOperand(1); + if (!Erased) + return std::unexpected(Erased.error()); + } + auto Added = Instruction.addOperand(MachineOperand::integerImmediate(std::move(*Extended))); + if (!Added) + return std::unexpected(Added.error()); + + if (Function.registerInfo().useEmpty(SourceRegister)) { + MachineInstr *Def = const_cast<MachineInstr *>(SourceDef); + MachineBasicBlock *Parent = Def->parent(); + if (Parent == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V constant extension source definition is detached"); + auto Removed = Parent->removeInstruction(*Def); + if (!Removed) + return std::unexpected(Removed.error()); + // Keep the source register record as a tombstone-free historical vreg until + // the legalizer worklist transaction has completed. + } + return {}; +} + +struct WideConstantUnmergePlan final { + MachineInstr *Extension = nullptr; + MachineInstr *Unmerge = nullptr; + std::vector<Register> Outputs; + bool SignExtend = false; +}; + +[[nodiscard]] MachineInstr *singleRegisterUse(const MachineFunction &Function, Register Reg) { + MachineInstr *UseInstruction = nullptr; + auto &RegisterInfo = const_cast<MachineRegisterInfo &>(Function.registerInfo()); + for (MachineOperand &Operand : RegisterInfo.useOperands(Reg)) { + if (Operand.parent() == nullptr) + continue; + if (UseInstruction != nullptr) + return nullptr; + UseInstruction = Operand.parent(); + } + return UseInstruction; +} + +[[nodiscard]] std::optional<WideConstantUnmergePlan> +wideConstantUnmergePlan(const MachineInstr &Constant, const MachineFunction &Function) { + const MachineOperand *ResultOperand = Constant.operandAt(0); + const Register *Result = ResultOperand == nullptr ? nullptr : ResultOperand->getIf<Register>(); + if (Result == nullptr || !Result->isVirtual()) + return std::nullopt; + + MachineInstr *FirstUse = singleRegisterUse(Function, *Result); + if (FirstUse == nullptr) + return std::nullopt; + MachineInstr *Extension = nullptr; + MachineInstr *Unmerge = FirstUse; + if (FirstUse->opcode() == opcode(GenericOpcode::G_ANYEXT) || + FirstUse->opcode() == opcode(GenericOpcode::G_SEXT) || + FirstUse->opcode() == opcode(GenericOpcode::G_ZEXT)) { + Extension = FirstUse; + const MachineOperand *ExtendedOperand = Extension->operandAt(0); + const Register *Extended = + ExtendedOperand == nullptr ? nullptr : ExtendedOperand->getIf<Register>(); + if (Extended == nullptr || !Extended->isVirtual()) + return std::nullopt; + Unmerge = singleRegisterUse(Function, *Extended); + if (Unmerge == nullptr) + return std::nullopt; + } + if (Unmerge->opcode() != opcode(GenericOpcode::G_UNMERGE_VALUES) || Unmerge->operandCount() < 3) + return std::nullopt; + + WideConstantUnmergePlan Plan; + Plan.Extension = Extension; + Plan.Unmerge = Unmerge; + Plan.SignExtend = Extension != nullptr && Extension->opcode() != opcode(GenericOpcode::G_ZEXT); + for (std::size_t Index = 0; Index + 1U < Unmerge->operandCount(); ++Index) { + const MachineOperand *Operand = Unmerge->operandAt(Index); + const Register *Reg = Operand == nullptr ? nullptr : Operand->getIf<Register>(); + if (Reg == nullptr || !Reg->isVirtual() || !Operand->isDef()) + return std::nullopt; + const auto Info = Function.registerInfo().virtualRegisterInfo(*Reg); + if (!Info || !Info->get().Type || Info->get().Type->scalarBitWidth() == 0 || + Info->get().Type->scalarBitWidth() > 64 || + (!Info->get().Type->isInteger() && !Info->get().Type->isAnyScalar())) + return std::nullopt; + Plan.Outputs.push_back(*Reg); + } + const MachineOperand *SourceOperand = Unmerge->operandAt(Unmerge->operandCount() - 1U); + const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>(); + const Register *ExpectedSource = Result; + if (Extension != nullptr) { + const MachineOperand *ExtendedOperand = Extension->operandAt(0); + ExpectedSource = ExtendedOperand == nullptr ? nullptr : ExtendedOperand->getIf<Register>(); + } + if (Source == nullptr || ExpectedSource == nullptr || *Source != *ExpectedSource) + return std::nullopt; + return Plan; +} + +[[nodiscard]] Expected<IntegerImmediate> wideConstantWord(const IntegerImmediate &Value, + std::size_t WordIndex, + std::uint32_t Width, bool SignExtend) { + std::uint64_t Raw = 0; + if (WordIndex < Value.words().size()) + Raw = Value.words()[WordIndex]; + else if (SignExtend && Value.isNegative()) + Raw = std::numeric_limits<std::uint64_t>::max(); + const std::size_t WordBitOffset = WordIndex * 64U; + if (SignExtend && Value.isNegative() && WordBitOffset < Value.bitWidth()) { + const std::uint32_t Remaining = Value.bitWidth() - static_cast<std::uint32_t>(WordBitOffset); + if (Remaining < 64) + Raw |= ~std::uint64_t{0} << Remaining; + } + if (Width < 64) + Raw &= (std::uint64_t{1} << Width) - 1U; + return IntegerImmediate::create(Width, std::span<const std::uint64_t>(&Raw, 1)); +} + +[[nodiscard]] Expected<void> foldWideConstant(MachineInstr &Instruction, MachineFunction &Function, + const TargetDescription &Target) { + const auto Plan = wideConstantUnmergePlan(Instruction, Function); + if (!Plan) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V wide constant requires a G_UNMERGE_VALUES use"); + const MachineOperand *ImmediateOperand = Instruction.operandAt(1); + const auto *Integer = + ImmediateOperand == nullptr ? nullptr : ImmediateOperand->getIf<IntegerImmediateOperand>(); + if (Integer == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V wide constant has no integer immediate"); + + std::vector<IntegerImmediate> Words; + Words.reserve(Plan->Outputs.size()); + for (std::size_t Index = 0; Index < Plan->Outputs.size(); ++Index) { + const auto Info = Function.registerInfo().virtualRegisterInfo(Plan->Outputs[Index]); + if (!Info || !Info->get().Type) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V wide constant output has no virtual-register type"); + auto Word = wideConstantWord(Integer->Value, Index, Info->get().Type->scalarBitWidth(), + Plan->SignExtend); + if (!Word) + return std::unexpected(Word.error()); + Words.push_back(std::move(*Word)); + } + + std::vector<std::size_t> Representatives(Plan->Outputs.size()); + for (std::size_t Index = 0; Index < Representatives.size(); ++Index) { + Representatives[Index] = Index; + for (std::size_t Previous = 0; Previous < Index; ++Previous) { + if (Words[Previous] == Words[Index]) { + Representatives[Index] = Previous; + break; + } + } + } + + const Register FirstOutput = Plan->Outputs.front(); + auto Retargeted = Instruction.operandAt(0)->setRegister(FirstOutput); + if (!Retargeted) + return std::unexpected(Retargeted.error()); + auto SetOpcode = Instruction.setOpcode(opcode(GenericOpcode::G_CONSTANT)); + if (!SetOpcode) + return std::unexpected(SetOpcode.error()); + while (Instruction.operandCount() > 1) { + auto Erased = Instruction.eraseOperand(1); + if (!Erased) + return std::unexpected(Erased.error()); + } + auto Added = Instruction.addOperand(MachineOperand::integerImmediate(std::move(Words.front()))); + if (!Added) + return std::unexpected(Added.error()); + + MachineBasicBlock *Block = Instruction.parent(); + if (Block == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V wide constant is detached during legalization"); + if (Plan->Unmerge != &Instruction) { + auto RemovedUnmerge = Plan->Unmerge->parent()->removeInstruction(*Plan->Unmerge); + if (!RemovedUnmerge) + return std::unexpected(RemovedUnmerge.error()); + } + if (Plan->Extension != nullptr) { + auto RemovedExtension = Plan->Extension->parent()->removeInstruction(*Plan->Extension); + if (!RemovedExtension) + return std::unexpected(RemovedExtension.error()); + } + + for (std::size_t Index = 1; Index < Plan->Outputs.size(); ++Index) { + if (Representatives[Index] == Index) + continue; + auto Replaced = Function.registerInfo().replaceRegWith(Plan->Outputs[Index], + Plan->Outputs[Representatives[Index]]); + if (!Replaced) + return std::unexpected(Replaced.error()); + } + + MachineInstr *Last = &Instruction; + for (std::size_t Index = 1; Index < Plan->Outputs.size(); ++Index) { + if (Representatives[Index] != Index) + continue; + MachineIRBuilder Builder(Function, Target); + auto Point = Builder.setInsertPointAfter(*Last); + if (!Point) + return std::unexpected(Point.error()); + auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_CONSTANT)); + if (!Pending) + return std::unexpected(Pending.error()); + auto Def = Pending->addDef(Plan->Outputs[Index]); + if (!Def) + return std::unexpected(Def.error()); + auto Value = Pending->addIntegerImmediate(std::move(Words[Index])); + if (!Value) + return std::unexpected(Value.error()); + auto Built = Pending->build(); + if (!Built) + return std::unexpected(Built.error()); + Last = *Built; + } + return {}; +} + +[[nodiscard]] Expected<Register> buildIntegerConstantBefore(MachineIRBuilder &Builder, + MachineInstr &Anchor, + std::uint32_t Width, + std::uint64_t Value); + +[[nodiscard]] Expected<Register> buildBinaryBefore(MachineIRBuilder &Builder, MachineInstr &Anchor, + GenericOpcode Generic, LLT Type, Register Left, + Register Right); + +[[nodiscard]] Expected<Register> buildUnaryBefore(MachineIRBuilder &Builder, MachineInstr &Anchor, + GenericOpcode Generic, LLT Type, Register Source); + +[[nodiscard]] Expected<void> eraseInstruction(MachineInstr &Instruction); + +[[nodiscard]] Expected<void> rewriteUnary(MachineInstr &Instruction, GenericOpcode Generic, + Register Destination, Register Source); + +struct NarrowAnyExtBinaryPlan final { + MachineInstr *Binary = nullptr; + GenericOpcode Generic = GenericOpcode::Invalid; + std::array<Register, 2> WideInputs{}; + std::vector<MachineInstr *> Artifacts; + std::vector<MachineInstr *> Constants; + std::uint32_t SourceWidth = 0; +}; + +[[nodiscard]] std::optional<NarrowAnyExtBinaryPlan> +narrowAnyExtBinaryPlan(const MachineInstr &Extension, const MachineFunction &Function) { + const MachineOperand *DestinationOperand = Extension.operandAt(0); + const MachineOperand *SourceOperand = Extension.operandAt(1); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); + const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>(); + if (Destination == nullptr || Source == nullptr || !Destination->isVirtual() || + !Source->isVirtual() || singleRegisterUse(Function, *Source) != &Extension) + return std::nullopt; + const auto DestinationInfo = Function.registerInfo().virtualRegisterInfo(*Destination); + const auto SourceInfo = Function.registerInfo().virtualRegisterInfo(*Source); + if (!DestinationInfo || !SourceInfo || !DestinationInfo->get().Type || !SourceInfo->get().Type) + return std::nullopt; + const LLT DestinationType = *DestinationInfo->get().Type; + const LLT SourceType = *SourceInfo->get().Type; + if (!isScalarIntegerExtension(DestinationType, SourceType) || + DestinationType.scalarBitWidth() != 64) + return std::nullopt; + + const MachineOperand *SourceDefinition = Function.registerInfo().getUniqueVRegDef(*Source); + MachineInstr *Binary = SourceDefinition == nullptr + ? nullptr + : const_cast<MachineInstr *>(SourceDefinition->parent()); + if (Binary == nullptr || Binary->operandCount() != 3) + return std::nullopt; + const GenericOpcode Generic = static_cast<GenericOpcode>(Binary->opcode().value()); + if (Generic != GenericOpcode::G_ADD && Generic != GenericOpcode::G_SUB && + Generic != GenericOpcode::G_MUL && Generic != GenericOpcode::G_AND && + Generic != GenericOpcode::G_OR && Generic != GenericOpcode::G_XOR && + Generic != GenericOpcode::G_SHL) + return std::nullopt; + + NarrowAnyExtBinaryPlan Plan; + Plan.Binary = Binary; + Plan.Generic = Generic; + Plan.SourceWidth = SourceType.scalarBitWidth(); + const auto AddArtifact = [&Plan](MachineInstr *Artifact) { + if (std::ranges::find(Plan.Artifacts, Artifact) == Plan.Artifacts.end()) + Plan.Artifacts.push_back(Artifact); + }; + const auto TruncatedWideSource = [&](MachineInstr *Truncation, + MachineInstr *OnlyUse) -> std::optional<Register> { + if (Truncation == nullptr || Truncation->opcode() != opcode(GenericOpcode::G_TRUNC) || + Truncation->operandCount() != 2) + return std::nullopt; + const MachineOperand *NarrowOperand = Truncation->operandAt(0); + const Register *Narrow = NarrowOperand == nullptr ? nullptr : NarrowOperand->getIf<Register>(); + const MachineOperand *WideOperand = Truncation->operandAt(1); + const Register *Wide = WideOperand == nullptr ? nullptr : WideOperand->getIf<Register>(); + const std::optional<LLT> WideType = + WideOperand == nullptr ? std::nullopt : registerType(Function, *WideOperand); + if (Narrow == nullptr || Wide == nullptr || !Wide->isVirtual() || !WideType || + *WideType != DestinationType || singleRegisterUse(Function, *Narrow) != OnlyUse) + return std::nullopt; + return *Wide; + }; + for (std::size_t Index = 0; Index != Plan.WideInputs.size(); ++Index) { + const MachineOperand *Operand = Binary->operandAt(Index + 1U); + const Register *Reg = Operand == nullptr ? nullptr : Operand->getIf<Register>(); + if (Reg == nullptr || !Reg->isVirtual()) + return std::nullopt; + const auto Info = Function.registerInfo().virtualRegisterInfo(*Reg); + if (!Info || !Info->get().Type) + return std::nullopt; + if (*Info->get().Type == DestinationType) { + Plan.WideInputs[Index] = *Reg; + continue; + } + if (*Info->get().Type != SourceType) + return std::nullopt; + + const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(*Reg); + MachineInstr *Def = + Definition == nullptr ? nullptr : const_cast<MachineInstr *>(Definition->parent()); + if (auto Wide = TruncatedWideSource(Def, Binary)) { + Plan.WideInputs[Index] = *Wide; + AddArtifact(Def); + continue; + } + if (Def != nullptr && + (Def->opcode() == opcode(GenericOpcode::COPY) || + Def->opcode() == opcode(GenericOpcode::G_ANYEXT)) && + Def->operandCount() == 2 && singleRegisterUse(Function, *Reg) == Binary) { + const MachineOperand *NarrowOperand = Def->operandAt(1); + const Register *Narrow = + NarrowOperand == nullptr ? nullptr : NarrowOperand->getIf<Register>(); + const MachineOperand *NarrowDefinition = + Narrow == nullptr ? nullptr : Function.registerInfo().getUniqueVRegDef(*Narrow); + MachineInstr *Truncation = NarrowDefinition == nullptr + ? nullptr + : const_cast<MachineInstr *>(NarrowDefinition->parent()); + if (auto Wide = TruncatedWideSource(Truncation, Def)) { + Plan.WideInputs[Index] = *Wide; + AddArtifact(Def); + AddArtifact(Truncation); + continue; + } + } + if (Def != nullptr && Def->opcode() == opcode(GenericOpcode::G_CONSTANT) && + singleRegisterUse(Function, *Reg) == Binary) { + Plan.WideInputs[Index] = *Reg; + Plan.Constants.push_back(Def); + continue; + } + return std::nullopt; + } + return Plan; +} + +[[nodiscard]] Expected<void> widenConstantToRV64(MachineInstr &Instruction, + MachineFunction &Function) { + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); + const MachineOperand *ImmediateOperand = Instruction.operandAt(1); + const auto *Immediate = + ImmediateOperand == nullptr ? nullptr : ImmediateOperand->getIf<IntegerImmediateOperand>(); + if (Destination == nullptr || !Destination->isVirtual() || Immediate == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V narrow constant legalization has invalid operands"); + auto Type = LLT::makeAnyScalar(64); + if (!Type) + return std::unexpected(Type.error()); + auto SetType = Function.registerInfo().setType(*Destination, *Type); + if (!SetType) + return std::unexpected(SetType.error()); + auto Extended = extendIntegerImmediate(Immediate->Value, 64, true); + if (!Extended) + return std::unexpected(Extended.error()); + auto Erased = Instruction.eraseOperand(1); + if (!Erased) + return std::unexpected(Erased.error()); + auto Added = Instruction.addOperand(MachineOperand::integerImmediate(std::move(*Extended))); + if (!Added) + return std::unexpected(Added.error()); + return {}; +} + +[[nodiscard]] Expected<Register> widenAtomicCompareExchangeSource(MachineInstr &Instruction, + MachineFunction &Function, + MachineIRBuilder &Builder, + Register Source, LLT WideType) { + const auto SourceInfo = Function.registerInfo().virtualRegisterInfo(Source); + if (!SourceInfo || !SourceInfo->get().Type) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V compare-exchange source has no scalar type"); + if (SourceInfo->get().Type->scalarBitWidth() == WideType.scalarBitWidth()) + return Source; + + const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(Source); + MachineInstr *Def = + Definition == nullptr ? nullptr : const_cast<MachineInstr *>(Definition->parent()); + if (Def != nullptr && Def->opcode() == opcode(GenericOpcode::G_CONSTANT) && + singleRegisterUse(Function, Source) == &Instruction) { + auto Widened = widenConstantToRV64(*Def, Function); + if (!Widened) + return std::unexpected(Widened.error()); + return Source; + } + return buildUnaryBefore(Builder, Instruction, GenericOpcode::G_ANYEXT, WideType, Source); +} + +[[nodiscard]] Expected<void> lowerAtomicCompareExchange(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target) { + const std::optional<LLT> OriginalType = resultType(Function, Instruction); + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const MachineOperand *CompareOperand = Instruction.operandAt(2); + const MachineOperand *NewValueOperand = Instruction.operandAt(3); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); + const Register *Compare = CompareOperand == nullptr ? nullptr : CompareOperand->getIf<Register>(); + const Register *NewValue = + NewValueOperand == nullptr ? nullptr : NewValueOperand->getIf<Register>(); + if (!OriginalType || Destination == nullptr || Compare == nullptr || NewValue == nullptr || + OriginalType->scalarBitWidth() >= 64) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V narrow compare-exchange has invalid operands"); + const Register OriginalDestination = *Destination; + + auto WideType = LLT::makeAnyScalar(64); + if (!WideType) + return std::unexpected(WideType.error()); + MachineIRBuilder Builder(Function, Target); + auto WideCompare = + widenAtomicCompareExchangeSource(Instruction, Function, Builder, *Compare, *WideType); + if (!WideCompare) + return std::unexpected(WideCompare.error()); + auto WideNewValue = + widenAtomicCompareExchangeSource(Instruction, Function, Builder, *NewValue, *WideType); + if (!WideNewValue) + return std::unexpected(WideNewValue.error()); + + Register WideDestination; + MachineInstr *FoldedExtension = singleRegisterUse(Function, OriginalDestination); + if (FoldedExtension != nullptr && FoldedExtension->opcode() == opcode(GenericOpcode::G_ANYEXT) && + operandType(Function, *FoldedExtension, 0) + .transform([](LLT Type) { return Type.scalarBitWidth() == 64; }) + .value_or(false)) { + const MachineOperand *WideDestinationOperand = FoldedExtension->operandAt(0); + const Register *FoldedDestination = + WideDestinationOperand == nullptr ? nullptr : WideDestinationOperand->getIf<Register>(); + if (FoldedDestination == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V compare-exchange extension has no destination"); + WideDestination = *FoldedDestination; + } else { + auto Created = Function.registerInfo().createVirtualRegister(*WideType); + if (!Created) + return std::unexpected(Created.error()); + WideDestination = *Created; + FoldedExtension = nullptr; + } + + if (auto Replaced = Instruction.operandAt(0)->setRegister(WideDestination); !Replaced) + return std::unexpected(Replaced.error()); + if (auto Replaced = Instruction.operandAt(2)->setRegister(*WideCompare); !Replaced) + return std::unexpected(Replaced.error()); + if (auto Replaced = Instruction.operandAt(3)->setRegister(*WideNewValue); !Replaced) + return std::unexpected(Replaced.error()); + + if (FoldedExtension != nullptr) + return eraseInstruction(*FoldedExtension); + + auto Point = Builder.setInsertPointAfter(Instruction); + if (!Point) + return std::unexpected(Point.error()); + auto Truncation = Builder.buildInstr(opcode(GenericOpcode::G_TRUNC)); + if (!Truncation) + return std::unexpected(Truncation.error()); + if (auto Added = Truncation->addDef(OriginalDestination); !Added) + return std::unexpected(Added.error()); + if (auto Added = Truncation->addUse(WideDestination); !Added) + return std::unexpected(Added.error()); + auto Built = Truncation->build(); + if (!Built) + return std::unexpected(Built.error()); + return {}; +} + +[[nodiscard]] Expected<void> +lowerAtomicCompareExchangeWithSuccess(MachineInstr &Instruction, MachineFunction &Function, + const TargetDescription &Target) { + const MachineOperand *OldValueOperand = Instruction.operandAt(0); + const MachineOperand *SuccessOperand = Instruction.operandAt(1); + const MachineOperand *AddressOperand = Instruction.operandAt(2); + const MachineOperand *CompareOperand = Instruction.operandAt(3); + const MachineOperand *NewValueOperand = Instruction.operandAt(4); + const Register *OldValue = + OldValueOperand == nullptr ? nullptr : OldValueOperand->getIf<Register>(); + const Register *Success = SuccessOperand == nullptr ? nullptr : SuccessOperand->getIf<Register>(); + const Register *Address = AddressOperand == nullptr ? nullptr : AddressOperand->getIf<Register>(); + const Register *Compare = CompareOperand == nullptr ? nullptr : CompareOperand->getIf<Register>(); + const Register *NewValue = + NewValueOperand == nullptr ? nullptr : NewValueOperand->getIf<Register>(); + const std::optional<LLT> ValueType = resultType(Function, Instruction); + if (OldValue == nullptr || Success == nullptr || Address == nullptr || Compare == nullptr || + NewValue == nullptr || !ValueType || Instruction.memOperands().size() != 1) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V compare-exchange-with-success has invalid operands"); + + auto AtomicResult = Function.registerInfo().createVirtualRegister(*ValueType); + if (!AtomicResult) + return std::unexpected(AtomicResult.error()); + MachineIRBuilder Builder(Function, Target); + auto Point = Builder.setInsertPointBefore(Instruction); + if (!Point) + return std::unexpected(Point.error()); + auto Atomic = Builder.buildInstr(opcode(GenericOpcode::G_ATOMIC_CMPXCHG)); + if (!Atomic) + return std::unexpected(Atomic.error()); + if (auto Added = Atomic->addDef(*AtomicResult); !Added) + return std::unexpected(Added.error()); + if (auto Added = Atomic->addUse(*Address); !Added) + return std::unexpected(Added.error()); + if (auto Added = Atomic->addUse(*Compare); !Added) + return std::unexpected(Added.error()); + if (auto Added = Atomic->addUse(*NewValue); !Added) + return std::unexpected(Added.error()); + if (auto Added = Atomic->addMemOperand(*Instruction.memOperands().front()); !Added) + return std::unexpected(Added.error()); + auto BuiltAtomic = Atomic->build(); + if (!BuiltAtomic) + return std::unexpected(BuiltAtomic.error()); + + Point = Builder.setInsertPointBefore(Instruction); + if (!Point) + return std::unexpected(Point.error()); + auto CompareResult = Builder.buildInstr(opcode(GenericOpcode::G_ICMP)); + if (!CompareResult) + return std::unexpected(CompareResult.error()); + if (auto Added = CompareResult->addDef(*Success); !Added) + return std::unexpected(Added.error()); + if (auto Added = CompareResult->addIntegerPredicate(IntegerPredicate::Equal); !Added) + return std::unexpected(Added.error()); + if (auto Added = CompareResult->addUse(*AtomicResult); !Added) + return std::unexpected(Added.error()); + if (auto Added = CompareResult->addUse(*Compare); !Added) + return std::unexpected(Added.error()); + auto BuiltCompare = CompareResult->build(); + if (!BuiltCompare) + return std::unexpected(BuiltCompare.error()); + + while (!Instruction.memOperands().empty()) { + auto Erased = Instruction.eraseMemOperand(0); + if (!Erased) + return std::unexpected(Erased.error()); + } + return rewriteUnary(Instruction, GenericOpcode::COPY, *OldValue, *AtomicResult); +} + +[[nodiscard]] Expected<Register> widenIntegerCompareSource(MachineInstr &Instruction, + MachineFunction &Function, + MachineIRBuilder &Builder, + Register Source, LLT WideType) { + const auto SourceInfo = Function.registerInfo().virtualRegisterInfo(Source); + if (!SourceInfo || !SourceInfo->get().Type) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V integer compare source has no scalar type"); + const LLT SourceType = *SourceInfo->get().Type; + if (SourceType.scalarBitWidth() == WideType.scalarBitWidth()) + return Source; + + const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(Source); + MachineInstr *Def = + Definition == nullptr ? nullptr : const_cast<MachineInstr *>(Definition->parent()); + std::optional<Register> ExistingExtension; + bool AmbiguousExtension = false; + auto &RegisterInfo = Function.registerInfo(); + for (MachineOperand &Use : RegisterInfo.useOperands(Source)) { + MachineInstr *User = Use.parent(); + if (User == nullptr || User == &Instruction || User->parent() == nullptr || + (User->opcode() != opcode(GenericOpcode::G_ANYEXT) && + User->opcode() != opcode(GenericOpcode::G_SEXT))) { + continue; + } + const MachineOperand *ExtendedOperand = User->operandAt(0); + const Register *Extended = + ExtendedOperand == nullptr ? nullptr : ExtendedOperand->getIf<Register>(); + const std::optional<LLT> ExtendedType = + ExtendedOperand == nullptr ? std::nullopt : registerType(Function, *ExtendedOperand); + if (Extended == nullptr || !ExtendedType || *ExtendedType != WideType || + (Def == nullptr || (Def->opcode() != opcode(GenericOpcode::G_CONSTANT) && + User->opcode() != opcode(GenericOpcode::G_SEXT)))) { + continue; + } + if (ExistingExtension) { + AmbiguousExtension = true; + break; + } + ExistingExtension = *Extended; + } + if (ExistingExtension && !AmbiguousExtension) + return *ExistingExtension; + if (Def != nullptr && Def->opcode() == opcode(GenericOpcode::G_CONSTANT) && + singleRegisterUse(Function, Source) == &Instruction) { + auto Widened = widenConstantToRV64(*Def, Function); + if (!Widened) + return std::unexpected(Widened.error()); + return Source; + } + + if (Def != nullptr && Def->opcode() == opcode(GenericOpcode::G_TRUNC) && + Def->operandCount() == 2) { + const MachineOperand *WideOperand = Def->operandAt(1); + const Register *WideSource = WideOperand == nullptr ? nullptr : WideOperand->getIf<Register>(); + const std::optional<LLT> ExistingWideType = + WideOperand == nullptr ? std::nullopt : registerType(Function, *WideOperand); + if (WideSource != nullptr && ExistingWideType && *ExistingWideType == WideType) { + auto Result = Function.registerInfo().createVirtualRegister(WideType); + if (!Result) + return std::unexpected(Result.error()); + auto Point = Builder.setInsertPointBefore(Instruction); + if (!Point) + return std::unexpected(Point.error()); + auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_SEXT_INREG)); + if (!Pending) + return std::unexpected(Pending.error()); + if (auto Added = Pending->addDef(*Result); !Added) + return std::unexpected(Added.error()); + if (auto Added = Pending->addUse(*WideSource); !Added) + return std::unexpected(Added.error()); + if (auto Added = + Pending->addImmediate(static_cast<std::int64_t>(SourceType.scalarBitWidth())); + !Added) { + return std::unexpected(Added.error()); + } + auto Built = Pending->build(); + if (!Built) + return std::unexpected(Built.error()); + return *Result; + } + } + return buildUnaryBefore(Builder, Instruction, GenericOpcode::G_SEXT, WideType, Source); +} + +[[nodiscard]] Expected<void> lowerIntegerCompare(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target) { + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const MachineOperand *LeftOperand = Instruction.operandAt(2); + const MachineOperand *RightOperand = Instruction.operandAt(3); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); + const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf<Register>(); + const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf<Register>(); + if (Destination == nullptr || Left == nullptr || Right == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V integer compare has invalid register operands"); + const Register OriginalDestination = *Destination; + const Register OriginalLeft = *Left; + const Register OriginalRight = *Right; + + auto WideType = LLT::makeAnyScalar(64); + if (!WideType) + return std::unexpected(WideType.error()); + MachineIRBuilder Builder(Function, Target); + auto WideLeft = + widenIntegerCompareSource(Instruction, Function, Builder, OriginalLeft, *WideType); + if (!WideLeft) + return std::unexpected(WideLeft.error()); + auto WideRight = + widenIntegerCompareSource(Instruction, Function, Builder, OriginalRight, *WideType); + if (!WideRight) + return std::unexpected(WideRight.error()); + if (auto Replaced = Instruction.operandAt(2)->setRegister(*WideLeft); !Replaced) + return std::unexpected(Replaced.error()); + if (auto Replaced = Instruction.operandAt(3)->setRegister(*WideRight); !Replaced) + return std::unexpected(Replaced.error()); + + const auto DestinationInfo = Function.registerInfo().virtualRegisterInfo(OriginalDestination); + if (!DestinationInfo || !DestinationInfo->get().Type) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V integer compare destination has no scalar type"); + if (DestinationInfo->get().Type->scalarBitWidth() == 64) + return {}; + + MachineInstr *FoldedExtension = singleRegisterUse(Function, OriginalDestination); + if (FoldedExtension != nullptr && + (FoldedExtension->opcode() == opcode(GenericOpcode::G_ANYEXT) || + FoldedExtension->opcode() == opcode(GenericOpcode::G_SEXT) || + FoldedExtension->opcode() == opcode(GenericOpcode::G_ZEXT)) && + operandType(Function, *FoldedExtension, 0) + .transform([](LLT Type) { return Type.scalarBitWidth() == 64; }) + .value_or(false)) { + const MachineOperand *ExtendedOperand = FoldedExtension->operandAt(0); + const Register *Extended = + ExtendedOperand == nullptr ? nullptr : ExtendedOperand->getIf<Register>(); + if (Extended == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V integer compare extension has no destination"); + const Register WideDestination = *Extended; + if (auto Replaced = Instruction.operandAt(0)->setRegister(WideDestination); !Replaced) + return std::unexpected(Replaced.error()); + return eraseInstruction(*FoldedExtension); + } + + auto WideDestination = Function.registerInfo().createVirtualRegister(*WideType); + if (!WideDestination) + return std::unexpected(WideDestination.error()); + if (auto Replaced = Instruction.operandAt(0)->setRegister(*WideDestination); !Replaced) + return std::unexpected(Replaced.error()); + if (Function.registerInfo().useEmpty(OriginalDestination)) + return {}; + auto Point = Builder.setInsertPointAfter(Instruction); + if (!Point) + return std::unexpected(Point.error()); + auto Truncation = Builder.buildInstr(opcode(GenericOpcode::G_TRUNC)); + if (!Truncation) + return std::unexpected(Truncation.error()); + if (auto Added = Truncation->addDef(OriginalDestination); !Added) + return std::unexpected(Added.error()); + if (auto Added = Truncation->addUse(*WideDestination); !Added) + return std::unexpected(Added.error()); + auto Built = Truncation->build(); + if (!Built) + return std::unexpected(Built.error()); + return {}; +} + +[[nodiscard]] Expected<void> foldNarrowAnyExtBinary(MachineInstr &Extension, + MachineFunction &Function, + const TargetDescription &Target) { + const auto Plan = narrowAnyExtBinaryPlan(Extension, Function); + if (!Plan) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V narrow any-extension does not have a legal binary chain"); + for (MachineInstr *Constant : Plan->Constants) { + auto Widened = widenConstantToRV64(*Constant, Function); + if (!Widened) + return std::unexpected(Widened.error()); + } + for (std::size_t Index = 0; Index != Plan->WideInputs.size(); ++Index) { + auto Replaced = Plan->Binary->operandAt(Index + 1U)->setRegister(Plan->WideInputs[Index]); + if (!Replaced) + return std::unexpected(Replaced.error()); + } + for (MachineInstr *Artifact : Plan->Artifacts) { + if (Artifact->parent() == nullptr) + continue; + auto Removed = Artifact->parent()->removeInstruction(*Artifact); + if (!Removed) + return std::unexpected(Removed.error()); + } + + if (Plan->Generic == GenericOpcode::G_SHL) { + Register Amount = Plan->WideInputs[1]; + if (Plan->SourceWidth < 32 || Plan->SourceWidth > 32) { + const std::uint64_t Mask = Plan->SourceWidth == 64 + ? std::numeric_limits<std::uint64_t>::max() + : (std::uint64_t{1} << Plan->SourceWidth) - 1U; + auto WideType = LLT::makeAnyScalar(64); + if (!WideType) + return std::unexpected(WideType.error()); + MachineIRBuilder Builder(Function, Target); + auto MaskValue = buildIntegerConstantBefore(Builder, Extension, 64, Mask); + if (!MaskValue) + return std::unexpected(MaskValue.error()); + auto MaskedAmount = buildBinaryBefore(Builder, Extension, GenericOpcode::G_AND, *WideType, + Amount, *MaskValue); + if (!MaskedAmount) + return std::unexpected(MaskedAmount.error()); + Amount = *MaskedAmount; + } + auto SetOpcode = + Extension.setOpcode(Plan->SourceWidth == 32 ? targetOpcode(InstructionOpcode::G_SLLW) + : opcode(GenericOpcode::G_SHL)); + if (!SetOpcode) + return std::unexpected(SetOpcode.error()); + while (Extension.operandCount() > 1) { + auto Erased = Extension.eraseOperand(1); + if (!Erased) + return std::unexpected(Erased.error()); + } + for (Register Input : {Plan->WideInputs[0], Amount}) { + auto Operand = + MachineOperand::registerOperand(Input, RegisterOperandFlags{RegisterOperandFlag::Use}); + if (!Operand) + return std::unexpected(Operand.error()); + auto Added = Extension.addOperand(std::move(*Operand)); + if (!Added) + return std::unexpected(Added.error()); + } + MachineBasicBlock *BinaryBlock = Plan->Binary->parent(); + if (BinaryBlock == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V narrow shift is detached during legalization"); + return BinaryBlock->removeInstruction(*Plan->Binary); + } + + if (Plan->SourceWidth == 32 && + (Plan->Generic == GenericOpcode::G_ADD || Plan->Generic == GenericOpcode::G_SUB)) { + auto WideType = LLT::makeAnyScalar(64); + if (!WideType) + return std::unexpected(WideType.error()); + auto Temporary = Function.registerInfo().createVirtualRegister(*WideType); + if (!Temporary) + return std::unexpected(Temporary.error()); + auto Retargeted = Plan->Binary->operandAt(0)->setRegister(*Temporary); + if (!Retargeted) + return std::unexpected(Retargeted.error()); + auto SetOpcode = Extension.setOpcode(opcode(GenericOpcode::G_SEXT_INREG)); + if (!SetOpcode) + return std::unexpected(SetOpcode.error()); + auto ReplacedSource = Extension.operandAt(1)->setRegister(*Temporary); + if (!ReplacedSource) + return std::unexpected(ReplacedSource.error()); + auto Width = Extension.addOperand(MachineOperand::immediate(32)); + if (!Width) + return std::unexpected(Width.error()); + return {}; + } + + auto SetOpcode = Extension.setOpcode(opcode(Plan->Generic)); + if (!SetOpcode) + return std::unexpected(SetOpcode.error()); + while (Extension.operandCount() > 1) { + auto Erased = Extension.eraseOperand(1); + if (!Erased) + return std::unexpected(Erased.error()); + } + for (Register Input : Plan->WideInputs) { + auto Operand = + MachineOperand::registerOperand(Input, RegisterOperandFlags{RegisterOperandFlag::Use}); + if (!Operand) + return std::unexpected(Operand.error()); + auto Added = Extension.addOperand(std::move(*Operand)); + if (!Added) + return std::unexpected(Added.error()); + } + MachineBasicBlock *BinaryBlock = Plan->Binary->parent(); + if (BinaryBlock == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V narrow binary instruction is detached during legalization"); + auto Removed = BinaryBlock->removeInstruction(*Plan->Binary); + if (!Removed) + return std::unexpected(Removed.error()); + (void)Target; + return {}; +} + +[[nodiscard]] Expected<void> lowerConstantToPool(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target) { + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); + const MachineOperand *ImmediateOperand = Instruction.operandAt(1); + const auto *Immediate = + ImmediateOperand == nullptr ? nullptr : ImmediateOperand->getIf<IntegerImmediateOperand>(); + if (Destination == nullptr || !Destination->isVirtual() || Immediate == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V constant-pool legalization has invalid operands"); + const auto DestinationInfo = Function.registerInfo().virtualRegisterInfo(*Destination); + if (!DestinationInfo || !DestinationInfo->get().Type || + DestinationInfo->get().Type->scalarBitWidth() != 64) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V constant-pool legalization requires a 64-bit destination"); + const LLT DestinationType = *DestinationInfo->get().Type; + auto Value = GenericConstantData::integer(Immediate->Value); + if (!Value) + return std::unexpected(Value.error()); + auto Align = Alignment::create(8); + if (!Align) + return std::unexpected(Align.error()); + MachineConstantPoolEntryDesc Description; + Description.Value = std::move(*Value); + Description.Size = 8; + Description.Align = *Align; + auto PoolIndex = Function.constantPool().addEntry(std::move(Description)); + if (!PoolIndex) + return std::unexpected(PoolIndex.error()); + + auto PointerType = LLT::makePointer(0, 64); + if (!PointerType) + return std::unexpected(PointerType.error()); + auto Address = Function.registerInfo().createVirtualRegister(*PointerType); + if (!Address) + return std::unexpected(Address.error()); + MachineIRBuilder Builder(Function, Target); + auto Point = Builder.setInsertPointBefore(Instruction); + if (!Point) + return std::unexpected(Point.error()); + auto AddressInstruction = Builder.buildInstr(opcode(GenericOpcode::G_CONSTANT_POOL)); + if (!AddressInstruction) + return std::unexpected(AddressInstruction.error()); + auto AddressDef = AddressInstruction->addDef(*Address); + if (!AddressDef) + return std::unexpected(AddressDef.error()); + auto AddressPool = AddressInstruction->addConstantPool(*PoolIndex); + if (!AddressPool) + return std::unexpected(AddressPool.error()); + auto BuiltAddress = AddressInstruction->build(); + if (!BuiltAddress) + return std::unexpected(BuiltAddress.error()); + + MachineMemOperandDesc MemoryDescription; + MemoryDescription.Accesses = {MemoryAccessFlag::Load}; + MemoryDescription.Size = *MemorySize::fixed(8); + MemoryDescription.MemoryType = DestinationType; + MemoryDescription.Align = *Align; + MemoryDescription.Base = MemoryBase::constantPoolPseudo(); + auto Memory = Function.createMemOperand(std::move(MemoryDescription)); + if (!Memory) + return std::unexpected(Memory.error()); + auto SetOpcode = Instruction.setOpcode(opcode(GenericOpcode::G_LOAD)); + if (!SetOpcode) + return std::unexpected(SetOpcode.error()); + while (Instruction.operandCount() > 1) { + auto Erased = Instruction.eraseOperand(1); + if (!Erased) + return std::unexpected(Erased.error()); + } + auto AddressUse = + MachineOperand::registerOperand(*Address, RegisterOperandFlags{RegisterOperandFlag::Use}); + if (!AddressUse) + return std::unexpected(AddressUse.error()); + auto AddedAddress = Instruction.addOperand(std::move(*AddressUse)); + if (!AddedAddress) + return std::unexpected(AddedAddress.error()); + auto AddedMemory = Instruction.addMemOperand(**Memory); + if (!AddedMemory) + return std::unexpected(AddedMemory.error()); + return {}; +} + +[[nodiscard]] bool isSupportedNonAtomicMemoryAccess(const MachineInstr &Instruction) noexcept { + if (Instruction.memOperands().size() != 1) + return false; + const MachineMemOperand &Memory = *Instruction.memOperands().front(); + const MachineMemOperandDesc &Description = Memory.description(); + return !Memory.isAtomic() && Description.AddressSpace == 0 && Description.Size.isKnown() && + !Description.Size.isScalable() && Description.Size.minimumBytes() != 0; +} + +[[nodiscard]] bool isSupportedAtomicMemoryAccess(const MachineInstr &Instruction, + AtomicOperation Operation, + std::uint64_t Bytes) noexcept { + if (Instruction.memOperands().size() != 1 || (Bytes != 4 && Bytes != 8)) + return false; + const MachineMemOperand &Memory = *Instruction.memOperands().front(); + const MachineMemOperandDesc &Description = Memory.description(); + return Description.AtomicKind == Operation && Description.AddressSpace == 0 && + Description.Size.isKnown() && !Description.Size.isScalable() && + Description.Size.minimumBytes() == Bytes && Description.Align.bytes() >= Bytes && + Description.SyncScope.value() == 0; +} + +[[nodiscard]] bool isSupportedAtomicContainerAccess(const MachineInstr &Instruction, + AtomicOperation Operation, + std::uint64_t ContainerBytes) noexcept { + if (Instruction.memOperands().size() != 1 || ContainerBytes == 0) + return false; + const MachineMemOperandDesc &Description = Instruction.memOperands().front()->description(); + if (Description.AtomicKind != Operation || Description.AddressSpace != 0 || + !Description.Size.isKnown() || Description.Size.isScalable() || + Description.SyncScope.value() != 0) + return false; + const std::uint64_t MemoryBytes = Description.Size.minimumBytes(); + return (MemoryBytes == 1 || MemoryBytes == 2 || MemoryBytes == 4 || MemoryBytes == 8) && + MemoryBytes <= ContainerBytes && Description.Align.bytes() >= MemoryBytes; +} + +[[nodiscard]] std::optional<std::uint64_t> +memoryAccessBytes(const MachineInstr &Instruction) noexcept { + if (Instruction.memOperands().size() != 1) + return std::nullopt; + const MachineMemOperandDesc &Description = Instruction.memOperands().front()->description(); + if (!Description.Size.isKnown() || Description.Size.isScalable()) + return std::nullopt; + return Description.Size.minimumBytes(); +} + +[[nodiscard]] bool memoryMatchesType(const MachineInstr &Instruction, LLT Type) noexcept { + const auto Bytes = memoryAccessBytes(Instruction); + if (!Bytes || (!isSupportedIntegerType(Type) && !isRVFloatType(Type))) + return false; + if (isRVFloatType(Type)) + return Type.scalarBitWidth() == 32 ? *Bytes == 4 : *Bytes == 8; + const std::uint64_t ValueBytes = (Type.scalarBitWidth() + 7U) / 8U; + return (*Bytes == 1 || *Bytes == 2 || *Bytes == 4 || *Bytes == 8) && *Bytes <= ValueBytes; +} + +[[nodiscard]] Expected<std::int64_t> signedImmediate(const MachineOperand &Operand) { + if (const std::int64_t *Value = Operand.getIf<std::int64_t>()) + return *Value; + const auto *Integer = Operand.getIf<IntegerImmediateOperand>(); + if (Integer == nullptr || Integer->Value.bitWidth() == 0 || Integer->Value.bitWidth() > 64) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V constant selection requires an immediate no wider than 64 bits"); + std::uint64_t Raw = Integer->Value.words().empty() ? 0 : Integer->Value.words().front(); + const std::uint32_t Width = Integer->Value.bitWidth(); + if (Width < 64 && (Raw & (std::uint64_t{1} << (Width - 1U))) != 0) + Raw |= ~std::uint64_t{0} << Width; + return std::bit_cast<std::int64_t>(Raw); +} + +[[nodiscard]] Expected<void> replaceRegisterUses(MachineFunction &Function, Register OldReg, + Register NewReg) { + std::vector<MachineOperand *> Uses; + for (MachineOperand &Operand : Function.registerInfo().useOperands(OldReg)) + Uses.push_back(&Operand); + for (MachineOperand *Operand : Uses) { + auto Replaced = Operand->setRegister(NewReg); + if (!Replaced) + return std::unexpected(Replaced.error()); + } + return {}; +} + +[[nodiscard]] Expected<void> eraseInstruction(MachineInstr &Instruction) { + MachineBasicBlock *Block = Instruction.parent(); + if (Block == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V legalizer cannot erase a detached instruction"); + return Block->removeInstruction(Instruction); +} + +[[nodiscard]] bool allDefsUnused(const MachineInstr &Instruction, const MachineFunction &Function) { + for (const MachineOperand &Operand : Instruction.operands()) { + const Register *Reg = Operand.getIf<Register>(); + if (Reg != nullptr && Operand.isDef() && Reg->isVirtual() && + !Function.registerInfo().useEmpty(*Reg)) + return false; + } + return true; +} + +void eraseDeadArtifacts(std::span<MachineInstr *const> Candidates, + const MachineFunction &Function) { + bool Changed = false; + do { + Changed = false; + for (auto It = Candidates.rbegin(); It != Candidates.rend(); ++It) { + MachineInstr *Instruction = *It; + if (Instruction == nullptr || Instruction->parent() == nullptr || + !allDefsUnused(*Instruction, Function)) + continue; + const Opcode Value = Instruction->opcode(); + if (Value != opcode(GenericOpcode::G_TRUNC) && Value != opcode(GenericOpcode::G_ANYEXT) && + Value != opcode(GenericOpcode::G_SEXT) && Value != opcode(GenericOpcode::G_ZEXT) && + Value != opcode(GenericOpcode::G_MERGE_VALUES) && + Value != opcode(GenericOpcode::G_UNMERGE_VALUES) && Value != opcode(GenericOpcode::COPY)) + continue; + if (eraseInstruction(*Instruction)) + Changed = true; + } + } while (Changed); +} + +[[nodiscard]] Expected<Register> buildIntegerConstantBefore(MachineIRBuilder &Builder, + MachineInstr &Anchor, + std::uint32_t Width, + std::uint64_t Value) { + auto Type = LLT::makeAnyScalar(Width); + if (!Type) + return std::unexpected(Type.error()); + auto Result = Builder.function().registerInfo().createVirtualRegister(*Type); + if (!Result) + return std::unexpected(Result.error()); + auto Immediate = IntegerImmediate::fromUnsigned(Width, Value); + if (!Immediate) + return std::unexpected(Immediate.error()); + auto Point = Builder.setInsertPointBefore(Anchor); + if (!Point) + return std::unexpected(Point.error()); + auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_CONSTANT)); + if (!Pending) + return std::unexpected(Pending.error()); + if (auto Added = Pending->addDef(*Result); !Added) + return std::unexpected(Added.error()); + if (auto Added = Pending->addIntegerImmediate(std::move(*Immediate)); !Added) + return std::unexpected(Added.error()); + auto Built = Pending->build(); + if (!Built) + return std::unexpected(Built.error()); + return *Result; +} + +[[nodiscard]] Expected<Register> buildUnaryBefore(MachineIRBuilder &Builder, MachineInstr &Anchor, + GenericOpcode Generic, LLT Type, + Register Source) { + auto Result = Builder.function().registerInfo().createVirtualRegister(Type); + if (!Result) + return std::unexpected(Result.error()); + auto Point = Builder.setInsertPointBefore(Anchor); + if (!Point) + return std::unexpected(Point.error()); + auto Pending = Builder.buildInstr(opcode(Generic)); + if (!Pending) + return std::unexpected(Pending.error()); + if (auto Added = Pending->addDef(*Result); !Added) + return std::unexpected(Added.error()); + if (auto Added = Pending->addUse(Source); !Added) + return std::unexpected(Added.error()); + auto Built = Pending->build(); + if (!Built) + return std::unexpected(Built.error()); + return *Result; +} + +[[nodiscard]] Expected<Register> buildBinaryBefore(MachineIRBuilder &Builder, MachineInstr &Anchor, + GenericOpcode Generic, LLT Type, Register Left, + Register Right) { + auto Result = Builder.function().registerInfo().createVirtualRegister(Type); + if (!Result) + return std::unexpected(Result.error()); + auto Point = Builder.setInsertPointBefore(Anchor); + if (!Point) + return std::unexpected(Point.error()); + auto Pending = Builder.buildInstr(opcode(Generic)); + if (!Pending) + return std::unexpected(Pending.error()); + if (auto Added = Pending->addDef(*Result); !Added) + return std::unexpected(Added.error()); + if (auto Added = Pending->addUse(Left); !Added) + return std::unexpected(Added.error()); + if (auto Added = Pending->addUse(Right); !Added) + return std::unexpected(Added.error()); + auto Built = Pending->build(); + if (!Built) + return std::unexpected(Built.error()); + return *Result; +} + +[[nodiscard]] Expected<void> rewriteUnary(MachineInstr &Instruction, GenericOpcode Generic, + Register Destination, Register Source) { + auto SetOpcode = Instruction.setOpcode(opcode(Generic)); + if (!SetOpcode) + return std::unexpected(SetOpcode.error()); + while (Instruction.operandCount() != 0) { + auto Erased = Instruction.eraseOperand(Instruction.operandCount() - 1U); + if (!Erased) + return std::unexpected(Erased.error()); + } + auto Def = MachineOperand::registerOperand(Destination, + RegisterOperandFlags{RegisterOperandFlag::Define}); + if (!Def) + return std::unexpected(Def.error()); + if (auto Added = Instruction.addOperand(std::move(*Def)); !Added) + return std::unexpected(Added.error()); + auto Use = + MachineOperand::registerOperand(Source, RegisterOperandFlags{RegisterOperandFlag::Use}); + if (!Use) + return std::unexpected(Use.error()); + return Instruction.addOperand(std::move(*Use)); +} + +[[nodiscard]] Expected<void> widenNarrowMultiply(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target) { + const std::optional<LLT> Type = resultType(Function, Instruction); + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const MachineOperand *LeftOperand = Instruction.operandAt(1); + const MachineOperand *RightOperand = Instruction.operandAt(2); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); + const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf<Register>(); + const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf<Register>(); + if (!Type || Destination == nullptr || Left == nullptr || Right == nullptr || + Type->scalarBitWidth() >= 64) { + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V narrow multiply has invalid operands"); + } + + MachineInstr *Extension = singleRegisterUse(Function, *Destination); + if (Extension != nullptr && Extension->opcode() == opcode(GenericOpcode::G_ANYEXT)) { + const auto Plan = narrowAnyExtBinaryPlan(*Extension, Function); + if (Plan && Plan->Binary == &Instruction) + return foldNarrowAnyExtBinary(*Extension, Function, Target); + } + + auto WideType = LLT::makeAnyScalar(64); + if (!WideType) + return std::unexpected(WideType.error()); + MachineIRBuilder Builder(Function, Target); + auto WideLeft = buildUnaryBefore(Builder, Instruction, GenericOpcode::G_ANYEXT, *WideType, *Left); + if (!WideLeft) + return std::unexpected(WideLeft.error()); + auto WideRight = + buildUnaryBefore(Builder, Instruction, GenericOpcode::G_ANYEXT, *WideType, *Right); + if (!WideRight) + return std::unexpected(WideRight.error()); + auto Product = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_MUL, *WideType, *WideLeft, + *WideRight); + if (!Product) + return std::unexpected(Product.error()); + return rewriteUnary(Instruction, GenericOpcode::G_TRUNC, *Destination, *Product); +} + +[[nodiscard]] Expected<void> lowerMultiplyLibcall(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target, + CallingConvention Convention, + const TargetCallLowering &CallLowering) { + const std::optional<LLT> Type = resultType(Function, Instruction); + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const MachineOperand *LeftOperand = Instruction.operandAt(1); + const MachineOperand *RightOperand = Instruction.operandAt(2); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); + const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf<Register>(); + const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf<Register>(); + if (!Type || Destination == nullptr || Left == nullptr || Right == nullptr || + Type->scalarBitWidth() != 64) { + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V multiply libcall requires three s64 registers"); + } + auto ABIAlignment = Alignment::create(8); + auto Callee = MachineCallTarget::externalSymbol("__muldi3"); + if (!ABIAlignment || !Callee) + return makeUnexpected(ErrorCode::InternalError, + "RISC-V multiply libcall ABI facts are unavailable"); + const auto Value = [Type = *Type, Alignment = *ABIAlignment](Register Reg, std::uint32_t Number) { + return ABIValue{ + .Value = Reg, + .Type = Type, + .OriginalValue = Number, + .FragmentIndex = 0, + .FragmentCount = 1, + .FragmentOffsetBytes = 0, + .OriginalAlignment = Alignment, + .Extension = ABIExtensionKind::None, + .Flags = {}, + .ByValSizeBytes = 0, + .ByValAlignment = Alignment, + }; + }; + + CallLoweringRequest Request; + Request.Convention = callingConventionID(Convention); + Request.Callee = std::move(*Callee); + Request.Arguments = {Value(*Left, 0), Value(*Right, 1)}; + Request.Results = {Value(*Destination, 0)}; + Request.FixedArgumentCount = Request.Arguments.size(); + + MachineIRBuilder Builder(Function, Target); + auto Point = Builder.setInsertPointBefore(Instruction); + if (!Point) + return std::unexpected(Point.error()); + auto Lowered = CallLowering.lowerCall(Builder, Request); + if (!Lowered) + return std::unexpected(Lowered.error()); + return eraseInstruction(Instruction); +} + +[[nodiscard]] Expected<void> rewriteBinary(MachineInstr &Instruction, GenericOpcode Generic, + Register Destination, Register Left, Register Right) { + auto Rewritten = rewriteUnary(Instruction, Generic, Destination, Left); + if (!Rewritten) + return std::unexpected(Rewritten.error()); + auto Use = MachineOperand::registerOperand(Right, RegisterOperandFlags{RegisterOperandFlag::Use}); + if (!Use) + return std::unexpected(Use.error()); + return Instruction.addOperand(std::move(*Use)); +} + +[[nodiscard]] bool isKnownZeroExtended(const MachineFunction &Function, Register Reg, + std::uint32_t ActiveBits, unsigned Depth = 0) { + if (Depth > 8 || !Reg.isVirtual()) + return false; + const auto Info = Function.registerInfo().virtualRegisterInfo(Reg); + if (!Info || !Info->get().Type) + return false; + const std::uint32_t Width = Info->get().Type->scalarBitWidth(); + if (Width <= ActiveBits) + return true; + const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(Reg); + const MachineInstr *Def = Definition == nullptr ? nullptr : Definition->parent(); + if (Def == nullptr) + return false; + if (Def->opcode() == opcode(GenericOpcode::G_ASSERT_ZEXT)) { + const MachineOperand *BitsOperand = Def->operandAt(2); + auto Bits = BitsOperand == nullptr + ? Expected<std::int64_t>(std::unexpected( + Error(ErrorCode::InvalidArgument, "missing asserted extension width"))) + : signedImmediate(*BitsOperand); + return Bits && *Bits >= 0 && static_cast<std::uint64_t>(*Bits) <= ActiveBits; + } + if (Def->opcode() == opcode(GenericOpcode::G_ZEXT)) { + const MachineOperand *SourceOperand = Def->operandAt(1); + const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>(); + if (Source == nullptr) + return false; + const auto SourceInfo = Function.registerInfo().virtualRegisterInfo(*Source); + return SourceInfo && SourceInfo->get().Type && + SourceInfo->get().Type->scalarBitWidth() <= ActiveBits; + } + if (Def->opcode() == opcode(GenericOpcode::G_TRUNC)) { + const MachineOperand *SourceOperand = Def->operandAt(1); + const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>(); + return Source != nullptr && isKnownZeroExtended(Function, *Source, ActiveBits, Depth + 1U); + } + return false; +} + +[[nodiscard]] Expected<Register> zeroExtendToRV64(MachineIRBuilder &Builder, MachineInstr &Anchor, + MachineFunction &Function, Register Source, + std::vector<MachineInstr *> &DeadCandidates) { + const auto Info = Function.registerInfo().virtualRegisterInfo(Source); + if (!Info || !Info->get().Type) + return makeUnexpected(ErrorCode::VerificationError, "RISC-V merge source has no scalar type"); + const LLT SourceType = *Info->get().Type; + if (SourceType.scalarBitWidth() == 64) + return Source; + const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(Source); + MachineInstr *Def = + Definition == nullptr ? nullptr : const_cast<MachineInstr *>(Definition->parent()); + if (Def != nullptr && Def->opcode() == opcode(GenericOpcode::G_TRUNC)) { + const MachineOperand *WideOperand = Def->operandAt(1); + const Register *Wide = WideOperand == nullptr ? nullptr : WideOperand->getIf<Register>(); + if (Wide != nullptr) { + const auto WideInfo = Function.registerInfo().virtualRegisterInfo(*Wide); + if (WideInfo && WideInfo->get().Type && WideInfo->get().Type->scalarBitWidth() == 64 && + isKnownZeroExtended(Function, *Wide, SourceType.scalarBitWidth())) { + DeadCandidates.push_back(Def); + return *Wide; + } + } + } + auto WideType = LLT::makeAnyScalar(64); + if (!WideType) + return std::unexpected(WideType.error()); + return buildUnaryBefore(Builder, Anchor, GenericOpcode::G_ZEXT, *WideType, Source); +} + +[[nodiscard]] MachineInstr *singleUseInstruction(const MachineFunction &Function, Register Reg) { + MachineInstr *Result = nullptr; + auto &RegisterInfo = const_cast<MachineRegisterInfo &>(Function.registerInfo()); + for (MachineOperand &Operand : RegisterInfo.useOperands(Reg)) { + MachineInstr *Instruction = Operand.parent(); + if (Instruction == nullptr || Instruction->parent() == nullptr) + continue; + if (Result != nullptr && Result != Instruction) + return nullptr; + Result = Instruction; + } + return Result; +} + +[[nodiscard]] Expected<void> lowerWideMergeValues(MachineInstr &Instruction, + MachineFunction &Function) { + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); + const MachineOperand *LowOperand = Instruction.operandAt(1); + const Register *Low = LowOperand == nullptr ? nullptr : LowOperand->getIf<Register>(); + if (Destination == nullptr || Low == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V wide merge has invalid register operands"); + + std::vector<MachineInstr *> Truncations; + auto &RegisterInfo = Function.registerInfo(); + std::vector<MachineOperand *> Uses; + for (MachineOperand &Use : RegisterInfo.useOperands(*Destination)) + Uses.push_back(&Use); + for (MachineOperand *Use : Uses) { + MachineInstr *Trunc = Use->parent(); + if (Trunc == nullptr || Trunc->parent() == nullptr || + Trunc->opcode() != opcode(GenericOpcode::G_TRUNC) || Trunc->operandAt(1) != Use) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V wide merge is only representable through low-part truncation"); + const MachineOperand *TruncResultOperand = Trunc->operandAt(0); + const Register *TruncResult = + TruncResultOperand == nullptr ? nullptr : TruncResultOperand->getIf<Register>(); + const auto LowInfo = RegisterInfo.virtualRegisterInfo(*Low); + const auto ResultInfo = + TruncResult == nullptr + ? Expected<std::reference_wrapper<const VirtualRegisterInfo>>( + std::unexpected(Error(ErrorCode::InvalidArgument, "missing truncation result"))) + : RegisterInfo.virtualRegisterInfo(*TruncResult); + if (!LowInfo || !LowInfo->get().Type || !ResultInfo || !ResultInfo->get().Type || + ResultInfo->get().Type->scalarBitWidth() > LowInfo->get().Type->scalarBitWidth()) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V wide merge truncation spans more than its low source"); + if (*ResultInfo->get().Type == *LowInfo->get().Type) { + auto Replaced = replaceRegisterUses(Function, *TruncResult, *Low); + if (!Replaced) + return std::unexpected(Replaced.error()); + Truncations.push_back(Trunc); + } else { + auto Replaced = Use->setRegister(*Low); + if (!Replaced) + return std::unexpected(Replaced.error()); + } + } + for (MachineInstr *Trunc : Truncations) { + auto Erased = eraseInstruction(*Trunc); + if (!Erased) + return std::unexpected(Erased.error()); + } + return eraseInstruction(Instruction); +} + +[[nodiscard]] Expected<void> lowerMergeValues(MachineInstr &Instruction, MachineFunction &Function, + const TargetDescription &Target) { + const std::optional<LLT> DestinationType = resultType(Function, Instruction); + if (!DestinationType || DestinationType->scalarBitWidth() == 0) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V merge destination has no scalar type"); + if (DestinationType->scalarBitWidth() > 64) + return lowerWideMergeValues(Instruction, Function); + + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); + if (Destination == nullptr || Instruction.operandCount() < 3) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V merge has invalid register operands"); + + Register Result = *Destination; + MachineInstr *FoldedExtension = nullptr; + if (DestinationType->scalarBitWidth() < 64) { + MachineInstr *Use = singleUseInstruction(Function, *Destination); + if (Use != nullptr && Use->opcode() == opcode(GenericOpcode::G_ZEXT) && + operandType(Function, *Use, 0) + .transform([](LLT Type) { return Type.scalarBitWidth() == 64; }) + .value_or(false)) { + const MachineOperand *ExtendedOperand = Use->operandAt(0); + const Register *Extended = + ExtendedOperand == nullptr ? nullptr : ExtendedOperand->getIf<Register>(); + if (Extended != nullptr) { + Result = *Extended; + FoldedExtension = Use; + } + } + } + + auto WideType = LLT::makeAnyScalar(64); + if (!WideType) + return std::unexpected(WideType.error()); + MachineIRBuilder Builder(Function, Target); + std::vector<MachineInstr *> DeadCandidates; + std::vector<Register> Terms; + Terms.reserve(Instruction.operandCount() - 1U); + for (std::size_t Index = 1; Index < Instruction.operandCount(); ++Index) { + const MachineOperand *Operand = Instruction.operandAt(Index); + const Register *Source = Operand == nullptr ? nullptr : Operand->getIf<Register>(); + if (Source == nullptr) + return makeUnexpected(ErrorCode::VerificationError, "RISC-V merge source is not a register"); + auto WideSource = zeroExtendToRV64(Builder, Instruction, Function, *Source, DeadCandidates); + if (!WideSource) + return std::unexpected(WideSource.error()); + if (Index == 1) { + Terms.push_back(*WideSource); + continue; + } + const std::optional<LLT> SourceType = registerType(Function, *Operand); + if (!SourceType) + return makeUnexpected(ErrorCode::VerificationError, "RISC-V merge source has no scalar type"); + const std::uint64_t Shift = + static_cast<std::uint64_t>(SourceType->scalarBitWidth()) * (Index - 1U); + auto Amount = buildIntegerConstantBefore(Builder, Instruction, 64, Shift); + if (!Amount) + return std::unexpected(Amount.error()); + auto Shifted = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_SHL, *WideType, + *WideSource, *Amount); + if (!Shifted) + return std::unexpected(Shifted.error()); + Terms.push_back(*Shifted); + } + + Register Accumulator = Terms.front(); + for (std::size_t Index = 1; Index + 1U < Terms.size(); ++Index) { + auto Combined = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_OR, *WideType, + Accumulator, Terms[Index]); + if (!Combined) + return std::unexpected(Combined.error()); + Accumulator = *Combined; + } + if (DestinationType->scalarBitWidth() == 64 || FoldedExtension != nullptr) { + auto Rewritten = + rewriteBinary(Instruction, GenericOpcode::G_OR, Result, Accumulator, Terms.back()); + if (!Rewritten) + return std::unexpected(Rewritten.error()); + } else { + auto Combined = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_OR, *WideType, + Accumulator, Terms.back()); + if (!Combined) + return std::unexpected(Combined.error()); + auto Rewritten = rewriteUnary(Instruction, GenericOpcode::G_TRUNC, *Destination, *Combined); + if (!Rewritten) + return std::unexpected(Rewritten.error()); + } + if (FoldedExtension != nullptr) { + auto Erased = eraseInstruction(*FoldedExtension); + if (!Erased) + return std::unexpected(Erased.error()); + } + eraseDeadArtifacts(DeadCandidates, Function); + return {}; +} + +[[nodiscard]] Expected<void> foldWideExtendedUnmerge(MachineInstr &Instruction, + MachineFunction &Function) { + const std::size_t OutputCount = Instruction.operandCount() - 1U; + const MachineOperand *SourceOperand = Instruction.operandAt(OutputCount); + const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>(); + const MachineOperand *Definition = + Source == nullptr ? nullptr : Function.registerInfo().getUniqueVRegDef(*Source); + MachineInstr *Extension = + Definition == nullptr ? nullptr : const_cast<MachineInstr *>(Definition->parent()); + if (Extension == nullptr || (Extension->opcode() != opcode(GenericOpcode::G_ZEXT) && + Extension->opcode() != opcode(GenericOpcode::G_ANYEXT) && + Extension->opcode() != opcode(GenericOpcode::G_SEXT))) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V wide unmerge requires an extension-defined source"); + const MachineOperand *NarrowOperand = Extension->operandAt(1); + const Register *Narrow = NarrowOperand == nullptr ? nullptr : NarrowOperand->getIf<Register>(); + if (Narrow == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V wide unmerge extension has no register source"); + const auto NarrowInfo = Function.registerInfo().virtualRegisterInfo(*Narrow); + if (!NarrowInfo || !NarrowInfo->get().Type) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V wide unmerge extension source has no type"); + const std::uint32_t NarrowWidth = NarrowInfo->get().Type->scalarBitWidth(); + + std::vector<MachineInstr *> DeadUsers; + for (std::size_t Index = 0; Index < OutputCount; ++Index) { + const MachineOperand *OutputOperand = Instruction.operandAt(Index); + const Register *Output = OutputOperand == nullptr ? nullptr : OutputOperand->getIf<Register>(); + if (Output == nullptr || Function.registerInfo().useEmpty(*Output)) + continue; + const auto OutputInfo = Function.registerInfo().virtualRegisterInfo(*Output); + if (!OutputInfo || !OutputInfo->get().Type) + return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide unmerge output has no type"); + const std::uint64_t Offset = + static_cast<std::uint64_t>(Index) * OutputInfo->get().Type->scalarBitWidth(); + if (Offset != 0) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V wide unmerge cannot materialize a used high wide part"); + if (OutputInfo->get().Type->scalarBitWidth() == NarrowWidth) { + auto Replaced = replaceRegisterUses(Function, *Output, *Narrow); + if (!Replaced) + return std::unexpected(Replaced.error()); + continue; + } + if (OutputInfo->get().Type->scalarBitWidth() < NarrowWidth) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V wide unmerge output unexpectedly narrows its source"); + std::vector<MachineOperand *> Uses; + for (MachineOperand &Use : Function.registerInfo().useOperands(*Output)) + Uses.push_back(&Use); + for (MachineOperand *Use : Uses) { + MachineInstr *Trunc = Use->parent(); + if (Trunc == nullptr || Trunc->parent() == nullptr || + Trunc->opcode() != opcode(GenericOpcode::G_TRUNC) || Trunc->operandAt(1) != Use) + return makeUnexpected( + ErrorCode::Unsupported, + "RISC-V wide unmerge result is only representable through truncation"); + const MachineOperand *ResultOperand = Trunc->operandAt(0); + const Register *Result = + ResultOperand == nullptr ? nullptr : ResultOperand->getIf<Register>(); + const auto ResultInfo = + Result == nullptr + ? Expected<std::reference_wrapper<const VirtualRegisterInfo>>( + std::unexpected(Error(ErrorCode::InvalidArgument, "missing truncation result"))) + : Function.registerInfo().virtualRegisterInfo(*Result); + if (!ResultInfo || !ResultInfo->get().Type || + ResultInfo->get().Type->scalarBitWidth() > NarrowWidth) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V wide unmerge truncation exceeds its source width"); + if (ResultInfo->get().Type->scalarBitWidth() == NarrowWidth) { + auto Replaced = replaceRegisterUses(Function, *Result, *Narrow); + if (!Replaced) + return std::unexpected(Replaced.error()); + DeadUsers.push_back(Trunc); + } else { + auto Replaced = Use->setRegister(*Narrow); + if (!Replaced) + return std::unexpected(Replaced.error()); + } + } + } + for (MachineInstr *User : DeadUsers) { + auto Erased = eraseInstruction(*User); + if (!Erased) + return std::unexpected(Erased.error()); + } + auto Erased = eraseInstruction(Instruction); + if (!Erased) + return std::unexpected(Erased.error()); + if (Function.registerInfo().useEmpty(*Source)) + return eraseInstruction(*Extension); + return {}; +} + +[[nodiscard]] Expected<void> lowerUnmergeValues(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target) { + if (Instruction.operandCount() < 3) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V unmerge has invalid register operands"); + const std::size_t OutputCount = Instruction.operandCount() - 1U; + const std::optional<LLT> OutputType = operandType(Function, Instruction, 0); + const std::optional<LLT> SourceType = operandType(Function, Instruction, OutputCount); + if (!OutputType || !SourceType) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V unmerge operands have no scalar types"); + if (SourceType->scalarBitWidth() > 64 || OutputType->scalarBitWidth() > 64) + return foldWideExtendedUnmerge(Instruction, Function); + + const MachineOperand *SourceOperand = Instruction.operandAt(OutputCount); + const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>(); + if (Source == nullptr) + return makeUnexpected(ErrorCode::VerificationError, "RISC-V unmerge source is not a register"); + auto WideType = LLT::makeAnyScalar(64); + if (!WideType) + return std::unexpected(WideType.error()); + MachineIRBuilder Builder(Function, Target); + std::vector<MachineInstr *> FoldedExtensions; + for (std::size_t Index = 0; Index < OutputCount; ++Index) { + const MachineOperand *OutputOperand = Instruction.operandAt(Index); + const Register *Output = OutputOperand == nullptr ? nullptr : OutputOperand->getIf<Register>(); + if (Output == nullptr || Function.registerInfo().useEmpty(*Output)) + continue; + Register Extracted = *Source; + const std::uint64_t Offset = static_cast<std::uint64_t>(Index) * OutputType->scalarBitWidth(); + if (Offset != 0) { + auto Amount = buildIntegerConstantBefore(Builder, Instruction, 64, Offset); + if (!Amount) + return std::unexpected(Amount.error()); + auto Shifted = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_LSHR, *WideType, + *Source, *Amount); + if (!Shifted) + return std::unexpected(Shifted.error()); + Extracted = *Shifted; + } + + MachineInstr *Extension = singleUseInstruction(Function, *Output); + const bool FoldZExt = Extension != nullptr && + Extension->opcode() == opcode(GenericOpcode::G_ZEXT) && + operandType(Function, *Extension, 0) + .transform([](LLT Type) { return Type.scalarBitWidth() == 64; }) + .value_or(false); + if (FoldZExt) { + const MachineOperand *ExtendedOperand = Extension->operandAt(0); + const Register *Extended = + ExtendedOperand == nullptr ? nullptr : ExtendedOperand->getIf<Register>(); + if (Extended == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V unmerge extension has no destination"); + const std::uint64_t Mask = OutputType->scalarBitWidth() == 64 + ? std::numeric_limits<std::uint64_t>::max() + : (std::uint64_t{1} << OutputType->scalarBitWidth()) - 1U; + auto MaskReg = buildIntegerConstantBefore(Builder, Instruction, 64, Mask); + if (!MaskReg) + return std::unexpected(MaskReg.error()); + auto Masked = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_AND, *WideType, + Extracted, *MaskReg); + if (!Masked) + return std::unexpected(Masked.error()); + auto Replaced = replaceRegisterUses(Function, *Extended, *Masked); + if (!Replaced) + return std::unexpected(Replaced.error()); + FoldedExtensions.push_back(Extension); + continue; + } + auto Truncated = + buildUnaryBefore(Builder, Instruction, GenericOpcode::G_TRUNC, *OutputType, Extracted); + if (!Truncated) + return std::unexpected(Truncated.error()); + auto Replaced = replaceRegisterUses(Function, *Output, *Truncated); + if (!Replaced) + return std::unexpected(Replaced.error()); + } + for (MachineInstr *Extension : FoldedExtensions) { + auto Erased = eraseInstruction(*Extension); + if (!Erased) + return std::unexpected(Erased.error()); + } + return eraseInstruction(Instruction); +} + +struct WideValueParts final { + std::uint32_t BitWidth = 0; + std::vector<Register> Parts; + std::vector<MachineInstr *> Artifacts; +}; + +[[nodiscard]] Expected<WideValueParts> +collectWideValueParts(Register Value, MachineFunction &Function, unsigned Depth = 0) { + if (Depth > 16) + return makeUnexpected(ErrorCode::Unsupported, "RISC-V wide-value artifact chain is too deep"); + const auto Info = Function.registerInfo().virtualRegisterInfo(Value); + if (!Info || !Info->get().Type) + return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide value has no scalar type"); + const LLT Type = *Info->get().Type; + if ((!Type.isInteger() && !Type.isAnyScalar()) || Type.scalarBitWidth() == 0) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V wide-value decomposition requires an integer scalar"); + if (Type.scalarBitWidth() <= 64) + return WideValueParts{.BitWidth = Type.scalarBitWidth(), .Parts = {Value}, .Artifacts = {}}; + + const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(Value); + MachineInstr *Def = + Definition == nullptr ? nullptr : const_cast<MachineInstr *>(Definition->parent()); + if (Def == nullptr) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V wide value has no decomposable definition"); + if (Def->opcode() == opcode(GenericOpcode::G_MERGE_VALUES)) { + WideValueParts Result; + Result.BitWidth = Type.scalarBitWidth(); + for (std::size_t Index = 1; Index < Def->operandCount(); ++Index) { + const MachineOperand *Operand = Def->operandAt(Index); + const Register *Part = Operand == nullptr ? nullptr : Operand->getIf<Register>(); + const std::optional<LLT> PartType = + Operand == nullptr ? std::nullopt : registerType(Function, *Operand); + if (Part == nullptr || !PartType || PartType->scalarBitWidth() != 64) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V wide merge must already use 64-bit limbs"); + Result.Parts.push_back(*Part); + } + Result.Artifacts.push_back(Def); + return Result; + } + if (Def->opcode() == opcode(GenericOpcode::G_TRUNC)) { + const MachineOperand *SourceOperand = Def->operandAt(1); + const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>(); + if (Source == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V wide truncation has no source register"); + auto Result = collectWideValueParts(*Source, Function, Depth + 1U); + if (!Result) + return std::unexpected(Result.error()); + Result->BitWidth = Type.scalarBitWidth(); + const std::size_t RequiredParts = (Type.scalarBitWidth() + 63U) / 64U; + if (Result->Parts.size() < RequiredParts) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V wide truncation source has too few limbs"); + Result->Parts.resize(RequiredParts); + Result->Artifacts.push_back(Def); + return Result; + } + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V wide value is not defined by merge/trunc artifacts"); +} + +struct WideResultPlan final { + std::vector<Register> Outputs; + MachineInstr *Extension = nullptr; + MachineInstr *Unmerge = nullptr; +}; + +[[nodiscard]] Expected<WideResultPlan> wideResultPlan(Register Result, MachineFunction &Function, + std::size_t RequiredParts) { + MachineInstr *Use = singleUseInstruction(Function, Result); + WideResultPlan Plan; + if (Use != nullptr && (Use->opcode() == opcode(GenericOpcode::G_ANYEXT) || + Use->opcode() == opcode(GenericOpcode::G_SEXT) || + Use->opcode() == opcode(GenericOpcode::G_ZEXT))) { + Plan.Extension = Use; + const MachineOperand *ExtendedOperand = Use->operandAt(0); + const Register *Extended = + ExtendedOperand == nullptr ? nullptr : ExtendedOperand->getIf<Register>(); + Use = Extended == nullptr ? nullptr : singleUseInstruction(Function, *Extended); + } + if (Use == nullptr || Use->opcode() != opcode(GenericOpcode::G_UNMERGE_VALUES)) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V wide result must feed an unmerge artifact"); + Plan.Unmerge = Use; + const std::size_t OutputCount = Use->operandCount() - 1U; + if (OutputCount < RequiredParts) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V wide result unmerge has too few outputs"); + Plan.Outputs.reserve(RequiredParts); + for (std::size_t Index = 0; Index < RequiredParts; ++Index) { + const MachineOperand *Operand = Use->operandAt(Index); + const Register *Output = Operand == nullptr ? nullptr : Operand->getIf<Register>(); + const std::optional<LLT> Type = + Operand == nullptr ? std::nullopt : registerType(Function, *Operand); + if (Output == nullptr || !Type || Type->scalarBitWidth() != 64) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V wide result must unmerge to 64-bit limbs"); + Plan.Outputs.push_back(*Output); + } + return Plan; +} + +[[nodiscard]] Expected<Register> buildIntegerCompareBefore(MachineIRBuilder &Builder, + MachineInstr &Anchor, + IntegerPredicate Predicate, + Register Left, Register Right) { + auto Type = LLT::makeAnyScalar(64); + if (!Type) + return std::unexpected(Type.error()); + auto Result = Builder.function().registerInfo().createVirtualRegister(*Type); + if (!Result) + return std::unexpected(Result.error()); + auto Point = Builder.setInsertPointBefore(Anchor); + if (!Point) + return std::unexpected(Point.error()); + auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_ICMP)); + if (!Pending) + return std::unexpected(Pending.error()); + if (auto Added = Pending->addDef(*Result); !Added) + return std::unexpected(Added.error()); + if (auto Added = Pending->addIntegerPredicate(Predicate); !Added) + return std::unexpected(Added.error()); + if (auto Added = Pending->addUse(Left); !Added) + return std::unexpected(Added.error()); + if (auto Added = Pending->addUse(Right); !Added) + return std::unexpected(Added.error()); + auto Built = Pending->build(); + if (!Built) + return std::unexpected(Built.error()); + return *Result; +} + +[[nodiscard]] Expected<Register> buildSelectBefore(MachineIRBuilder &Builder, MachineInstr &Anchor, + LLT Type, Register Condition, Register TrueValue, + Register FalseValue) { + auto Result = Builder.function().registerInfo().createVirtualRegister(Type); + if (!Result) + return std::unexpected(Result.error()); + auto Point = Builder.setInsertPointBefore(Anchor); + if (!Point) + return std::unexpected(Point.error()); + auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_SELECT)); + if (!Pending) + return std::unexpected(Pending.error()); + if (auto Added = Pending->addDef(*Result); !Added) + return std::unexpected(Added.error()); + if (auto Added = Pending->addUse(Condition); !Added) + return std::unexpected(Added.error()); + if (auto Added = Pending->addUse(TrueValue); !Added) + return std::unexpected(Added.error()); + if (auto Added = Pending->addUse(FalseValue); !Added) + return std::unexpected(Added.error()); + auto Built = Pending->build(); + if (!Built) + return std::unexpected(Built.error()); + return *Result; +} + +struct TwoLimbShiftResult final { + Register Low; + Register High; +}; + +[[nodiscard]] Expected<TwoLimbShiftResult> +emitTwoLimbLeftShiftBody(MachineIRBuilder &Builder, MachineInstr &Anchor, Register Low, + Register High, Register Amount, Register AmountExcess, Register AmountLack, + Register Zero, Register IsShort, Register IsZero) { + auto Type = LLT::makeAnyScalar(64); + if (!Type) + return std::unexpected(Type.error()); + auto LowShort = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SHL, *Type, Low, Amount); + if (!LowShort) + return std::unexpected(LowShort.error()); + auto CrossLow = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_LSHR, *Type, Low, AmountLack); + if (!CrossLow) + return std::unexpected(CrossLow.error()); + auto HighShort = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SHL, *Type, High, Amount); + if (!HighShort) + return std::unexpected(HighShort.error()); + auto ShortHigh = + buildBinaryBefore(Builder, Anchor, GenericOpcode::G_OR, *Type, *CrossLow, *HighShort); + if (!ShortHigh) + return std::unexpected(ShortHigh.error()); + auto LongHigh = + buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SHL, *Type, Low, AmountExcess); + if (!LongHigh) + return std::unexpected(LongHigh.error()); + auto ResultLow = buildSelectBefore(Builder, Anchor, *Type, IsShort, *LowShort, Zero); + if (!ResultLow) + return std::unexpected(ResultLow.error()); + auto ShiftedHigh = buildSelectBefore(Builder, Anchor, *Type, IsShort, *ShortHigh, *LongHigh); + if (!ShiftedHigh) + return std::unexpected(ShiftedHigh.error()); + auto ResultHigh = buildSelectBefore(Builder, Anchor, *Type, IsZero, High, *ShiftedHigh); + if (!ResultHigh) + return std::unexpected(ResultHigh.error()); + return TwoLimbShiftResult{.Low = *ResultLow, .High = *ResultHigh}; +} + +[[nodiscard]] Expected<TwoLimbShiftResult> emitTwoLimbLeftShift(MachineIRBuilder &Builder, + MachineInstr &Anchor, Register Low, + Register High, Register Amount) { + auto Type = LLT::makeAnyScalar(64); + if (!Type) + return std::unexpected(Type.error()); + auto Boundary = buildIntegerConstantBefore(Builder, Anchor, 64, 64); + if (!Boundary) + return std::unexpected(Boundary.error()); + auto AmountExcess = + buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, Amount, *Boundary); + if (!AmountExcess) + return std::unexpected(AmountExcess.error()); + auto AmountLack = + buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, *Boundary, Amount); + if (!AmountLack) + return std::unexpected(AmountLack.error()); + auto Zero = buildIntegerConstantBefore(Builder, Anchor, 64, 0); + if (!Zero) + return std::unexpected(Zero.error()); + auto IsShort = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan, + Amount, *Boundary); + if (!IsShort) + return std::unexpected(IsShort.error()); + auto IsZero = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::Equal, Amount, *Zero); + if (!IsZero) + return std::unexpected(IsZero.error()); + return emitTwoLimbLeftShiftBody(Builder, Anchor, Low, High, Amount, *AmountExcess, *AmountLack, + *Zero, *IsShort, *IsZero); +} + +[[nodiscard]] Expected<Register> +emitLowLimbOfTwoLimbLogicalRightShift(MachineIRBuilder &Builder, MachineInstr &Anchor, Register Low, + Register High, Register Amount, Register Boundary, + Register Zero) { + auto Type = LLT::makeAnyScalar(64); + if (!Type) + return std::unexpected(Type.error()); + auto AmountExcess = + buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, Amount, Boundary); + if (!AmountExcess) + return std::unexpected(AmountExcess.error()); + auto AmountLack = + buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, Boundary, Amount); + if (!AmountLack) + return std::unexpected(AmountLack.error()); + auto IsShort = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan, + Amount, Boundary); + if (!IsShort) + return std::unexpected(IsShort.error()); + auto IsZero = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::Equal, Amount, Zero); + if (!IsZero) + return std::unexpected(IsZero.error()); + auto LowShort = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_LSHR, *Type, Low, Amount); + if (!LowShort) + return std::unexpected(LowShort.error()); + auto HighShort = + buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SHL, *Type, High, *AmountLack); + if (!HighShort) + return std::unexpected(HighShort.error()); + auto ShortValue = + buildBinaryBefore(Builder, Anchor, GenericOpcode::G_OR, *Type, *LowShort, *HighShort); + if (!ShortValue) + return std::unexpected(ShortValue.error()); + auto LongValue = + buildBinaryBefore(Builder, Anchor, GenericOpcode::G_LSHR, *Type, High, *AmountExcess); + if (!LongValue) + return std::unexpected(LongValue.error()); + auto Shifted = buildSelectBefore(Builder, Anchor, *Type, *IsShort, *ShortValue, *LongValue); + if (!Shifted) + return std::unexpected(Shifted.error()); + return buildSelectBefore(Builder, Anchor, *Type, *IsZero, Low, *Shifted); +} + +[[nodiscard]] Expected<std::vector<Register>> +emitThreeLimbLeftShift(MachineIRBuilder &Builder, MachineInstr &Anchor, + std::span<const Register> Parts, Register Amount) { + if (Parts.size() != 3) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V three-limb shift requires exactly three limbs"); + auto Type = LLT::makeAnyScalar(64); + if (!Type) + return std::unexpected(Type.error()); + + auto OuterBoundary = buildIntegerConstantBefore(Builder, Anchor, 64, 128); + if (!OuterBoundary) + return std::unexpected(OuterBoundary.error()); + auto OuterExcess = + buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, Amount, *OuterBoundary); + if (!OuterExcess) + return std::unexpected(OuterExcess.error()); + auto OuterLack = + buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, *OuterBoundary, Amount); + if (!OuterLack) + return std::unexpected(OuterLack.error()); + auto Zero = buildIntegerConstantBefore(Builder, Anchor, 64, 0); + if (!Zero) + return std::unexpected(Zero.error()); + auto OuterIsShort = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan, + Amount, *OuterBoundary); + if (!OuterIsShort) + return std::unexpected(OuterIsShort.error()); + auto OuterIsZero = + buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::Equal, Amount, *Zero); + if (!OuterIsZero) + return std::unexpected(OuterIsZero.error()); + + auto InnerBoundary = buildIntegerConstantBefore(Builder, Anchor, 64, 64); + if (!InnerBoundary) + return std::unexpected(InnerBoundary.error()); + auto InnerExcess = + buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, Amount, *InnerBoundary); + if (!InnerExcess) + return std::unexpected(InnerExcess.error()); + auto InnerLack = + buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, *InnerBoundary, Amount); + if (!InnerLack) + return std::unexpected(InnerLack.error()); + auto InnerIsShort = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan, + Amount, *InnerBoundary); + if (!InnerIsShort) + return std::unexpected(InnerIsShort.error()); + auto InnerIsZero = + buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::Equal, Amount, *Zero); + if (!InnerIsZero) + return std::unexpected(InnerIsZero.error()); + auto Lower = emitTwoLimbLeftShiftBody(Builder, Anchor, Parts[0], Parts[1], Amount, *InnerExcess, + *InnerLack, *Zero, *InnerIsShort, *InnerIsZero); + if (!Lower) + return std::unexpected(Lower.error()); + + auto Cross = emitLowLimbOfTwoLimbLogicalRightShift(Builder, Anchor, Parts[0], Parts[1], + *OuterLack, *InnerBoundary, *Zero); + if (!Cross) + return std::unexpected(Cross.error()); + auto HighIsShort = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan, + Amount, *InnerBoundary); + if (!HighIsShort) + return std::unexpected(HighIsShort.error()); + auto HighShort = + buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SHL, *Type, Parts[2], Amount); + if (!HighShort) + return std::unexpected(HighShort.error()); + auto BoundedHigh = buildSelectBefore(Builder, Anchor, *Type, *HighIsShort, *HighShort, *Zero); + if (!BoundedHigh) + return std::unexpected(BoundedHigh.error()); + auto ShortHigh = + buildBinaryBefore(Builder, Anchor, GenericOpcode::G_OR, *Type, *Cross, *BoundedHigh); + if (!ShortHigh) + return std::unexpected(ShortHigh.error()); + + auto LongIsShort = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan, + *OuterExcess, *InnerBoundary); + if (!LongIsShort) + return std::unexpected(LongIsShort.error()); + auto LongShift = + buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SHL, *Type, Parts[0], *OuterExcess); + if (!LongShift) + return std::unexpected(LongShift.error()); + auto LongHigh = buildSelectBefore(Builder, Anchor, *Type, *LongIsShort, *LongShift, *Zero); + if (!LongHigh) + return std::unexpected(LongHigh.error()); + + auto ResultLow = buildSelectBefore(Builder, Anchor, *Type, *OuterIsShort, Lower->Low, *Zero); + if (!ResultLow) + return std::unexpected(ResultLow.error()); + auto ResultMiddle = buildSelectBefore(Builder, Anchor, *Type, *OuterIsShort, Lower->High, *Zero); + if (!ResultMiddle) + return std::unexpected(ResultMiddle.error()); + auto ShiftedHigh = + buildSelectBefore(Builder, Anchor, *Type, *OuterIsShort, *ShortHigh, *LongHigh); + if (!ShiftedHigh) + return std::unexpected(ShiftedHigh.error()); + auto ResultHigh = buildSelectBefore(Builder, Anchor, *Type, *OuterIsZero, Parts[2], *ShiftedHigh); + if (!ResultHigh) + return std::unexpected(ResultHigh.error()); + return std::vector<Register>{*ResultLow, *ResultMiddle, *ResultHigh}; +} + +[[nodiscard]] Expected<void> lowerWideShiftLeft(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target) { + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const MachineOperand *ValueOperand = Instruction.operandAt(1); + const MachineOperand *AmountOperand = Instruction.operandAt(2); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); + const Register *Value = ValueOperand == nullptr ? nullptr : ValueOperand->getIf<Register>(); + const Register *Amount = AmountOperand == nullptr ? nullptr : AmountOperand->getIf<Register>(); + if (Destination == nullptr || Value == nullptr || Amount == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V wide shift has invalid register operands"); + auto ValueParts = collectWideValueParts(*Value, Function); + if (!ValueParts) + return std::unexpected(ValueParts.error()); + auto AmountParts = collectWideValueParts(*Amount, Function); + if (!AmountParts) + return std::unexpected(AmountParts.error()); + if (ValueParts->Parts.size() < 2 || AmountParts->Parts.empty()) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V wide shift has an invalid limb decomposition"); + auto ResultPlan = wideResultPlan(*Destination, Function, ValueParts->Parts.size()); + if (!ResultPlan) + return std::unexpected(ResultPlan.error()); + + MachineIRBuilder Builder(Function, Target); + Expected<std::vector<Register>> Results = + ValueParts->Parts.size() == 2 + ? emitTwoLimbLeftShift(Builder, Instruction, ValueParts->Parts[0], ValueParts->Parts[1], + AmountParts->Parts[0]) + .transform([](TwoLimbShiftResult Result) { + return std::vector<Register>{Result.Low, Result.High}; + }) + : ValueParts->Parts.size() == 3 + ? emitThreeLimbLeftShift(Builder, Instruction, ValueParts->Parts, AmountParts->Parts[0]) + : Expected<std::vector<Register>>(std::unexpected( + Error(ErrorCode::Unsupported, + "RISC-V wide shift currently requires two or three 64-bit limbs"))); + if (!Results) + return std::unexpected(Results.error()); + for (std::size_t Index = 0; Index < Results->size(); ++Index) { + auto Replaced = replaceRegisterUses(Function, ResultPlan->Outputs[Index], (*Results)[Index]); + if (!Replaced) + return std::unexpected(Replaced.error()); + } + if (auto Erased = eraseInstruction(*ResultPlan->Unmerge); !Erased) + return std::unexpected(Erased.error()); + if (ResultPlan->Extension != nullptr) { + if (auto Erased = eraseInstruction(*ResultPlan->Extension); !Erased) + return std::unexpected(Erased.error()); + } + if (auto Erased = eraseInstruction(Instruction); !Erased) + return std::unexpected(Erased.error()); + for (std::size_t Index = 1; Index < AmountParts->Parts.size(); ++Index) { + const MachineOperand *Definition = + Function.registerInfo().getUniqueVRegDef(AmountParts->Parts[Index]); + if (Definition != nullptr && Definition->parent() != nullptr) + AmountParts->Artifacts.push_back(const_cast<MachineInstr *>(Definition->parent())); + } + ValueParts->Artifacts.insert(ValueParts->Artifacts.end(), AmountParts->Artifacts.begin(), + AmountParts->Artifacts.end()); + eraseDeadArtifacts(ValueParts->Artifacts, Function); + return {}; +} + +[[nodiscard]] Expected<std::vector<Register>> emitWideAdd(MachineIRBuilder &Builder, + MachineInstr &Anchor, + std::span<const Register> Left, + std::span<const Register> Right) { + if (Left.size() != Right.size() || Left.size() < 2) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V wide add requires matching multi-limb operands"); + auto Type = LLT::makeAnyScalar(64); + if (!Type) + return std::unexpected(Type.error()); + std::vector<Register> Results; + Results.reserve(Left.size()); + auto Low = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_ADD, *Type, Left[0], Right[0]); + if (!Low) + return std::unexpected(Low.error()); + auto Carry = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan, *Low, + Right[0]); + if (!Carry) + return std::unexpected(Carry.error()); + auto LowCopy = buildUnaryBefore(Builder, Anchor, GenericOpcode::COPY, *Type, *Low); + if (!LowCopy) + return std::unexpected(LowCopy.error()); + Results.push_back(*LowCopy); + for (std::size_t Index = 1; Index < Left.size(); ++Index) { + auto Base = + buildBinaryBefore(Builder, Anchor, GenericOpcode::G_ADD, *Type, Left[Index], Right[Index]); + if (!Base) + return std::unexpected(Base.error()); + std::optional<Register> BaseCarry; + if (Index + 1U < Left.size()) { + auto Compared = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan, + *Base, Left[Index]); + if (!Compared) + return std::unexpected(Compared.error()); + BaseCarry = *Compared; + } + auto Sum = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_ADD, *Type, *Base, *Carry); + if (!Sum) + return std::unexpected(Sum.error()); + if (Index + 1U < Left.size()) { + auto Zero = buildIntegerConstantBefore(Builder, Anchor, 64, 0); + if (!Zero) + return std::unexpected(Zero.error()); + auto IsZero = + buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::Equal, *Sum, *Zero); + if (!IsZero) + return std::unexpected(IsZero.error()); + auto CarryFromIncoming = + buildBinaryBefore(Builder, Anchor, GenericOpcode::G_AND, *Type, *IsZero, *Carry); + if (!CarryFromIncoming) + return std::unexpected(CarryFromIncoming.error()); + auto NextCarry = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_OR, *Type, *BaseCarry, + *CarryFromIncoming); + if (!NextCarry) + return std::unexpected(NextCarry.error()); + Carry = *NextCarry; + } + auto Copy = buildUnaryBefore(Builder, Anchor, GenericOpcode::COPY, *Type, *Sum); + if (!Copy) + return std::unexpected(Copy.error()); + Results.push_back(*Copy); + } + return Results; +} + +[[nodiscard]] Expected<std::vector<Register>> emitWideSub(MachineIRBuilder &Builder, + MachineInstr &Anchor, + std::span<const Register> Left, + std::span<const Register> Right) { + if (Left.size() != Right.size() || Left.size() < 2) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V wide sub requires matching multi-limb operands"); + auto Type = LLT::makeAnyScalar(64); + if (!Type) + return std::unexpected(Type.error()); + std::vector<Register> Results; + Results.reserve(Left.size()); + auto Low = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, Left[0], Right[0]); + if (!Low) + return std::unexpected(Low.error()); + Results.push_back(*Low); + auto Borrow = buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::UnsignedLessThan, + Left[0], Right[0]); + if (!Borrow) + return std::unexpected(Borrow.error()); + for (std::size_t Index = 1; Index < Left.size(); ++Index) { + auto Base = + buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, Left[Index], Right[Index]); + if (!Base) + return std::unexpected(Base.error()); + std::optional<Register> BaseBorrow; + if (Index + 1U < Left.size()) { + auto Compared = buildIntegerCompareBefore( + Builder, Anchor, IntegerPredicate::UnsignedGreaterThan, *Base, Left[Index]); + if (!Compared) + return std::unexpected(Compared.error()); + BaseBorrow = *Compared; + } + auto Difference = + buildBinaryBefore(Builder, Anchor, GenericOpcode::G_SUB, *Type, *Base, *Borrow); + if (!Difference) + return std::unexpected(Difference.error()); + Results.push_back(*Difference); + if (Index + 1U < Left.size()) { + auto Zero = buildIntegerConstantBefore(Builder, Anchor, 64, 0); + if (!Zero) + return std::unexpected(Zero.error()); + auto IsZero = + buildIntegerCompareBefore(Builder, Anchor, IntegerPredicate::Equal, *Base, *Zero); + if (!IsZero) + return std::unexpected(IsZero.error()); + auto BorrowFromIncoming = + buildBinaryBefore(Builder, Anchor, GenericOpcode::G_AND, *Type, *IsZero, *Borrow); + if (!BorrowFromIncoming) + return std::unexpected(BorrowFromIncoming.error()); + auto NextBorrow = buildBinaryBefore(Builder, Anchor, GenericOpcode::G_OR, *Type, *BaseBorrow, + *BorrowFromIncoming); + if (!NextBorrow) + return std::unexpected(NextBorrow.error()); + Borrow = *NextBorrow; + } + } + return Results; +} + +[[nodiscard]] Expected<std::vector<Register>> +emitWideBitwise(MachineIRBuilder &Builder, MachineInstr &Anchor, GenericOpcode Generic, + std::span<const Register> Left, std::span<const Register> Right) { + if (Left.size() != Right.size() || Left.size() < 2) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V wide bitwise operation requires matching multi-limb operands"); + auto Type = LLT::makeAnyScalar(64); + if (!Type) + return std::unexpected(Type.error()); + std::vector<Register> Results; + Results.reserve(Left.size()); + for (std::size_t Index = 0; Index < Left.size(); ++Index) { + auto Part = buildBinaryBefore(Builder, Anchor, Generic, *Type, Left[Index], Right[Index]); + if (!Part) + return std::unexpected(Part.error()); + Results.push_back(*Part); + } + return Results; +} + +[[nodiscard]] Expected<void> lowerWideBinary(MachineInstr &Instruction, MachineFunction &Function, + const TargetDescription &Target, + GenericOpcode Generic) { + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const MachineOperand *LeftOperand = Instruction.operandAt(1); + const MachineOperand *RightOperand = Instruction.operandAt(2); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); + const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf<Register>(); + const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf<Register>(); + if (Destination == nullptr || Left == nullptr || Right == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V wide binary instruction has invalid operands"); + auto LeftParts = collectWideValueParts(*Left, Function); + if (!LeftParts) + return std::unexpected(LeftParts.error()); + auto RightParts = collectWideValueParts(*Right, Function); + if (!RightParts) + return std::unexpected(RightParts.error()); + if (LeftParts->Parts.size() != RightParts->Parts.size()) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V wide binary operands have different limb counts"); + auto ResultPlan = wideResultPlan(*Destination, Function, LeftParts->Parts.size()); + if (!ResultPlan) + return std::unexpected(ResultPlan.error()); + + MachineIRBuilder Builder(Function, Target); + Expected<std::vector<Register>> Results = + Generic == GenericOpcode::G_ADD + ? emitWideAdd(Builder, Instruction, LeftParts->Parts, RightParts->Parts) + : Generic == GenericOpcode::G_SUB + ? emitWideSub(Builder, Instruction, LeftParts->Parts, RightParts->Parts) + : emitWideBitwise(Builder, Instruction, Generic, LeftParts->Parts, RightParts->Parts); + if (!Results) + return std::unexpected(Results.error()); + for (std::size_t Index = 0; Index < Results->size(); ++Index) { + auto Replaced = replaceRegisterUses(Function, ResultPlan->Outputs[Index], (*Results)[Index]); + if (!Replaced) + return std::unexpected(Replaced.error()); + } + if (auto Erased = eraseInstruction(*ResultPlan->Unmerge); !Erased) + return std::unexpected(Erased.error()); + if (ResultPlan->Extension != nullptr) { + if (auto Erased = eraseInstruction(*ResultPlan->Extension); !Erased) + return std::unexpected(Erased.error()); + } + if (auto Erased = eraseInstruction(Instruction); !Erased) + return std::unexpected(Erased.error()); + LeftParts->Artifacts.insert(LeftParts->Artifacts.end(), RightParts->Artifacts.begin(), + RightParts->Artifacts.end()); + eraseDeadArtifacts(LeftParts->Artifacts, Function); + return {}; +} + +[[nodiscard]] bool canLowerNarrowBranchCondition(const MachineInstr &Instruction, + const MachineFunction &Function) { + if (Instruction.opcode() != opcode(GenericOpcode::G_BRCOND) || Instruction.operandCount() < 2) + return false; + const MachineOperand *ConditionOperand = Instruction.operandAt(0); + const Register *Condition = + ConditionOperand == nullptr ? nullptr : ConditionOperand->getIf<Register>(); + if (Condition == nullptr || !Condition->isVirtual() || + singleRegisterUse(Function, *Condition) != &Instruction) + return false; + const auto ConditionInfo = Function.registerInfo().virtualRegisterInfo(*Condition); + const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(*Condition); + const MachineInstr *Truncation = Definition == nullptr ? nullptr : Definition->parent(); + if (!ConditionInfo || !ConditionInfo->get().Type || + ConditionInfo->get().Type->scalarBitWidth() != 1 || Truncation == nullptr || + Truncation->opcode() != opcode(GenericOpcode::G_TRUNC) || Truncation->operandCount() != 2) + return false; + return operandType(Function, *Truncation, 1) + .transform([](LLT Type) { return isScalarGPRType(Type) && Type.scalarBitWidth() == 64; }) + .value_or(false); +} + +[[nodiscard]] Expected<void> lowerNarrowBranchCondition(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target) { + if (!canLowerNarrowBranchCondition(Instruction, Function)) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V branch condition is not a narrow truncation"); + const MachineOperand *ConditionOperand = Instruction.operandAt(0); + const Register Condition = *ConditionOperand->getIf<Register>(); + const MachineOperand *Definition = Function.registerInfo().getUniqueVRegDef(Condition); + MachineInstr *Truncation = const_cast<MachineInstr *>(Definition->parent()); + const Register Source = *Truncation->operandAt(1)->getIf<Register>(); + auto Type = LLT::makeAnyScalar(64); + if (!Type) + return std::unexpected(Type.error()); + MachineIRBuilder Builder(Function, Target); + auto One = buildIntegerConstantBefore(Builder, Instruction, 64, 1); + if (!One) + return std::unexpected(One.error()); + auto Masked = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_AND, *Type, Source, *One); + if (!Masked) + return std::unexpected(Masked.error()); + auto Replaced = Instruction.operandAt(0)->setRegister(*Masked); + if (!Replaced) + return std::unexpected(Replaced.error()); + if (Function.registerInfo().useEmpty(Condition)) + return eraseInstruction(*Truncation); + return {}; +} + +struct NarrowPhiIncoming final { + Register Value; + MachineBasicBlock *Block = nullptr; + MachineInstr *ReusableTruncation = nullptr; +}; + +[[nodiscard]] MachineInstr *firstInstructionAfterLeadingCopies(MachineBasicBlock &Block) { + MachineInstr *Instruction = Block.firstInstruction(); + while (Instruction != nullptr && Instruction->opcode() == opcode(GenericOpcode::COPY)) + Instruction = Instruction->next(); + return Instruction; +} + +[[nodiscard]] Expected<Register> +buildIncomingConversion(MachineFunction &Function, const TargetDescription &Target, + MachineBasicBlock &Block, MachineInstr *Anchor, GenericOpcode Generic, + LLT Type, Register Source) { + auto Result = Function.registerInfo().createVirtualRegister(Type); + if (!Result) + return std::unexpected(Result.error()); + MachineIRBuilder Builder(Function, Target); + auto Point = + Anchor == nullptr ? Builder.setInsertPoint(Block) : Builder.setInsertPointBefore(*Anchor); + if (!Point) + return std::unexpected(Point.error()); + auto Pending = Builder.buildInstr(opcode(Generic)); + if (!Pending) + return std::unexpected(Pending.error()); + if (auto Added = Pending->addDef(*Result); !Added) + return std::unexpected(Added.error()); + if (auto Added = Pending->addUse(Source); !Added) + return std::unexpected(Added.error()); + auto Built = Pending->build(); + if (!Built) + return std::unexpected(Built.error()); + return *Result; +} + +[[nodiscard]] Expected<void> lowerNarrowPhi(MachineInstr &Instruction, MachineFunction &Function, + const TargetDescription &Target) { + const std::optional<LLT> OriginalType = resultType(Function, Instruction); + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); + if (!OriginalType || Destination == nullptr || OriginalType->scalarBitWidth() >= 32 || + Instruction.operandCount() < 3 || (Instruction.operandCount() % 2U) == 0) + return makeUnexpected(ErrorCode::VerificationError, "RISC-V narrow PHI has invalid operands"); + auto LegalType = LLT::makeAnyScalar(32); + if (!LegalType) + return std::unexpected(LegalType.error()); + + std::vector<NarrowPhiIncoming> Incoming; + Incoming.reserve((Instruction.operandCount() - 1U) / 2U); + for (std::size_t Index = 1; Index < Instruction.operandCount(); Index += 2) { + const MachineOperand *ValueOperand = Instruction.operandAt(Index); + const Register *Value = ValueOperand == nullptr ? nullptr : ValueOperand->getIf<Register>(); + const MachineOperand *BlockOperand = Instruction.operandAt(Index + 1U); + const auto *Block = + BlockOperand == nullptr ? nullptr : BlockOperand->getIf<BasicBlockOperand>(); + if (Value == nullptr || Block == nullptr || Block->Block == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V narrow PHI has an invalid incoming edge"); + MachineInstr *Reusable = nullptr; + const MachineOperand *Definition = + Value->isVirtual() ? Function.registerInfo().getUniqueVRegDef(*Value) : nullptr; + MachineInstr *Def = + Definition == nullptr ? nullptr : const_cast<MachineInstr *>(Definition->parent()); + if (Def != nullptr && Def->opcode() == opcode(GenericOpcode::G_TRUNC) && + Def->operandCount() == 2 && singleRegisterUse(Function, *Value) == &Instruction && + operandType(Function, *Def, 1) + .transform([](LLT Type) { return Type.scalarBitWidth() >= 32; }) + .value_or(false)) { + Reusable = Def; + } + Incoming.push_back( + NarrowPhiIncoming{.Value = *Value, .Block = Block->Block, .ReusableTruncation = Reusable}); + } + for (NarrowPhiIncoming &Entry : Incoming) { + if (Entry.ReusableTruncation == nullptr) + continue; + const std::size_t Count = static_cast<std::size_t>(std::ranges::count( + Incoming, Entry.ReusableTruncation, &NarrowPhiIncoming::ReusableTruncation)); + if (Count != 1) + Entry.ReusableTruncation = nullptr; + } + for (NarrowPhiIncoming &Entry : Incoming) { + if (Entry.ReusableTruncation == nullptr) + continue; + MachineBasicBlock *OriginalBlock = Entry.ReusableTruncation->parent(); + if (OriginalBlock == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V narrow PHI input truncation is detached"); + auto Removed = OriginalBlock->removeInstruction(*Entry.ReusableTruncation); + if (!Removed) + return std::unexpected(Removed.error()); + } + + std::vector<std::pair<MachineBasicBlock *, MachineInstr *>> Anchors; + for (const NarrowPhiIncoming &Entry : Incoming) { + if (std::ranges::find(Anchors, Entry.Block, &decltype(Anchors)::value_type::first) == + Anchors.end()) { + Anchors.emplace_back(Entry.Block, firstInstructionAfterLeadingCopies(*Entry.Block)); + } + } + std::vector<Register> LegalIncoming; + LegalIncoming.reserve(Incoming.size()); + for (NarrowPhiIncoming &Entry : Incoming) { + const auto AnchorPosition = + std::ranges::find(Anchors, Entry.Block, &decltype(Anchors)::value_type::first); + MachineInstr *Anchor = AnchorPosition->second; + if (Entry.ReusableTruncation != nullptr) { + auto SetType = Function.registerInfo().setType(Entry.Value, *LegalType); + if (!SetType) + return std::unexpected(SetType.error()); + auto Inserted = + Anchor == nullptr + ? Entry.Block->appendInstruction(*Entry.ReusableTruncation) + : Entry.Block->insertInstructionBefore(*Anchor, *Entry.ReusableTruncation); + if (!Inserted) + return std::unexpected(Inserted.error()); + LegalIncoming.push_back(Entry.Value); + continue; + } + const auto ValueInfo = Function.registerInfo().virtualRegisterInfo(Entry.Value); + if (!ValueInfo || !ValueInfo->get().Type) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V narrow PHI input has no scalar type"); + const std::uint32_t Width = ValueInfo->get().Type->scalarBitWidth(); + const GenericOpcode Conversion = Width < 32 ? GenericOpcode::G_ANYEXT + : Width > 32 ? GenericOpcode::G_TRUNC + : GenericOpcode::COPY; + auto Converted = buildIncomingConversion(Function, Target, *Entry.Block, Anchor, Conversion, + *LegalType, Entry.Value); + if (!Converted) + return std::unexpected(Converted.error()); + LegalIncoming.push_back(*Converted); + } + + auto LegalResult = Function.registerInfo().createVirtualRegister(*LegalType); + if (!LegalResult) + return std::unexpected(LegalResult.error()); + MachineIRBuilder Builder(Function, Target); + auto Point = Builder.setInsertPointBefore(Instruction); + if (!Point) + return std::unexpected(Point.error()); + auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_PHI)); + if (!Pending) + return std::unexpected(Pending.error()); + if (auto Added = Pending->addDef(*LegalResult); !Added) + return std::unexpected(Added.error()); + for (std::size_t Index = 0; Index < Incoming.size(); ++Index) { + if (auto Added = Pending->addUse(LegalIncoming[Index]); !Added) + return std::unexpected(Added.error()); + if (auto Added = Pending->addBasicBlock(*Incoming[Index].Block); !Added) + return std::unexpected(Added.error()); + } + auto Built = Pending->build(); + if (!Built) + return std::unexpected(Built.error()); + auto Replaced = replaceRegisterUses(Function, *Destination, *LegalResult); + if (!Replaced) + return std::unexpected(Replaced.error()); + return eraseInstruction(Instruction); +} + +[[nodiscard]] Expected<void> lowerWidePhi(MachineInstr &Instruction, MachineFunction &Function, + const TargetDescription &Target) { + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); + if (Destination == nullptr || Instruction.operandCount() < 5 || + (Instruction.operandCount() % 2U) == 0) + return makeUnexpected(ErrorCode::VerificationError, "RISC-V wide PHI has invalid operands"); + + std::vector<WideValueParts> IncomingParts; + std::vector<MachineBasicBlock *> IncomingBlocks; + for (std::size_t Index = 1; Index < Instruction.operandCount(); Index += 2) { + const MachineOperand *ValueOperand = Instruction.operandAt(Index); + const Register *Value = ValueOperand == nullptr ? nullptr : ValueOperand->getIf<Register>(); + const MachineOperand *BlockOperand = Instruction.operandAt(Index + 1U); + const auto *Block = + BlockOperand == nullptr ? nullptr : BlockOperand->getIf<BasicBlockOperand>(); + if (Value == nullptr || Block == nullptr || Block->Block == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V wide PHI has an invalid incoming edge"); + auto Parts = collectWideValueParts(*Value, Function); + if (!Parts) + return std::unexpected(Parts.error()); + if (!IncomingParts.empty() && Parts->Parts.size() != IncomingParts.front().Parts.size()) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V wide PHI incoming values have different limb counts"); + IncomingParts.push_back(std::move(*Parts)); + IncomingBlocks.push_back(Block->Block); + } + const std::size_t PartCount = IncomingParts.front().Parts.size(); + auto ResultPlan = wideResultPlan(*Destination, Function, PartCount); + if (!ResultPlan) + return std::unexpected(ResultPlan.error()); + auto Type = LLT::makeAnyScalar(64); + if (!Type) + return std::unexpected(Type.error()); + MachineIRBuilder Builder(Function, Target); + std::vector<Register> Results; + Results.reserve(PartCount); + for (std::size_t PartIndex = 0; PartIndex < PartCount; ++PartIndex) { + auto Result = Function.registerInfo().createVirtualRegister(*Type); + if (!Result) + return std::unexpected(Result.error()); + auto Point = Builder.setInsertPointBefore(Instruction); + if (!Point) + return std::unexpected(Point.error()); + auto Pending = Builder.buildInstr(opcode(GenericOpcode::G_PHI)); + if (!Pending) + return std::unexpected(Pending.error()); + if (auto Added = Pending->addDef(*Result); !Added) + return std::unexpected(Added.error()); + for (std::size_t IncomingIndex = 0; IncomingIndex < IncomingParts.size(); ++IncomingIndex) { + if (auto Added = Pending->addUse(IncomingParts[IncomingIndex].Parts[PartIndex]); !Added) + return std::unexpected(Added.error()); + if (auto Added = Pending->addBasicBlock(*IncomingBlocks[IncomingIndex]); !Added) + return std::unexpected(Added.error()); + } + auto Built = Pending->build(); + if (!Built) + return std::unexpected(Built.error()); + Results.push_back(*Result); + } + for (std::size_t Index = 0; Index < PartCount; ++Index) { + auto Replaced = replaceRegisterUses(Function, ResultPlan->Outputs[Index], Results[Index]); + if (!Replaced) + return std::unexpected(Replaced.error()); + } + if (auto Erased = eraseInstruction(*ResultPlan->Unmerge); !Erased) + return std::unexpected(Erased.error()); + if (ResultPlan->Extension != nullptr) { + if (auto Erased = eraseInstruction(*ResultPlan->Extension); !Erased) + return std::unexpected(Erased.error()); + } + if (auto Erased = eraseInstruction(Instruction); !Erased) + return std::unexpected(Erased.error()); + std::vector<MachineInstr *> Artifacts; + for (WideValueParts &Parts : IncomingParts) + Artifacts.insert(Artifacts.end(), Parts.Artifacts.begin(), Parts.Artifacts.end()); + eraseDeadArtifacts(Artifacts, Function); + return {}; +} + +[[nodiscard]] bool canFoldWideMergeImmediately(const MachineInstr &Instruction, + const MachineFunction &Function) { + const MachineOperand *LowOperand = Instruction.operandAt(1); + const Register *Low = LowOperand == nullptr ? nullptr : LowOperand->getIf<Register>(); + if (Low == nullptr) + return false; + const auto LowInfo = Function.registerInfo().virtualRegisterInfo(*Low); + if (!LowInfo || !LowInfo->get().Type) + return false; + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); + if (Destination == nullptr) + return false; + auto &RegisterInfo = const_cast<MachineRegisterInfo &>(Function.registerInfo()); + for (MachineOperand &Use : RegisterInfo.useOperands(*Destination)) { + MachineInstr *Trunc = Use.parent(); + if (Trunc == nullptr || Trunc->opcode() != opcode(GenericOpcode::G_TRUNC) || + Trunc->operandAt(1) != &Use) + return false; + const std::optional<LLT> Result = resultType(Function, *Trunc); + if (!Result || Result->scalarBitWidth() > LowInfo->get().Type->scalarBitWidth()) + return false; + } + return true; +} + +struct RV64ScalarLayout final { + std::uint64_t StoreSize = 0; + std::uint64_t AllocationSize = 0; + std::uint64_t ABIAlignment = 0; +}; + +[[nodiscard]] std::optional<RV64ScalarLayout> rv64ScalarLayout(LLT Type) noexcept { + if (Type.isPointer()) { + return Type.addressSpace() == 0 && Type.scalarBitWidth() == 64 + ? std::optional<RV64ScalarLayout>{{8, 8, 8}} + : std::nullopt; + } + if ((!isSupportedIntegerType(Type) && !isRVFloatType(Type)) || Type.scalarBitWidth() == 0) { + return std::nullopt; + } + const std::uint64_t StoreBytes = (Type.scalarBitWidth() + 7U) / 8U; + const std::uint64_t AllocationBytes = std::bit_ceil(StoreBytes); + return RV64ScalarLayout{StoreBytes, AllocationBytes, std::min<std::uint64_t>(AllocationBytes, 8)}; +} + +[[nodiscard]] Expected<const MachineMemOperand *> +createMemoryAccess(MachineFunction &Function, MemoryAccessFlag Access, LLT Type, + std::uint64_t SizeBytes, std::uint64_t AlignmentBytes) { + auto Size = MemorySize::fixed(SizeBytes); + auto Align = Alignment::create(AlignmentBytes); + if (!Size || !Align) + return makeUnexpected(ErrorCode::InternalError, "RISC-V memory facts are invalid"); + MachineMemOperandDesc Description; + Description.Accesses = {Access}; + Description.Size = *Size; + Description.MemoryType = Type; + Description.Align = *Align; + return Function.createMemOperand(std::move(Description)); +} + +struct VAArgPlan final { + Register Destination; + Register ListPointer; + LLT DestinationType; + LLT PointerType; + std::uint64_t Alignment = 0; + RV64ScalarLayout ValueLayout; +}; + +[[nodiscard]] std::optional<VAArgPlan> vaargPlan(const MachineInstr &Instruction, + const MachineFunction &Function) { + if (Instruction.opcode() != opcode(GenericOpcode::G_VAARG) || Instruction.operandCount() != 3 || + !Instruction.memOperands().empty()) { + return std::nullopt; + } + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const MachineOperand *ListOperand = Instruction.operandAt(1); + const MachineOperand *AlignmentOperand = Instruction.operandAt(2); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); + const Register *ListPointer = ListOperand == nullptr ? nullptr : ListOperand->getIf<Register>(); + const std::optional<LLT> DestinationType = resultType(Function, Instruction); + const std::optional<LLT> PointerType = operandType(Function, Instruction, 1); + const auto AlignmentValue = AlignmentOperand == nullptr + ? Expected<std::int64_t>(std::unexpected( + Error(ErrorCode::InvalidArgument, "missing alignment"))) + : signedImmediate(*AlignmentOperand); + const std::optional<RV64ScalarLayout> ValueLayout = + DestinationType ? rv64ScalarLayout(*DestinationType) : std::nullopt; + if (Destination == nullptr || ListPointer == nullptr || !Destination->isVirtual() || + !ListPointer->isVirtual() || !DestinationOperand->isDef() || !ListOperand->isUse() || + !DestinationType || !PointerType || !PointerType->isPointer() || + PointerType->addressSpace() != 0 || PointerType->scalarBitWidth() != 64 || !AlignmentValue || + *AlignmentValue <= 0 || !std::has_single_bit(static_cast<std::uint64_t>(*AlignmentValue)) || + !ValueLayout) { + return std::nullopt; + } + + return VAArgPlan{*Destination, + *ListPointer, + *DestinationType, + *PointerType, + static_cast<std::uint64_t>(*AlignmentValue), + *ValueLayout}; +} + +[[nodiscard]] Expected<void> lowerVAArg(MachineInstr &Instruction, MachineFunction &Function, + const TargetDescription &Target) { + const auto Plan = vaargPlan(Instruction, Function); + if (!Plan) { + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V va_arg legalization requires a scalar result, p0 list, and " + "power-of-two alignment"); + } + + auto ListLoadMemory = + createMemoryAccess(Function, MemoryAccessFlag::Load, Plan->PointerType, 8, 8); + if (!ListLoadMemory) + return std::unexpected(ListLoadMemory.error()); + auto ListStoreMemory = + createMemoryAccess(Function, MemoryAccessFlag::Store, Plan->PointerType, 8, 8); + if (!ListStoreMemory) + return std::unexpected(ListStoreMemory.error()); + + auto CurrentList = Function.registerInfo().createVirtualRegister(Plan->PointerType); + if (!CurrentList) + return std::unexpected(CurrentList.error()); + MachineIRBuilder Builder(Function, Target); + if (auto Point = Builder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + auto ListLoad = Builder.buildInstr(opcode(GenericOpcode::G_LOAD)); + if (!ListLoad) + return std::unexpected(ListLoad.error()); + if (auto Added = ListLoad->addDef(*CurrentList); !Added) + return std::unexpected(Added.error()); + if (auto Added = ListLoad->addUse(Plan->ListPointer); !Added) + return std::unexpected(Added.error()); + if (auto Added = ListLoad->addMemOperand(**ListLoadMemory); !Added) + return std::unexpected(Added.error()); + if (auto Built = ListLoad->build(); !Built) + return std::unexpected(Built.error()); + + Register ValueAddress = *CurrentList; + if (Plan->Alignment > 1) { + auto AlignmentOffset = + buildIntegerConstantBefore(Builder, Instruction, 64, Plan->Alignment - 1U); + if (!AlignmentOffset) + return std::unexpected(AlignmentOffset.error()); + auto BiasedAddress = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_PTR_ADD, + Plan->PointerType, ValueAddress, *AlignmentOffset); + if (!BiasedAddress) + return std::unexpected(BiasedAddress.error()); + auto AlignmentMask = + buildIntegerConstantBefore(Builder, Instruction, 64, ~(Plan->Alignment - std::uint64_t{1})); + if (!AlignmentMask) + return std::unexpected(AlignmentMask.error()); + auto AlignedAddress = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_PTRMASK, + Plan->PointerType, *BiasedAddress, *AlignmentMask); + if (!AlignedAddress) + return std::unexpected(AlignedAddress.error()); + ValueAddress = *AlignedAddress; + } + + auto Increment = + buildIntegerConstantBefore(Builder, Instruction, 64, Plan->ValueLayout.AllocationSize); + if (!Increment) + return std::unexpected(Increment.error()); + auto NextList = buildBinaryBefore(Builder, Instruction, GenericOpcode::G_PTR_ADD, + Plan->PointerType, ValueAddress, *Increment); + if (!NextList) + return std::unexpected(NextList.error()); + auto ListStore = Builder.buildInstr(opcode(GenericOpcode::G_STORE)); + if (!ListStore) + return std::unexpected(ListStore.error()); + if (auto Added = ListStore->addUse(*NextList); !Added) + return std::unexpected(Added.error()); + if (auto Added = ListStore->addUse(Plan->ListPointer); !Added) + return std::unexpected(Added.error()); + if (auto Added = ListStore->addMemOperand(**ListStoreMemory); !Added) + return std::unexpected(Added.error()); + if (auto Built = ListStore->build(); !Built) + return std::unexpected(Built.error()); + + if (!Function.registerInfo().useEmpty(Plan->Destination)) { + auto ValueLoadMemory = + createMemoryAccess(Function, MemoryAccessFlag::Load, Plan->DestinationType, + Plan->ValueLayout.StoreSize, Plan->ValueLayout.ABIAlignment); + if (!ValueLoadMemory) + return std::unexpected(ValueLoadMemory.error()); + auto ValueLoad = Builder.buildInstr(opcode(GenericOpcode::G_LOAD)); + if (!ValueLoad) + return std::unexpected(ValueLoad.error()); + if (auto Added = ValueLoad->addDef(Plan->Destination); !Added) + return std::unexpected(Added.error()); + if (auto Added = ValueLoad->addUse(ValueAddress); !Added) + return std::unexpected(Added.error()); + if (auto Added = ValueLoad->addMemOperand(**ValueLoadMemory); !Added) + return std::unexpected(Added.error()); + if (auto Built = ValueLoad->build(); !Built) + return std::unexpected(Built.error()); + } + return eraseInstruction(Instruction); +} + +struct VACopyPlan final { + Register Destination; + Register Source; + LLT PointerType; +}; + +[[nodiscard]] std::optional<VACopyPlan> vacopyPlan(const MachineInstr &Instruction, + const MachineFunction &Function) { + if (Instruction.opcode() != opcode(GenericOpcode::G_INTRINSIC_W_SIDE_EFFECTS) || + Instruction.operandCount() != 3 || !Instruction.memOperands().empty()) { + return std::nullopt; + } + const MachineOperand *IntrinsicIDOperand = Instruction.operandAt(0); + const MachineOperand *DestinationOperand = Instruction.operandAt(1); + const MachineOperand *SourceOperand = Instruction.operandAt(2); + const auto *Intrinsic = + IntrinsicIDOperand == nullptr ? nullptr : IntrinsicIDOperand->getIf<IntrinsicOperand>(); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); + const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>(); + const std::optional<LLT> DestinationType = operandType(Function, Instruction, 1); + const std::optional<LLT> SourceType = operandType(Function, Instruction, 2); + if (Intrinsic == nullptr || Intrinsic->ID != intrinsicID(GenericIntrinsic::VACopy) || + Destination == nullptr || Source == nullptr || !Destination->isVirtual() || + !Source->isVirtual() || !DestinationOperand->isUse() || !SourceOperand->isUse() || + !DestinationType || !SourceType || *DestinationType != *SourceType || + !DestinationType->isPointer() || DestinationType->addressSpace() != 0 || + DestinationType->scalarBitWidth() != 64) { + return std::nullopt; + } + return VACopyPlan{*Destination, *Source, *DestinationType}; +} + +[[nodiscard]] Expected<void> lowerVACopy(MachineInstr &Instruction, MachineFunction &Function, + const TargetDescription &Target) { + const auto Plan = vacopyPlan(Instruction, Function); + if (!Plan) { + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V va_copy legalization requires two p0 operands"); + } + auto Size = MemorySize::fixed(8); + auto Align = Alignment::create(8); + if (!Size || !Align) + return makeUnexpected(ErrorCode::InternalError, "RISC-V pointer memory facts are invalid"); + + MachineMemOperandDesc LoadDescription; + LoadDescription.Accesses = {MemoryAccessFlag::Load}; + LoadDescription.Size = *Size; + LoadDescription.MemoryType = Plan->PointerType; + LoadDescription.Align = *Align; + auto LoadMemory = Function.createMemOperand(std::move(LoadDescription)); + if (!LoadMemory) + return std::unexpected(LoadMemory.error()); + + MachineMemOperandDesc StoreDescription; + StoreDescription.Accesses = {MemoryAccessFlag::Store}; + StoreDescription.Size = *Size; + StoreDescription.MemoryType = Plan->PointerType; + StoreDescription.Align = *Align; + auto StoreMemory = Function.createMemOperand(std::move(StoreDescription)); + if (!StoreMemory) + return std::unexpected(StoreMemory.error()); + + auto Value = Function.registerInfo().createVirtualRegister(Plan->PointerType); + if (!Value) + return std::unexpected(Value.error()); + MachineIRBuilder Builder(Function, Target); + if (auto Point = Builder.setInsertPointBefore(Instruction); !Point) + return std::unexpected(Point.error()); + + auto Load = Builder.buildInstr(opcode(GenericOpcode::G_LOAD)); + if (!Load) + return std::unexpected(Load.error()); + if (auto Added = Load->addDef(*Value); !Added) + return std::unexpected(Added.error()); + if (auto Added = Load->addUse(Plan->Source); !Added) + return std::unexpected(Added.error()); + if (auto Added = Load->addMemOperand(**LoadMemory); !Added) + return std::unexpected(Added.error()); + if (auto Built = Load->build(); !Built) + return std::unexpected(Built.error()); + + auto Store = Builder.buildInstr(opcode(GenericOpcode::G_STORE)); + if (!Store) + return std::unexpected(Store.error()); + if (auto Added = Store->addUse(*Value); !Added) + return std::unexpected(Added.error()); + if (auto Added = Store->addUse(Plan->Destination); !Added) + return std::unexpected(Added.error()); + if (auto Added = Store->addMemOperand(**StoreMemory); !Added) + return std::unexpected(Added.error()); + if (auto Built = Store->build(); !Built) + return std::unexpected(Built.error()); + return eraseInstruction(Instruction); +} + +} // namespace + +Expected<LegalizationAction> RISCVLegalizerInfo::classify(const MachineInstr &Instruction, + const MachineFunction &Function, + const TargetDescription &) const { + if (!Subtarget.features().has(RISCVExtension::I)) { + return makeUnexpected(ErrorCode::InternalError, + "RISC-V legalizer requires the base I extension"); + } + if (!isGenericOpcode(Instruction.opcode())) + return LegalizationAction::Legal; + + const GenericOpcode Generic = static_cast<GenericOpcode>(Instruction.opcode().value()); + const std::optional<LLT> Type = resultType(Function, Instruction); + switch (Generic) { + case GenericOpcode::G_INTRINSIC_W_SIDE_EFFECTS: + return vacopyPlan(Instruction, Function) ? LegalizationAction::Custom + : LegalizationAction::Unsupported; + case GenericOpcode::G_VAARG: + return vaargPlan(Instruction, Function) ? LegalizationAction::Custom + : LegalizationAction::Unsupported; + case GenericOpcode::G_FRAME_INDEX: + return Type && Type->isPointer() && Type->addressSpace() == 0 && Type->scalarBitWidth() == 64 + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + case GenericOpcode::G_BR: + return LegalizationAction::Legal; + case GenericOpcode::G_BRCOND: { + const std::optional<LLT> Condition = operandType(Function, Instruction, 0); + if (!Condition || !isScalarGPRType(*Condition)) + return LegalizationAction::Unsupported; + return canLowerNarrowBranchCondition(Instruction, Function) ? LegalizationAction::Custom + : LegalizationAction::Legal; + } + case GenericOpcode::G_PHI: { + if (!Type) + return LegalizationAction::Unsupported; + for (std::size_t Index = 1; Index < Instruction.operandCount(); Index += 2) { + const std::optional<LLT> Incoming = operandType(Function, Instruction, Index); + if (!Incoming || *Incoming != *Type) + return LegalizationAction::Unsupported; + } + if ((Type->isInteger() || Type->isAnyScalar()) && Type->scalarBitWidth() > 64) + return LegalizationAction::Custom; + if ((Type->isInteger() || Type->isAnyScalar()) && Type->scalarBitWidth() < 32) + return LegalizationAction::Custom; + if (!isSupportedIntegerType(*Type) && !isRVFloatType(*Type)) + return LegalizationAction::Unsupported; + return LegalizationAction::Legal; + } + case GenericOpcode::G_IMPLICIT_DEF: + return Type && (isSupportedIntegerType(*Type) || isRVFloatType(*Type)) + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + case GenericOpcode::G_STORE: + if (Instruction.operandCount() == 0) + return LegalizationAction::Unsupported; + if (const std::optional<LLT> ValueType = registerType(Function, *Instruction.operandAt(0)); + ValueType) { + const std::optional<LLT> AddressType = operandType(Function, Instruction, 1); + if (!AddressType || !AddressType->isPointer() || AddressType->addressSpace() != 0 || + AddressType->scalarBitWidth() != 64) { + return LegalizationAction::Unsupported; + } + if (Instruction.memOperands().size() == 1 && Instruction.memOperands().front()->isAtomic()) { + const std::uint64_t Bytes = ValueType->scalarBitWidth() / 8U; + return isSupportedIntegerType(*ValueType) && + isSupportedAtomicMemoryAccess(Instruction, AtomicOperation::Store, Bytes) + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + if ((isSupportedIntegerType(*ValueType) || isRVFloatType(*ValueType)) && + isSupportedNonAtomicMemoryAccess(Instruction) && + memoryMatchesType(Instruction, *ValueType)) { + return LegalizationAction::Legal; + } + } + return LegalizationAction::Unsupported; + case GenericOpcode::G_SEXTLOAD: + case GenericOpcode::G_ZEXTLOAD: { + const auto Bytes = memoryAccessBytes(Instruction); + const std::optional<LLT> AddressType = operandType(Function, Instruction, 1); + return Type && Bytes && AddressType && isSupportedIntegerType(*Type) && + AddressType->isPointer() && AddressType->addressSpace() == 0 && + AddressType->scalarBitWidth() == 64 && + *Bytes < ((Type->scalarBitWidth() + 7U) / 8U) && + (*Bytes == 1 || *Bytes == 2 || *Bytes == 4) && + isSupportedNonAtomicMemoryAccess(Instruction) + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_FPEXTLOAD: { + const auto Bytes = memoryAccessBytes(Instruction); + const std::optional<LLT> AddressType = operandType(Function, Instruction, 1); + return Type && Bytes && AddressType && Type->isFloat() && Type->scalarBitWidth() == 64 && + AddressType->isPointer() && AddressType->addressSpace() == 0 && + AddressType->scalarBitWidth() == 64 && *Bytes == 4 && + isSupportedNonAtomicMemoryAccess(Instruction) + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_FPTRUNCSTORE: { + const std::optional<LLT> ValueType = operandType(Function, Instruction, 0); + const std::optional<LLT> AddressType = operandType(Function, Instruction, 1); + const auto Bytes = memoryAccessBytes(Instruction); + return ValueType && AddressType && Bytes && ValueType->isFloat() && + ValueType->scalarBitWidth() == 64 && AddressType->isPointer() && + AddressType->addressSpace() == 0 && AddressType->scalarBitWidth() == 64 && + *Bytes == 4 && isSupportedNonAtomicMemoryAccess(Instruction) + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_LOAD: + if (Type) { + const std::optional<LLT> AddressType = operandType(Function, Instruction, 1); + if (!AddressType || !AddressType->isPointer() || AddressType->addressSpace() != 0 || + AddressType->scalarBitWidth() != 64) { + return LegalizationAction::Unsupported; + } + if (Instruction.memOperands().size() == 1 && Instruction.memOperands().front()->isAtomic()) { + const std::uint64_t Bytes = Type->scalarBitWidth() / 8U; + return isSupportedIntegerType(*Type) && + isSupportedAtomicMemoryAccess(Instruction, AtomicOperation::Load, Bytes) + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + if ((isSupportedIntegerType(*Type) || isRVFloatType(*Type)) && + isSupportedNonAtomicMemoryAccess(Instruction) && memoryMatchesType(Instruction, *Type)) { + return LegalizationAction::Legal; + } + } + return LegalizationAction::Unsupported; + case GenericOpcode::G_FADD: + case GenericOpcode::G_FSUB: + case GenericOpcode::G_FMUL: + case GenericOpcode::G_FDIV: { + const std::optional<LLT> Left = operandType(Function, Instruction, 1); + const std::optional<LLT> Right = operandType(Function, Instruction, 2); + return Type && Left && Right && isRVFloatType(*Type) && *Left == *Type && *Right == *Type + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_FMA: { + const std::optional<LLT> First = operandType(Function, Instruction, 1); + const std::optional<LLT> Second = operandType(Function, Instruction, 2); + const std::optional<LLT> Third = operandType(Function, Instruction, 3); + return Type && First && Second && Third && isRVFloatType(*Type) && *First == *Type && + *Second == *Type && *Third == *Type + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_FNEG: + case GenericOpcode::G_FABS: + case GenericOpcode::G_FSQRT: { + const std::optional<LLT> Source = operandType(Function, Instruction, 1); + return Type && Source && isRVFloatType(*Type) && *Source == *Type + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_FCOPYSIGN: { + const std::optional<LLT> Left = operandType(Function, Instruction, 1); + const std::optional<LLT> Right = operandType(Function, Instruction, 2); + return Type && Left && Right && isRVFloatType(*Type) && *Left == *Type && *Right == *Type + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_FCONSTANT: { + const MachineOperand *Immediate = Instruction.operandAt(1); + const auto *Floating = + Immediate == nullptr ? nullptr : Immediate->getIf<FloatingImmediateOperand>(); + const std::optional<FloatSemantics> Semantics = Type ? Type->floatSemantics() : std::nullopt; + return Type && isRVFloatType(*Type) && Floating != nullptr && Semantics && + Floating->Bits.semantics() == *Semantics + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_ASSERT_SEXT: + case GenericOpcode::G_ASSERT_ZEXT: + case GenericOpcode::G_ASSERT_ALIGN: { + const std::optional<LLT> Source = operandType(Function, Instruction, 1); + return Type && Source && *Type == *Source && isScalarGPRType(*Type) && + Instruction.operandCount() == 3 + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_FREEZE: + case GenericOpcode::G_CONSTANT_FOLD_BARRIER: { + const std::optional<LLT> Source = operandType(Function, Instruction, 1); + return Type && Source && *Type == *Source && + (isSupportedIntegerType(*Type) || isRVFloatType(*Type)) + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_PTRTOINT: { + const std::optional<LLT> Source = operandType(Function, Instruction, 1); + return Type && Source && Type->isInteger() && Type->scalarBitWidth() == 64 && + Source->isPointer() && Source->scalarBitWidth() == 64 + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_INTTOPTR: { + const std::optional<LLT> Source = operandType(Function, Instruction, 1); + return Type && Source && Type->isPointer() && Type->scalarBitWidth() == 64 && + Source->isInteger() && Source->scalarBitWidth() == 64 + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_BITCAST: { + const std::optional<LLT> Source = operandType(Function, Instruction, 1); + const bool IntegerFloatPair = + Type && Source && Type->scalarBitWidth() == Source->scalarBitWidth() && + ((Type->isFloat() && Source->isInteger()) || (Type->isInteger() && Source->isFloat())) && + (Type->scalarBitWidth() == 32 || Type->scalarBitWidth() == 64); + return IntegerFloatPair ? LegalizationAction::Legal : LegalizationAction::Unsupported; + } + case GenericOpcode::G_ANYEXT: + case GenericOpcode::G_SEXT: + case GenericOpcode::G_ZEXT: { + const std::optional<LLT> Source = operandType(Function, Instruction, 1); + if (!Type || !Source) + return LegalizationAction::Unsupported; + if (!isScalarIntegerExtension(*Type, *Source)) { + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); + MachineInstr *OnlyUse = + Destination == nullptr ? nullptr : singleUseInstruction(Function, *Destination); + return (Type->isInteger() || Type->isAnyScalar()) && + (Source->isInteger() || Source->isAnyScalar()) && + Type->scalarBitWidth() > 64 && + Source->scalarBitWidth() < Type->scalarBitWidth() && OnlyUse != nullptr && + OnlyUse->opcode() == opcode(GenericOpcode::G_UNMERGE_VALUES) + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + const MachineOperand *SourceOperand = Instruction.operandAt(1); + const Register *SourceRegister = + SourceOperand == nullptr ? nullptr : SourceOperand->getIf<Register>(); + const MachineOperand *SourceDefinition = + SourceRegister == nullptr || !SourceRegister->isVirtual() + ? nullptr + : Function.registerInfo().getUniqueVRegDef(*SourceRegister); + const MachineInstr *SourceDef = + SourceDefinition == nullptr ? nullptr : SourceDefinition->parent(); + if (SourceDef != nullptr && SourceDef->opcode() == opcode(GenericOpcode::G_CONSTANT)) + return LegalizationAction::Custom; + if (Generic == GenericOpcode::G_ANYEXT && + narrowAnyExtBinaryPlan(Instruction, Function).has_value()) + return LegalizationAction::Custom; + return LegalizationAction::Legal; + } + case GenericOpcode::G_TRUNC: { + const std::optional<LLT> Source = operandType(Function, Instruction, 1); + if (Type && Source && (Type->isInteger() || Type->isAnyScalar()) && + (Source->isInteger() || Source->isAnyScalar()) && Type->scalarBitWidth() > 64 && + Source->scalarBitWidth() > Type->scalarBitWidth()) + return LegalizationAction::Legal; + return Type && Source && isSupportedIntegerScalar(*Type) && isSupportedIntegerScalar(*Source) && + Type->scalarBitWidth() != 0 && Source->scalarBitWidth() >= Type->scalarBitWidth() + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_MERGE_VALUES: + case GenericOpcode::G_UNMERGE_VALUES: { + if (Instruction.operandCount() < 3) + return LegalizationAction::Unsupported; + const std::size_t SmallTypeIndex = Generic == GenericOpcode::G_MERGE_VALUES ? 1 : 0; + const std::size_t BigTypeIndex = + Generic == GenericOpcode::G_MERGE_VALUES ? 0 : Instruction.operandCount() - 1U; + const std::optional<LLT> SmallType = operandType(Function, Instruction, SmallTypeIndex); + const std::optional<LLT> BigType = operandType(Function, Instruction, BigTypeIndex); + if (SmallType && BigType && Generic == GenericOpcode::G_MERGE_VALUES && + BigType->scalarBitWidth() > 64 && !canFoldWideMergeImmediately(Instruction, Function)) + return LegalizationAction::Legal; + return SmallType && BigType && (SmallType->isInteger() || SmallType->isAnyScalar()) && + (BigType->isInteger() || BigType->isAnyScalar()) && + SmallType->scalarBitWidth() != 0 && BigType->scalarBitWidth() != 0 + ? LegalizationAction::Custom + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_SEXT_INREG: { + const std::optional<LLT> Source = operandType(Function, Instruction, 1); + const MachineOperand *WidthOperand = Instruction.operandAt(2); + const auto Width = + WidthOperand == nullptr + ? Expected<std::int64_t>(std::unexpected( + Error(ErrorCode::InvalidArgument, "missing in-register extension width"))) + : signedImmediate(*WidthOperand); + return Type && Source && *Type == *Source && isScalarGPRType(*Type) && Width && *Width > 0 && + static_cast<std::uint64_t>(*Width) <= Type->scalarBitWidth() + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_ICMP: { + const MachineOperand *Predicate = Instruction.operandAt(1); + const auto *IntegerPredicate = + Predicate == nullptr ? nullptr : Predicate->getIf<PredicateOperand>(); + const std::optional<LLT> Left = operandType(Function, Instruction, 2); + const std::optional<LLT> Right = operandType(Function, Instruction, 3); + if (!Type || !isSupportedIntegerScalar(*Type) || IntegerPredicate == nullptr || + IntegerPredicate->integerPredicate() == nullptr || !Left || !Right || *Left != *Right || + !isSupportedIntegerType(*Left)) { + return LegalizationAction::Unsupported; + } + return Type->scalarBitWidth() == 64 && Left->scalarBitWidth() == 64 + ? LegalizationAction::Legal + : LegalizationAction::Custom; + } + case GenericOpcode::G_FCMP: { + const MachineOperand *Predicate = Instruction.operandAt(1); + const auto *FloatingPredicate = + Predicate == nullptr ? nullptr : Predicate->getIf<PredicateOperand>(); + const std::optional<LLT> Left = operandType(Function, Instruction, 2); + const std::optional<LLT> Right = operandType(Function, Instruction, 3); + return Type && isI1Type(*Type) && FloatingPredicate != nullptr && + FloatingPredicate->floatingPredicate() != nullptr && Left && Right && + *Left == *Right && isRVFloatType(*Left) + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_FPEXT: + case GenericOpcode::G_FPTRUNC: { + const std::optional<LLT> Source = operandType(Function, Instruction, 1); + const bool IsExtension = Generic == GenericOpcode::G_FPEXT; + return Type && Source && isRVFloatType(*Type) && isRVFloatType(*Source) && + ((IsExtension && Source->scalarBitWidth() == 32 && + Type->scalarBitWidth() == 64) || + (!IsExtension && Source->scalarBitWidth() == 64 && + Type->scalarBitWidth() == 32)) + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_FPTOSI: + case GenericOpcode::G_FPTOUI: { + const std::optional<LLT> Source = operandType(Function, Instruction, 1); + return Type && Source && isSupportedIntegerType(*Type) && isRVFloatType(*Source) + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_SITOFP: + case GenericOpcode::G_UITOFP: { + const std::optional<LLT> Source = operandType(Function, Instruction, 1); + return Type && Source && isRVFloatType(*Type) && isSupportedIntegerType(*Source) + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_FMINNUM: + case GenericOpcode::G_FMAXNUM: { + const std::optional<LLT> Left = operandType(Function, Instruction, 1); + const std::optional<LLT> Right = operandType(Function, Instruction, 2); + return Type && Left && Right && isRVFloatType(*Type) && *Left == *Type && *Right == *Type + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_SELECT: { + const std::optional<LLT> Condition = operandType(Function, Instruction, 1); + const std::optional<LLT> TrueValue = operandType(Function, Instruction, 2); + const std::optional<LLT> FalseValue = operandType(Function, Instruction, 3); + return Type && Condition && TrueValue && FalseValue && isScalarGPRType(*Condition) && + *TrueValue == *Type && *FalseValue == *Type && + (isSupportedIntegerType(*Type) || isRVFloatType(*Type)) + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_PTRMASK: { + const std::optional<LLT> Pointer = operandType(Function, Instruction, 1); + const std::optional<LLT> Mask = operandType(Function, Instruction, 2); + return Type && Pointer && Mask && Type->isPointer() && Type->addressSpace() == 0 && + Type->scalarBitWidth() == 64 && *Pointer == *Type && + isSupportedIntegerScalar(*Mask) && Mask->scalarBitWidth() == 64 + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_ABS: { + const std::optional<LLT> Source = operandType(Function, Instruction, 1); + return Type && Source && *Source == *Type && isSupportedIntegerScalar(*Type) + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_SDIVREM: + case GenericOpcode::G_UDIVREM: { + const std::optional<LLT> Remainder = operandType(Function, Instruction, 1); + const std::optional<LLT> Left = operandType(Function, Instruction, 2); + const std::optional<LLT> Right = operandType(Function, Instruction, 3); + return Type && Remainder && Left && Right && isSupportedIntegerScalar(*Type) && + *Remainder == *Type && *Left == *Type && *Right == *Type + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_SMULH: + case GenericOpcode::G_UMULH: { + const std::optional<LLT> Left = operandType(Function, Instruction, 1); + const std::optional<LLT> Right = operandType(Function, Instruction, 2); + return Type && Left && Right && isNativeIntegerScalar(*Type) && *Left == *Type && + *Right == *Type + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_SMIN: + case GenericOpcode::G_SMAX: + case GenericOpcode::G_UMIN: + case GenericOpcode::G_UMAX: { + const std::optional<LLT> Left = operandType(Function, Instruction, 1); + const std::optional<LLT> Right = operandType(Function, Instruction, 2); + return Type && Left && Right && isSupportedIntegerScalar(*Type) && *Left == *Type && + *Right == *Type + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_UADDO: + case GenericOpcode::G_USUBO: + case GenericOpcode::G_SADDO: + case GenericOpcode::G_SSUBO: + case GenericOpcode::G_UMULO: + case GenericOpcode::G_SMULO: { + const std::optional<LLT> Overflow = operandType(Function, Instruction, 1); + const std::optional<LLT> Left = operandType(Function, Instruction, 2); + const std::optional<LLT> Right = operandType(Function, Instruction, 3); + return Type && Overflow && Left && Right && isNativeIntegerScalar(*Type) && + isI1Type(*Overflow) && *Left == *Type && *Right == *Type + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_UADDE: + case GenericOpcode::G_USUBE: + case GenericOpcode::G_SADDE: + case GenericOpcode::G_SSUBE: { + const std::optional<LLT> CarryOut = operandType(Function, Instruction, 1); + const std::optional<LLT> Left = operandType(Function, Instruction, 2); + const std::optional<LLT> Right = operandType(Function, Instruction, 3); + const std::optional<LLT> CarryIn = operandType(Function, Instruction, 4); + return Type && CarryOut && Left && Right && CarryIn && isNativeIntegerScalar(*Type) && + isI1Type(*CarryOut) && isI1Type(*CarryIn) && *Left == *Type && *Right == *Type + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_ATOMICRMW_XCHG: + case GenericOpcode::G_ATOMICRMW_ADD: + case GenericOpcode::G_ATOMICRMW_SUB: + case GenericOpcode::G_ATOMICRMW_AND: + case GenericOpcode::G_ATOMICRMW_OR: + case GenericOpcode::G_ATOMICRMW_XOR: + case GenericOpcode::G_ATOMICRMW_MAX: + case GenericOpcode::G_ATOMICRMW_MIN: + case GenericOpcode::G_ATOMICRMW_UMAX: + case GenericOpcode::G_ATOMICRMW_UMIN: { + const std::optional<LLT> Address = operandType(Function, Instruction, 1); + const std::optional<LLT> Value = operandType(Function, Instruction, 2); + const std::uint64_t Bytes = Type ? Type->scalarBitWidth() / 8U : 0; + return Type && Address && Value && isNativeIntegerScalar(*Type) && *Value == *Type && + Address->isPointer() && Address->addressSpace() == 0 && + Address->scalarBitWidth() == 64 && + isSupportedAtomicMemoryAccess(Instruction, AtomicOperation::ReadModifyWrite, + Bytes) + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_ATOMIC_CMPXCHG: { + const std::optional<LLT> Address = operandType(Function, Instruction, 1); + const std::optional<LLT> Compare = operandType(Function, Instruction, 2); + const std::optional<LLT> NewValue = operandType(Function, Instruction, 3); + if (!Type || !Address || !Compare || !NewValue || *Compare != *Type || *NewValue != *Type || + (!Type->isInteger() && !Type->isAnyScalar()) || + (Type->scalarBitWidth() != 8 && Type->scalarBitWidth() != 16 && + Type->scalarBitWidth() != 32 && Type->scalarBitWidth() != 64) || + !Address->isPointer() || Address->addressSpace() != 0 || Address->scalarBitWidth() != 64 || + !isSupportedAtomicContainerAccess(Instruction, AtomicOperation::CompareExchange, + (Type->scalarBitWidth() + 7U) / 8U)) { + return LegalizationAction::Unsupported; + } + return Type->scalarBitWidth() == 64 ? LegalizationAction::Legal : LegalizationAction::Custom; + } + case GenericOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: { + const std::optional<LLT> Success = operandType(Function, Instruction, 1); + const std::optional<LLT> Address = operandType(Function, Instruction, 2); + const std::optional<LLT> Compare = operandType(Function, Instruction, 3); + const std::optional<LLT> NewValue = operandType(Function, Instruction, 4); + return Type && Success && Address && Compare && NewValue && isI1Type(*Success) && + *Compare == *Type && *NewValue == *Type && + (Type->isInteger() || Type->isAnyScalar()) && + (Type->scalarBitWidth() == 8 || Type->scalarBitWidth() == 16 || + Type->scalarBitWidth() == 32 || Type->scalarBitWidth() == 64) && + Address->isPointer() && Address->addressSpace() == 0 && + Address->scalarBitWidth() == 64 && + isSupportedAtomicContainerAccess(Instruction, AtomicOperation::CompareExchange, + (Type->scalarBitWidth() + 7U) / 8U) + ? LegalizationAction::Custom + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_FENCE: { + const MachineOperand *OrderingOperand = Instruction.operandAt(0); + const MachineOperand *ScopeOperand = Instruction.operandAt(1); + const auto Ordering = OrderingOperand == nullptr + ? Expected<std::int64_t>(std::unexpected( + Error(ErrorCode::InvalidArgument, "missing fence ordering"))) + : signedImmediate(*OrderingOperand); + const auto Scope = ScopeOperand == nullptr + ? Expected<std::int64_t>(std::unexpected( + Error(ErrorCode::InvalidArgument, "missing fence scope"))) + : signedImmediate(*ScopeOperand); + return Ordering && Scope && *Ordering >= 4 && *Ordering <= 7 && *Scope == 0 + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_CONSTANT_POOL: { + const MachineOperand *PoolOperand = Instruction.operandAt(1); + const auto *Pool = PoolOperand == nullptr ? nullptr : PoolOperand->getIf<ConstantPoolOperand>(); + return Type && Type->isPointer() && Type->addressSpace() == 0 && Type->scalarBitWidth() == 64 && + Pool != nullptr && Function.constantPool().entry(Pool->Index) != nullptr + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_GLOBAL_VALUE: { + const MachineOperand *GlobalOperand = Instruction.operandAt(1); + const auto *Global = + GlobalOperand == nullptr ? nullptr : GlobalOperand->getIf<GlobalSymbolOperand>(); + return Type && Type->isPointer() && Type->addressSpace() == 0 && Type->scalarBitWidth() == 64 && + Global != nullptr && !Global->Name.empty() + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_BRINDIRECT: + return operandType(Function, Instruction, 0) + .transform([](LLT Value) { + return Value.isPointer() && Value.addressSpace() == 0 && + Value.scalarBitWidth() == 64; + }) + .value_or(false) + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + case GenericOpcode::G_TRAP: + case GenericOpcode::G_DEBUGTRAP: + return Instruction.operandCount() == 0 ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + case GenericOpcode::G_UBSANTRAP: + return Instruction.operandCount() == 1 ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + case GenericOpcode::G_CONSTANT: { + const MachineOperand *Immediate = Instruction.operandAt(1); + if (!Type || Immediate == nullptr || + (Immediate->kind() != MachineOperand::Kind::Immediate && + Immediate->kind() != MachineOperand::Kind::IntegerImmediate)) + return LegalizationAction::Unsupported; + if (Type->scalarBitWidth() > 64) + return wideConstantUnmergePlan(Instruction, Function).has_value() + ? LegalizationAction::Custom + : LegalizationAction::Unsupported; + if ((Type->isInteger() || Type->isAnyScalar()) && Type->scalarBitWidth() == 64) { + auto Value = signedImmediate(*Immediate); + if (Value && shouldUseRV64IntegerConstantPool(*Value)) + return LegalizationAction::Custom; + } + return isSupportedIntegerType(*Type) ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_ADD: + case GenericOpcode::G_SUB: + case GenericOpcode::G_MUL: + case GenericOpcode::G_SDIV: + case GenericOpcode::G_UDIV: + case GenericOpcode::G_SREM: + case GenericOpcode::G_UREM: + case GenericOpcode::G_AND: + case GenericOpcode::G_OR: + case GenericOpcode::G_XOR: + case GenericOpcode::G_SHL: + case GenericOpcode::G_LSHR: + case GenericOpcode::G_ASHR: { + const std::optional<LLT> Left = operandType(Function, Instruction, 1); + const std::optional<LLT> Right = operandType(Function, Instruction, 2); + if (!Type || !Left || !Right || *Left != *Type || *Right != *Type || + (!Type->isInteger() && !Type->isAnyScalar())) + return LegalizationAction::Unsupported; + if (Type->scalarBitWidth() > 64) { + return Generic == GenericOpcode::G_ADD || Generic == GenericOpcode::G_SUB || + Generic == GenericOpcode::G_AND || Generic == GenericOpcode::G_OR || + Generic == GenericOpcode::G_XOR || Generic == GenericOpcode::G_SHL + ? LegalizationAction::Custom + : LegalizationAction::Unsupported; + } + if (Generic == GenericOpcode::G_MUL && !Subtarget.features().has(RISCVExtension::M)) { + if (Type->scalarBitWidth() == 64) + return LegalizationAction::Libcall; + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf<Register>(); + MachineInstr *Extension = + Destination == nullptr ? nullptr : singleRegisterUse(Function, *Destination); + const std::optional<LLT> ExtensionType = + Extension == nullptr ? std::nullopt : resultType(Function, *Extension); + if (Extension != nullptr && Extension->opcode() == opcode(GenericOpcode::G_ANYEXT) && + ExtensionType && ExtensionType->scalarBitWidth() == 64) { + return LegalizationAction::Legal; + } + return LegalizationAction::Custom; + } + return isSupportedIntegerScalar(*Type) ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_PTR_ADD: { + const std::optional<LLT> Pointer = operandType(Function, Instruction, 1); + const std::optional<LLT> Offset = operandType(Function, Instruction, 2); + return Type && Pointer && Offset && Type->isPointer() && Type->addressSpace() == 0 && + Type->scalarBitWidth() == 64 && *Pointer == *Type && + isSupportedIntegerScalar(*Offset) && Offset->scalarBitWidth() == 64 + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + default: + break; + } + + // Keep the unsupported policy mechanically synchronized with the complete + // target-independent opcode inventory. Supported cases return above. + switch (Generic) { +#define RCC_CORE_OPCODE(Name) +#define RCC_GENERIC_OPCODE(Name) case GenericOpcode::Name: +#include "rcc/CodeGen/GenericOpcode.def" +#undef RCC_GENERIC_OPCODE +#undef RCC_CORE_OPCODE + return LegalizationAction::Unsupported; + case GenericOpcode::Invalid: + case GenericOpcode::CORE_PSEUDO_END: + case GenericOpcode::G_GENERIC_END: + default: + break; + } + return makeUnexpected(ErrorCode::InternalError, + "RISC-V legalizer received an invalid generic opcode value"); +} + +Expected<void> RISCVLegalizerInfo::lower(MachineInstr &Instruction, MachineFunction &Function, + const TargetDescription &Target, + LegalizationAction Action) const { + if (Action == LegalizationAction::Custom && + Instruction.opcode() == opcode(GenericOpcode::G_INTRINSIC_W_SIDE_EFFECTS)) { + return lowerVACopy(Instruction, Function, Target); + } + if (Action == LegalizationAction::Custom && + Instruction.opcode() == opcode(GenericOpcode::G_VAARG)) { + return lowerVAArg(Instruction, Function, Target); + } + if (Instruction.opcode() == opcode(GenericOpcode::G_MUL) && + !Subtarget.features().has(RISCVExtension::M)) { + if (Action == LegalizationAction::Custom) + return widenNarrowMultiply(Instruction, Function, Target); + if (Action == LegalizationAction::Libcall) + return lowerMultiplyLibcall(Instruction, Function, Target, + Subtarget.features().has(RISCVExtension::D) + ? CallingConvention::LP64D + : CallingConvention::LP64, + *Subtarget.getCallLowering()); + } + if (Action == LegalizationAction::Custom && + resultType(Function, Instruction) + .transform([](LLT Type) { return Type.scalarBitWidth() > 64; }) + .value_or(false)) { + const Opcode Value = Instruction.opcode(); + if (Value == opcode(GenericOpcode::G_ADD) || Value == opcode(GenericOpcode::G_SUB) || + Value == opcode(GenericOpcode::G_AND) || Value == opcode(GenericOpcode::G_OR) || + Value == opcode(GenericOpcode::G_XOR)) { + return lowerWideBinary(Instruction, Function, Target, + static_cast<GenericOpcode>(Value.value())); + } + if (Value == opcode(GenericOpcode::G_SHL)) + return lowerWideShiftLeft(Instruction, Function, Target); + if (Value == opcode(GenericOpcode::G_PHI)) + return lowerWidePhi(Instruction, Function, Target); + } + if (Action == LegalizationAction::Custom && + Instruction.opcode() == opcode(GenericOpcode::G_MERGE_VALUES)) + return lowerMergeValues(Instruction, Function, Target); + if (Action == LegalizationAction::Custom && + Instruction.opcode() == opcode(GenericOpcode::G_UNMERGE_VALUES)) + return lowerUnmergeValues(Instruction, Function, Target); + if (Action == LegalizationAction::Custom && + Instruction.opcode() == opcode(GenericOpcode::G_BRCOND)) + return lowerNarrowBranchCondition(Instruction, Function, Target); + if (Action == LegalizationAction::Custom && Instruction.opcode() == opcode(GenericOpcode::G_PHI)) + return lowerNarrowPhi(Instruction, Function, Target); + if (Action == LegalizationAction::Custom && Instruction.opcode() == opcode(GenericOpcode::G_ICMP)) + return lowerIntegerCompare(Instruction, Function, Target); + if (Action == LegalizationAction::Custom && + Instruction.opcode() == opcode(GenericOpcode::G_ATOMIC_CMPXCHG)) + return lowerAtomicCompareExchange(Instruction, Function, Target); + if (Action == LegalizationAction::Custom && + Instruction.opcode() == opcode(GenericOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS)) + return lowerAtomicCompareExchangeWithSuccess(Instruction, Function, Target); + if (Action == LegalizationAction::Custom && + (Instruction.opcode() == opcode(GenericOpcode::G_ANYEXT) || + Instruction.opcode() == opcode(GenericOpcode::G_SEXT) || + Instruction.opcode() == opcode(GenericOpcode::G_ZEXT))) { + if (Instruction.opcode() == opcode(GenericOpcode::G_ANYEXT) && + narrowAnyExtBinaryPlan(Instruction, Function).has_value()) + return foldNarrowAnyExtBinary(Instruction, Function, Target); + return foldConstantExtension(Instruction, Function, + static_cast<GenericOpcode>(Instruction.opcode().value())); + } + if (Action == LegalizationAction::Custom && + Instruction.opcode() == opcode(GenericOpcode::G_CONSTANT)) { + const MachineOperand *Immediate = Instruction.operandAt(1); + const std::optional<LLT> Type = resultType(Function, Instruction); + if (Immediate != nullptr && Type && (Type->isInteger() || Type->isAnyScalar()) && + Type->scalarBitWidth() == 64) { + auto Value = signedImmediate(*Immediate); + if (Value && shouldUseRV64IntegerConstantPool(*Value)) + return lowerConstantToPool(Instruction, Function, Target); + } + return foldWideConstant(Instruction, Function, Target); + } + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V legalizer has no lowering rule for this instruction"); +} + +} // namespace rcc::riscv |