From 10b7e69448f866bcd6893b21d7f78d766252eb11 Mon Sep 17 00:00:00 2001 From: Richard-Qin-X Date: Sat, 8 Aug 2026 02:58:50 +0800 Subject: [CodeGen] Implement RV64GC legalization and instruction selection --- .../include/rcc/CodeGen/MachinePipelinePasses.h | 63 + backend/include/rcc/CodeGen/TargetCodeGen.h | 94 + backend/include/rcc/Target/RISCV/RISCVCodeGen.h | 16 + backend/lib/CodeGen/CMakeLists.txt | 5 + backend/lib/CodeGen/MachinePipelinePasses.cpp | 614 ++++ backend/lib/Target/RISCV/CMakeLists.txt | 4 + backend/lib/Target/RISCV/RISCVCodeGen.cpp | 3239 ++++++++++++++++++++ backend/unittests/CodeGen/CMakeLists.txt | 14 + .../CodeGen/MachinePipelinePassesTest.cpp | 629 ++++ backend/unittests/Target/CMakeLists.txt | 7 + backend/unittests/Target/RISCVCodeGenTest.cpp | 1108 +++++++ 11 files changed, 5793 insertions(+) create mode 100644 backend/include/rcc/CodeGen/MachinePipelinePasses.h create mode 100644 backend/include/rcc/CodeGen/TargetCodeGen.h create mode 100644 backend/include/rcc/Target/RISCV/RISCVCodeGen.h create mode 100644 backend/lib/CodeGen/MachinePipelinePasses.cpp create mode 100644 backend/lib/Target/RISCV/RISCVCodeGen.cpp create mode 100644 backend/unittests/CodeGen/MachinePipelinePassesTest.cpp create mode 100644 backend/unittests/Target/RISCVCodeGenTest.cpp diff --git a/backend/include/rcc/CodeGen/MachinePipelinePasses.h b/backend/include/rcc/CodeGen/MachinePipelinePasses.h new file mode 100644 index 0000000..c5bdac0 --- /dev/null +++ b/backend/include/rcc/CodeGen/MachinePipelinePasses.h @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RCC_CODEGEN_MACHINEPIPELINEPASSES_H +#define RCC_CODEGEN_MACHINEPIPELINEPASSES_H + +#include "rcc/CodeGen/MachinePass.h" +#include "rcc/CodeGen/TargetCodeGen.h" + +namespace rcc { + +class LegalizerPass final : public MachineFunctionPass { +public: + explicit LegalizerPass(const TargetLegalizerInfo *Info) : Info(Info) {} + explicit LegalizerPass(const TargetLegalizerInfo &Info) : Info(&Info) {} + + [[nodiscard]] const MachinePassContract &contract() const noexcept override; + [[nodiscard]] Expected run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) override; + +private: + [[nodiscard]] Expected runImplementation(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses); + + const TargetLegalizerInfo *Info; +}; + +class RegisterBankSelectPass final : public MachineFunctionPass { +public: + explicit RegisterBankSelectPass(const TargetRegisterBankInfo *Info) : Info(Info) {} + explicit RegisterBankSelectPass(const TargetRegisterBankInfo &Info) : Info(&Info) {} + + [[nodiscard]] const MachinePassContract &contract() const noexcept override; + [[nodiscard]] Expected run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) override; + +private: + const TargetRegisterBankInfo *Info; +}; + +class InstructionSelectPass final : public MachineFunctionPass { +public: + explicit InstructionSelectPass(const TargetInstructionSelector *Info) : Info(Info) {} + explicit InstructionSelectPass(const TargetInstructionSelector &Info) : Info(&Info) {} + + [[nodiscard]] const MachinePassContract &contract() const noexcept override; + [[nodiscard]] Expected run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) override; + +private: + [[nodiscard]] Expected runImplementation(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses); + + const TargetInstructionSelector *Info; +}; + +} // namespace rcc + +#endif // RCC_CODEGEN_MACHINEPIPELINEPASSES_H diff --git a/backend/include/rcc/CodeGen/TargetCodeGen.h b/backend/include/rcc/CodeGen/TargetCodeGen.h new file mode 100644 index 0000000..b1be1f6 --- /dev/null +++ b/backend/include/rcc/CodeGen/TargetCodeGen.h @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RCC_CODEGEN_TARGETCODEGEN_H +#define RCC_CODEGEN_TARGETCODEGEN_H + +#include "rcc/CodeGen/MachineIRTypes.h" +#include "rcc/CodeGen/Register.h" +#include "rcc/Support/Error.h" + +#include +#include + +namespace rcc { + +class MachineFunction; +class MachineInstr; +class MachineRegisterInfo; +class TargetDescription; +struct VirtualRegisterInfo; + +enum class LegalizationAction : std::uint8_t { + Legal, + Unsupported, + Custom, + Lower, + Libcall, + WidenScalar, + NarrowScalar, + MoreElements, + FewerElements, + Bitcast, + ChangeElementType, + ChangeAddressSpace, +}; + +class TargetLegalizerInfo { +public: + virtual ~TargetLegalizerInfo() = default; + + [[nodiscard]] virtual Expected + classify(const MachineInstr &Instruction, const MachineFunction &Function, + const TargetDescription &Target) const = 0; + + [[nodiscard]] virtual Expected lower(MachineInstr &Instruction, MachineFunction &Function, + const TargetDescription &Target, + LegalizationAction Action) const = 0; + + // This hook performs read-only target checks before a lowering mutation is committed. + [[nodiscard]] virtual Expected validateLowering(const MachineInstr &, + const MachineFunction &, + const TargetDescription &, + LegalizationAction) const { + return {}; + } +}; + +class TargetRegisterBankInfo { +public: + virtual ~TargetRegisterBankInfo() = default; + + [[nodiscard]] virtual Expected + selectBank(const MachineFunction &Function, Register Reg, const VirtualRegisterInfo &Info, + const MachineInstr *Context, const TargetDescription &Target) const = 0; + + [[nodiscard]] virtual Expected + constrainInstruction(MachineInstr &Instruction, MachineFunction &Function, + const TargetDescription &Target) const = 0; + + // This hook must not mutate MachineIR. It allows the pass to reject a bad mapping + // before publishing any bank or class assignment. + [[nodiscard]] virtual Expected validateInstruction(const MachineInstr &, + const MachineFunction &, + const TargetDescription &) const { + return {}; + } +}; + +class TargetInstructionSelector { +public: + virtual ~TargetInstructionSelector() = default; + + [[nodiscard]] virtual Expected select(MachineInstr &Instruction, MachineFunction &Function, + const TargetDescription &Target) const = 0; + + // Selection matching is read-only and must complete before select() mutates MachineIR. + [[nodiscard]] virtual Expected canSelect(const MachineInstr &, const MachineFunction &, + const TargetDescription &) const { + return {}; + } +}; + +} // namespace rcc + +#endif // RCC_CODEGEN_TARGETCODEGEN_H diff --git a/backend/include/rcc/Target/RISCV/RISCVCodeGen.h b/backend/include/rcc/Target/RISCV/RISCVCodeGen.h new file mode 100644 index 0000000..b06f20d --- /dev/null +++ b/backend/include/rcc/Target/RISCV/RISCVCodeGen.h @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef RCC_TARGET_RISCV_RISCVCODEGEN_H +#define RCC_TARGET_RISCV_RISCVCODEGEN_H + +#include "rcc/CodeGen/TargetCodeGen.h" + +namespace rcc::riscv { + +[[nodiscard]] const TargetLegalizerInfo &legalizerInfo() noexcept; +[[nodiscard]] const TargetRegisterBankInfo ®isterBankInfo() noexcept; +[[nodiscard]] const TargetInstructionSelector &instructionSelector() noexcept; + +} // namespace rcc::riscv + +#endif // RCC_TARGET_RISCV_RISCVCODEGEN_H diff --git a/backend/lib/CodeGen/CMakeLists.txt b/backend/lib/CodeGen/CMakeLists.txt index 5a12055..f2c81f2 100644 --- a/backend/lib/CodeGen/CMakeLists.txt +++ b/backend/lib/CodeGen/CMakeLists.txt @@ -27,6 +27,7 @@ set( MachineMetadata.cpp MachineOperand.cpp MachinePass.cpp + MachinePipelinePasses.cpp MachineRegisterInfo.cpp MachineVerifier.cpp MachineFunctionProperties.cpp @@ -67,6 +68,8 @@ set( "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineMetadata.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineOperand.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachinePass.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachinePipelinePasses.h" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/TargetCodeGen.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineRegisterInfo.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineVerifier.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/CodeGen/MachineFunctionProperties.h" @@ -136,6 +139,7 @@ rcc_backend_register_format_files( "${CMAKE_CURRENT_SOURCE_DIR}/MachineMetadata.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineOperand.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachinePass.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachinePipelinePasses.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineRegisterInfo.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineVerifier.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineFunctionProperties.cpp" @@ -176,6 +180,7 @@ rcc_backend_register_tidy_files( "${CMAKE_CURRENT_SOURCE_DIR}/MachineMetadata.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineOperand.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachinePass.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachinePipelinePasses.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineRegisterInfo.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineVerifier.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineFunctionProperties.cpp" diff --git a/backend/lib/CodeGen/MachinePipelinePasses.cpp b/backend/lib/CodeGen/MachinePipelinePasses.cpp new file mode 100644 index 0000000..c3ee653 --- /dev/null +++ b/backend/lib/CodeGen/MachinePipelinePasses.cpp @@ -0,0 +1,614 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachinePipelinePasses.h" + +#include "rcc/CodeGen/GenericOpcodes.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineFunctionCloner.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/TargetDescription.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace rcc { +namespace { + +[[nodiscard]] const MachinePassContract &legalizerContract() { + static const MachinePassContract Contract{ + .Name = "Legalizer", + .Requires = {MachineFunctionProperty::IsSSA, MachineFunctionProperty::TracksLiveness}, + .Sets = {MachineFunctionProperty::Legalized}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::Legalized, + }; + return Contract; +} + +[[nodiscard]] const MachinePassContract &bankSelectContract() { + static const MachinePassContract Contract{ + .Name = "RegisterBankSelect", + .Requires = {MachineFunctionProperty::Legalized}, + .Sets = {MachineFunctionProperty::RegBankSelected}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::RegBankSelected, + }; + return Contract; +} + +[[nodiscard]] const MachinePassContract &instructionSelectContract() { + static const MachinePassContract Contract{ + .Name = "InstructionSelect", + .Requires = {MachineFunctionProperty::Legalized, MachineFunctionProperty::RegBankSelected}, + .Sets = {MachineFunctionProperty::Selected}, + .Clears = {}, + .RequiredAnalyses = {}, + .VerificationStage = PipelineStage::Selected, + }; + return Contract; +} + +struct OpcodeState final { + MachineInstrStableID Instruction; + Opcode OpcodeValue; + std::uint64_t OperandFingerprint = 0; + + friend bool operator==(const OpcodeState &Left, const OpcodeState &Right) noexcept = default; +}; + +struct OpcodeStateHash final { + [[nodiscard]] std::size_t operator()(const OpcodeState &State) const noexcept { + return std::hash{}(State.Instruction.value()) ^ + (std::hash{}(State.OpcodeValue.value()) << 1U) ^ + (std::hash{}(State.OperandFingerprint) << 2U); + } +}; + +void combineFingerprint(std::uint64_t &State, std::uint64_t Value) noexcept { + State ^= Value + 0x9e3779b97f4a7c15ULL + (State << 6U) + (State >> 2U); +} + +[[nodiscard]] std::uint64_t +registerConfigurationFingerprint(const MachineRegisterInfo &RegInfo) noexcept { + std::uint64_t Result = 0xcbf29ce484222325ULL; + for (const VirtualRegisterInfo &Info : RegInfo.virtualRegisters()) { + combineFingerprint(Result, Info.IsTombstone ? 1U : 0U); + combineFingerprint(Result, Info.Bank ? Info.Bank->value() : 0U); + combineFingerprint(Result, Info.RegClass ? Info.RegClass->value() : 0U); + } + return Result; +} + +[[nodiscard]] std::uint64_t operandFingerprint(const MachineInstr &Instruction) noexcept { + std::uint64_t Result = 0xcbf29ce484222325ULL; + combineFingerprint(Result, Instruction.flags().rawValue()); + combineFingerprint(Result, Instruction.operandCount()); + for (const MachineOperand &Operand : Instruction.operands()) { + combineFingerprint(Result, static_cast(Operand.kind())); + combineFingerprint(Result, Operand.targetFlags()); + combineFingerprint(Result, Operand.registerFlags().rawValue()); + combineFingerprint(Result, Operand.subReg().value()); + if (const Register *Reg = Operand.getIf()) + combineFingerprint(Result, Reg->opaqueValue()); + else if (const std::int64_t *Immediate = Operand.getIf()) + combineFingerprint(Result, static_cast(*Immediate)); + else if (const auto *Integer = Operand.getIf()) + combineFingerprint(Result, Integer->Value.bitWidth()); + else if (const auto *Floating = Operand.getIf()) + combineFingerprint(Result, static_cast(Floating->Bits.semantics())); + else if (const auto *Block = Operand.getIf()) + combineFingerprint(Result, Block->Block == nullptr ? 0 : Block->Block->stableID().value()); + } + return Result; +} + +[[nodiscard]] OpcodeState stateOf(const MachineInstr &Instruction) noexcept { + return {Instruction.stableID(), Instruction.opcode(), operandFingerprint(Instruction)}; +} + +[[nodiscard]] std::vector layoutInstructions(MachineFunction &Function) { + std::vector Result; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) + for (MachineInstr *Instruction = Block->firstInstruction(); Instruction != nullptr; + Instruction = Instruction->next()) + Result.push_back(Instruction); + return Result; +} + +[[nodiscard]] std::vector stableCFGPostOrder(MachineFunction &Function) { + std::vector Roots; + for (MachineBasicBlock *Block = Function.firstBlock(); Block != nullptr; Block = Block->next()) + Roots.push_back(Block); + std::ranges::sort(Roots, {}, + [](const MachineBasicBlock *Block) { return Block->stableID().value(); }); + if (MachineBasicBlock *Entry = Function.firstBlock(); Entry != nullptr) { + const auto Position = std::ranges::find(Roots, Entry); + if (Position != Roots.end()) + std::rotate(Roots.begin(), Position, std::next(Position)); + } + + struct TraversalFrame final { + MachineBasicBlock *Block = nullptr; + std::vector Successors; + std::size_t NextSuccessor = 0; + }; + const auto MakeFrame = [](MachineBasicBlock &Block) { + TraversalFrame Frame; + Frame.Block = &Block; + for (const MachineSuccessor &Successor : Block.successors()) { + if (Successor.Block != nullptr) + Frame.Successors.push_back(Successor.Block); + } + std::ranges::sort(Frame.Successors, {}, [](const MachineBasicBlock *Successor) { + return Successor->stableID().value(); + }); + return Frame; + }; + + std::vector Result; + std::unordered_set Visited; + std::vector Stack; + for (MachineBasicBlock *Root : Roots) { + if (!Visited.insert(Root).second) + continue; + Stack.push_back(MakeFrame(*Root)); + while (!Stack.empty()) { + TraversalFrame &Frame = Stack.back(); + if (Frame.NextSuccessor < Frame.Successors.size()) { + MachineBasicBlock *Successor = Frame.Successors[Frame.NextSuccessor++]; + if (Visited.insert(Successor).second) + Stack.push_back(MakeFrame(*Successor)); + continue; + } + Result.push_back(Frame.Block); + Stack.pop_back(); + } + } + return Result; +} + +[[nodiscard]] std::vector bottomUpInstructions(MachineFunction &Function) { + std::vector Result; + for (MachineBasicBlock *Block : stableCFGPostOrder(Function)) { + for (MachineInstr *Instruction = Block->lastInstruction(); Instruction != nullptr; + Instruction = Instruction->previous()) + Result.push_back(Instruction); + } + return Result; +} + +[[nodiscard]] Expected requireProperties(const MachineFunction &Function, + const MachinePassContract &Contract) { + const MachineFunctionProperties Missing = Contract.Requires.missingFrom(Function.properties()); + if (!Missing.empty()) { + return makeUnexpected(ErrorCode::VerificationError, + "pass '" + Contract.Name + + "' requires properties: " + Missing.toString()); + } + return {}; +} + +[[nodiscard]] Expected restoreRegisterConfiguration( + MachineRegisterInfo &RegInfo, + const std::vector, + std::optional>>> &Snapshot) { + for (const auto &[Reg, Configuration] : Snapshot) { + auto ClearBank = RegInfo.clearRegisterBank(Reg); + if (!ClearBank) + return std::unexpected(ClearBank.error()); + if (Configuration.first) { + auto SetBank = RegInfo.setRegisterBank(Reg, *Configuration.first); + if (!SetBank) + return std::unexpected(SetBank.error()); + } + auto ClearClass = RegInfo.clearRegisterClass(Reg); + if (!ClearClass) + return std::unexpected(ClearClass.error()); + if (Configuration.second) { + auto SetClass = RegInfo.setRegisterClass(Reg, *Configuration.second); + if (!SetClass) + return std::unexpected(SetClass.error()); + } + } + return {}; +} + +[[nodiscard]] Error passFailure(std::string Message, + ErrorCode Code = ErrorCode::VerificationError) { + return Error(Code, std::move(Message)); +} + +[[nodiscard]] Error prefixedError(const Error &Source, std::string Prefix) { + Prefix.append(Source.message()); + return Error(Source.code(), std::move(Prefix)); +} + +} // namespace + +const MachinePassContract &LegalizerPass::contract() const noexcept { + return legalizerContract(); +} + +Expected LegalizerPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) { + if (Info == nullptr) + return std::unexpected( + passFailure("legalizer has no target behavior interface", ErrorCode::InvalidArgument)); + auto Required = requireProperties(Function, contract()); + if (!Required) + return std::unexpected(Required.error()); + + auto Trial = cloneMachineFunction( + Function, Target, + MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); + if (!Trial) + return std::unexpected(prefixedError(Trial.error(), "legalizer transaction setup failed: ")); + MachineAnalysisManager TrialAnalyses; + auto TrialResult = runImplementation(*Trial->ClonedFunction, Target, TrialAnalyses); + if (!TrialResult) + return std::unexpected(TrialResult.error()); + auto Result = runImplementation(Function, Target, Analyses); + if (!Result) { + return std::unexpected(prefixedError( + Result.error(), "legalizer diverged after a successful transaction preflight: ")); + } + return Result; +} + +Expected LegalizerPass::runImplementation(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &) { + + MachinePassResult Result; + std::unordered_set SeenStates; + + std::vector Worklist = layoutInstructions(Function); + std::unordered_set DiscoveredInstructions; + for (const MachineInstr *Instruction : Worklist) + DiscoveredInstructions.insert(Instruction->stableID()); + for (std::size_t Index = 0; Index < Worklist.size(); ++Index) { + MachineInstr *Instruction = Worklist[Index]; + if (Instruction == nullptr || Instruction->parent() == nullptr) + continue; + + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor == nullptr) + return std::unexpected( + passFailure("legalizer encountered an unknown instruction descriptor")); + if (!Descriptor->Flags.has(InstructionFlag::Generic)) + continue; + + auto Validation = Target.validateInstruction(*Instruction, Function.registerInfo()); + if (!Validation) + return std::unexpected(prefixedError(Validation.error(), "legalizer input is invalid: ")); + + auto Action = Info->classify(*Instruction, Function, Target); + if (!Action) + return std::unexpected(Action.error()); + if (*Action == LegalizationAction::Legal) + continue; + if (*Action == LegalizationAction::Unsupported) { + return std::unexpected( + passFailure("target legalizer rejected generic instruction '" + Descriptor->Name + "'", + ErrorCode::Unsupported)); + } + + const OpcodeState Before = stateOf(*Instruction); + if (!SeenStates.insert(Before).second) + return std::unexpected( + passFailure("target legalizer entered a repeated state for '" + Descriptor->Name + "'", + ErrorCode::InternalError)); + + auto LoweringValidation = Info->validateLowering(*Instruction, Function, Target, *Action); + if (!LoweringValidation) + return std::unexpected( + prefixedError(LoweringValidation.error(), "legalizer lowering precondition failed: ")); + auto Lowered = Info->lower(*Instruction, Function, Target, *Action); + if (!Lowered) + return std::unexpected(Lowered.error()); + if (Instruction->parent() == nullptr) + return std::unexpected(passFailure("legalizer detached the instruction it was lowering")); + const OpcodeState After = stateOf(*Instruction); + if (After == Before) + return std::unexpected( + passFailure("target legalizer made no observable progress for '" + Descriptor->Name + "'", + ErrorCode::InternalError)); + Result.Changed = true; + Result.Changes.add(MachinePassChange::Instructions); + + const InstructionDescriptor *LoweredDescriptor = + Target.instructionDescriptor(Instruction->opcode()); + if (LoweredDescriptor == nullptr) + return std::unexpected(passFailure("legalizer produced an unknown instruction descriptor")); + if (LoweredDescriptor->Flags.has(InstructionFlag::Generic)) + Worklist.push_back(Instruction); + const std::vector Current = layoutInstructions(Function); + for (MachineInstr *Candidate : Current) { + if (DiscoveredInstructions.insert(Candidate->stableID()).second) + Worklist.push_back(Candidate); + } + } + + for (MachineInstr *Instruction : layoutInstructions(Function)) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor == nullptr) + return std::unexpected(passFailure("legalizer produced an unknown instruction descriptor")); + if (!Descriptor->Flags.has(InstructionFlag::Generic)) + continue; + auto Action = Info->classify(*Instruction, Function, Target); + if (!Action) + return std::unexpected(Action.error()); + if (*Action != LegalizationAction::Legal) + return std::unexpected( + passFailure("legalizer reached a fixed point with an illegal generic instruction '" + + Descriptor->Name + "'", + *Action == LegalizationAction::Unsupported ? ErrorCode::Unsupported + : ErrorCode::InternalError)); + auto FinalValidation = Target.validateInstruction(*Instruction, Function.registerInfo()); + if (!FinalValidation) + return std::unexpected( + prefixedError(FinalValidation.error(), "legalizer produced invalid MachineIR: ")); + } + Result.Preserved = Result.Changed ? PreservedAnalyses::none() : PreservedAnalyses::all(); + return Result; +} + +const MachinePassContract &RegisterBankSelectPass::contract() const noexcept { + return bankSelectContract(); +} + +Expected RegisterBankSelectPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &) { + if (Info == nullptr) + return std::unexpected(passFailure("register-bank select has no target behavior interface", + ErrorCode::InvalidArgument)); + auto Required = requireProperties(Function, contract()); + if (!Required) + return std::unexpected(Required.error()); + + MachinePassResult Result; + MachineRegisterInfo &RegInfo = Function.registerInfo(); + const std::uint64_t RegisterConfigurationBefore = registerConfigurationFingerprint(RegInfo); + using RegisterConfiguration = + std::pair, std::optional>; + std::vector> Snapshot; + std::vector> PlannedBanks; + for (std::size_t Index = 0; Index < RegInfo.virtualRegisters().size(); ++Index) { + auto Reg = Register::makeVirtual(Index); + if (!Reg) + return std::unexpected(Reg.error()); + const VirtualRegisterInfo &StoredInfo = RegInfo.virtualRegisters()[Index]; + if (StoredInfo.IsTombstone) + continue; + Snapshot.emplace_back(*Reg, RegisterConfiguration{StoredInfo.Bank, StoredInfo.RegClass}); + auto InfoResult = RegInfo.virtualRegisterInfo(*Reg); + if (!InfoResult) + return std::unexpected(InfoResult.error()); + const VirtualRegisterInfo &InfoValue = InfoResult->get(); + const MachineInstr *Context = nullptr; + for (const MachineOperand &Operand : RegInfo.regOperands(*Reg)) { + Context = Operand.parent(); + if (Context != nullptr) + break; + } + auto Bank = Info->selectBank(Function, *Reg, InfoValue, Context, Target); + if (!Bank) + return std::unexpected(Bank.error()); + if (Target.registers().registerBank(*Bank) == nullptr) + return std::unexpected(passFailure("target register-bank selector returned an unknown bank")); + if (InfoValue.Bank && *InfoValue.Bank != *Bank) + return std::unexpected( + passFailure("target register-bank selector conflicts with an existing bank")); + if (InfoValue.RegClass && !Target.registers().bankCoversClass(*Bank, *InfoValue.RegClass)) + return std::unexpected( + passFailure("existing register class is not covered by selected bank")); + PlannedBanks.emplace_back(*Reg, *Bank); + } + + for (MachineInstr *Instruction : layoutInstructions(Function)) { + auto Validation = Info->validateInstruction(*Instruction, Function, Target); + if (!Validation) + return std::unexpected( + prefixedError(Validation.error(), "register-bank mapping precondition failed: ")); + } + + for (const auto &[Reg, Bank] : PlannedBanks) { + const auto InfoResult = RegInfo.virtualRegisterInfo(Reg); + if (!InfoResult) + return std::unexpected(InfoResult.error()); + if (!InfoResult->get().Bank) { + auto SetBank = RegInfo.setRegisterBank(Reg, Bank); + if (!SetBank) + return std::unexpected(SetBank.error()); + Result.Changed = true; + Result.Changes.add(MachinePassChange::Registers); + } + } + + const auto Restore = [&]() -> Expected { + return restoreRegisterConfiguration(RegInfo, Snapshot); + }; + for (MachineInstr *Instruction : layoutInstructions(Function)) { + auto Constraint = Info->constrainInstruction(*Instruction, Function, Target); + if (!Constraint) { + auto Restored = Restore(); + if (!Restored) + return std::unexpected(prefixedError(Restored.error(), "register-bank rollback failed: ")); + return std::unexpected(Constraint.error()); + } + auto Validation = Target.validateInstruction(*Instruction, Function.registerInfo()); + if (!Validation) { + auto Restored = Restore(); + if (!Restored) + return std::unexpected(prefixedError(Restored.error(), "register-bank rollback failed: ")); + return std::unexpected(prefixedError(Validation.error(), + "register-bank selection produced invalid MachineIR: ")); + } + } + auto RegisterValidation = Target.validateRegisterInfo(RegInfo); + if (!RegisterValidation) { + auto Restored = Restore(); + if (!Restored) + return std::unexpected(prefixedError(Restored.error(), "register-bank rollback failed: ")); + return std::unexpected(prefixedError(RegisterValidation.error(), + "register-bank selection produced invalid registers: ")); + } + + if (registerConfigurationFingerprint(RegInfo) != RegisterConfigurationBefore) { + Result.Changed = true; + Result.Changes.add(MachinePassChange::Registers); + } + + Result.Preserved = Result.Changed ? PreservedAnalyses::none() : PreservedAnalyses::all(); + return Result; +} + +const MachinePassContract &InstructionSelectPass::contract() const noexcept { + return instructionSelectContract(); +} + +Expected InstructionSelectPass::run(MachineFunction &Function, + const TargetDescription &Target, + MachineAnalysisManager &Analyses) { + if (Info == nullptr) + return std::unexpected(passFailure("instruction select has no target behavior interface", + ErrorCode::InvalidArgument)); + auto Required = requireProperties(Function, contract()); + if (!Required) + return std::unexpected(Required.error()); + + auto Trial = cloneMachineFunction( + Function, Target, + MachineFunctionCloneOptions{.NewStableID = Function.stableID(), .NewName = std::nullopt}); + if (!Trial) { + return std::unexpected( + prefixedError(Trial.error(), "instruction-selection transaction setup failed: ")); + } + MachineAnalysisManager TrialAnalyses; + auto TrialResult = runImplementation(*Trial->ClonedFunction, Target, TrialAnalyses); + if (!TrialResult) + return std::unexpected(TrialResult.error()); + auto Result = runImplementation(Function, Target, Analyses); + if (!Result) { + return std::unexpected( + prefixedError(Result.error(), + "instruction selection diverged after a successful transaction preflight: ")); + } + return Result; +} + +Expected +InstructionSelectPass::runImplementation(MachineFunction &Function, const TargetDescription &Target, + MachineAnalysisManager &) { + + MachinePassResult Result; + std::vector Worklist; + std::unordered_set DiscoveredInstructions; + std::unordered_set SeenStates; + for (MachineInstr *Instruction : bottomUpInstructions(Function)) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor == nullptr) + return std::unexpected( + passFailure("instruction selector encountered an unknown instruction descriptor")); + auto Validation = Target.validateInstruction(*Instruction, Function.registerInfo()); + if (!Validation) + return std::unexpected( + prefixedError(Validation.error(), "instruction selector input is invalid: ")); + if (!Descriptor->Flags.has(InstructionFlag::Generic)) + continue; + auto Match = Info->canSelect(*Instruction, Function, Target); + if (!Match) + return std::unexpected(Match.error()); + Worklist.push_back(Instruction); + DiscoveredInstructions.insert(Instruction->stableID()); + } + + for (std::size_t Index = 0; Index < Worklist.size(); ++Index) { + MachineInstr *Instruction = Worklist[Index]; + if (Instruction->parent() == nullptr) + return std::unexpected(passFailure("instruction selector lost its input instruction")); + const InstructionDescriptor *CurrentDescriptor = + Target.instructionDescriptor(Instruction->opcode()); + if (CurrentDescriptor == nullptr) + return std::unexpected( + passFailure("instruction selector encountered an unknown instruction descriptor")); + if (!CurrentDescriptor->Flags.has(InstructionFlag::Generic)) + continue; + const OpcodeState Before = stateOf(*Instruction); + if (!SeenStates.insert(Before).second) { + return std::unexpected(passFailure("instruction selector entered a repeated state for '" + + CurrentDescriptor->Name + "'", + ErrorCode::InternalError)); + } + auto Match = Info->canSelect(*Instruction, Function, Target); + if (!Match) + return std::unexpected(Match.error()); + auto Selection = Info->select(*Instruction, Function, Target); + if (!Selection) + return std::unexpected(Selection.error()); + const InstructionDescriptor *Selected = Target.instructionDescriptor(Instruction->opcode()); + if (Selected == nullptr) + return std::unexpected( + passFailure("instruction selector produced an unknown instruction descriptor")); + auto Validation = Target.validateInstruction(*Instruction, Function.registerInfo()); + if (!Validation) + return std::unexpected( + prefixedError(Validation.error(), "instruction selector produced invalid MachineIR: ")); + const OpcodeState After = stateOf(*Instruction); + if (After == Before) + return std::unexpected( + passFailure("instruction selector reported success without observable progress", + ErrorCode::InternalError)); + Result.Changed = true; + Result.Changes.add(MachinePassChange::Instructions); + if (Selected->Flags.has(InstructionFlag::Generic)) + Worklist.push_back(Instruction); + for (MachineInstr *Candidate : bottomUpInstructions(Function)) { + const InstructionDescriptor *CandidateDescriptor = + Target.instructionDescriptor(Candidate->opcode()); + if (CandidateDescriptor == nullptr) + return std::unexpected( + passFailure("instruction selector produced an unknown instruction descriptor")); + if (!CandidateDescriptor->Flags.has(InstructionFlag::Generic) || + !DiscoveredInstructions.insert(Candidate->stableID()).second) { + continue; + } + auto CandidateMatch = Info->canSelect(*Candidate, Function, Target); + if (!CandidateMatch) + return std::unexpected(CandidateMatch.error()); + Worklist.push_back(Candidate); + } + } + for (MachineInstr *Instruction : layoutInstructions(Function)) { + const InstructionDescriptor *Descriptor = Target.instructionDescriptor(Instruction->opcode()); + if (Descriptor == nullptr) + return std::unexpected( + passFailure("instruction selector produced an unknown instruction descriptor")); + auto Validation = Target.validateInstruction(*Instruction, Function.registerInfo()); + if (!Validation) + return std::unexpected( + prefixedError(Validation.error(), "instruction selector produced invalid MachineIR: ")); + if (Descriptor->Flags.has(InstructionFlag::Generic)) + return std::unexpected( + passFailure("instruction selector left a generic opcode in the function")); + } + auto RegisterValidation = Target.validateRegisterInfo(Function.registerInfo()); + if (!RegisterValidation) + return std::unexpected(prefixedError(RegisterValidation.error(), + "instruction selection produced invalid registers: ")); + Result.Preserved = Result.Changed ? PreservedAnalyses::none() : PreservedAnalyses::all(); + return Result; +} + +} // namespace rcc diff --git a/backend/lib/Target/RISCV/CMakeLists.txt b/backend/lib/Target/RISCV/CMakeLists.txt index ff3f334..4728117 100644 --- a/backend/lib/Target/RISCV/CMakeLists.txt +++ b/backend/lib/Target/RISCV/CMakeLists.txt @@ -47,6 +47,7 @@ set_source_files_properties( add_library( RCCBackendRISCV + RISCVCodeGen.cpp RISCVCallLowering.cpp RISCVTarget.cpp "${RCCBackendRISCVGeneratedRegisterHeader}" @@ -90,12 +91,15 @@ install( ) rcc_backend_register_format_files( + "${CMAKE_CURRENT_SOURCE_DIR}/RISCVCodeGen.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVCallLowering.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVTarget.cpp" + "${RCC_BACKEND_SOURCE_DIR}/include/rcc/Target/RISCV/RISCVCodeGen.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/Target/RISCV/RISCVCallLowering.h" "${RCC_BACKEND_SOURCE_DIR}/include/rcc/Target/RISCV/RISCVTarget.h" ) rcc_backend_register_tidy_files( + "${CMAKE_CURRENT_SOURCE_DIR}/RISCVCodeGen.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVCallLowering.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVTarget.cpp" ) diff --git a/backend/lib/Target/RISCV/RISCVCodeGen.cpp b/backend/lib/Target/RISCV/RISCVCodeGen.cpp new file mode 100644 index 0000000..631ac33 --- /dev/null +++ b/backend/lib/Target/RISCV/RISCVCodeGen.cpp @@ -0,0 +1,3239 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/Target/RISCV/RISCVCodeGen.h" + +#include "rcc/CodeGen/GenericOpcodes.h" +#include "rcc/CodeGen/LowLevelType.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineIRBuilder.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineMemOperand.h" +#include "rcc/CodeGen/MachineOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/CodeGen/TargetDescription.h" +#include "rcc/Target/RISCV/RISCVTarget.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace rcc::riscv { +namespace { + +[[nodiscard]] Opcode targetOpcode(InstructionOpcode Value) noexcept { + return Opcode::fromValue(static_cast(Value)); +} + +[[nodiscard]] std::optional registerType(const MachineFunction &Function, + const MachineOperand &Operand) { + const Register *Reg = Operand.getIf(); + if (Reg == nullptr || !Reg->isVirtual()) + return std::nullopt; + auto Info = Function.registerInfo().virtualRegisterInfo(*Reg); + if (!Info) + return std::nullopt; + return Info->get().Type; +} + +[[nodiscard]] std::optional resultType(const MachineFunction &Function, + const MachineInstr &Instruction) { + const MachineOperand *Result = Instruction.operandAt(0); + return Result == nullptr ? std::nullopt : registerType(Function, *Result); +} + +[[nodiscard]] std::optional operandType(const MachineFunction &Function, + const MachineInstr &Instruction, std::size_t Index) { + const MachineOperand *Operand = Instruction.operandAt(Index); + return Operand == nullptr ? std::nullopt : registerType(Function, *Operand); +} + +[[nodiscard]] bool isRVFloatType(LLT Type) noexcept { + if (!Type.isFloat()) + return false; + const std::optional Semantics = Type.floatSemantics(); + return Semantics && + (*Semantics == FloatSemantics::IEEESingle || *Semantics == FloatSemantics::IEEEDouble); +} + +[[nodiscard]] bool isSupportedIntegerType(LLT Type) noexcept { + return (Type.isInteger() && Type.scalarBitWidth() != 0 && Type.scalarBitWidth() <= 64) || + (Type.isPointer() && Type.scalarBitWidth() == 64); +} + +[[nodiscard]] bool isSupportedIntegerScalar(LLT Type) noexcept { + return Type.isInteger() && Type.scalarBitWidth() != 0 && Type.scalarBitWidth() <= 64; +} + +[[nodiscard]] bool isNativeIntegerScalar(LLT Type) noexcept { + return Type.isInteger() && (Type.scalarBitWidth() == 32 || Type.scalarBitWidth() == 64); +} + +[[nodiscard]] bool isScalarGPRType(LLT Type) noexcept { + return (Type.isInteger() && Type.scalarBitWidth() != 0 && Type.scalarBitWidth() <= 64) || + (Type.isPointer() && Type.scalarBitWidth() == 64); +} + +[[nodiscard]] bool isI1Type(LLT Type) noexcept { + return Type.isInteger() && Type.scalarBitWidth() == 1; +} + +[[nodiscard]] bool isScalarIntegerExtension(LLT Destination, LLT Source) noexcept { + return Destination.isInteger() && Source.isInteger() && Destination.scalarBitWidth() != 0 && + Source.scalarBitWidth() != 0 && Source.scalarBitWidth() < Destination.scalarBitWidth() && + Destination.scalarBitWidth() <= 64; +} + +[[nodiscard]] bool fitsSigned(std::int64_t Value, std::uint32_t Bits) noexcept { + const std::int64_t Minimum = -(std::int64_t{1} << (Bits - 1U)); + const std::int64_t Maximum = (std::int64_t{1} << (Bits - 1U)) - 1; + return Value >= Minimum && Value <= Maximum; +} + +[[nodiscard]] bool isSupportedNonAtomicMemoryAccess(const MachineInstr &Instruction) noexcept { + if (Instruction.memOperands().size() != 1) + return false; + const MachineMemOperand &Memory = *Instruction.memOperands().front(); + const MachineMemOperandDesc &Description = Memory.description(); + return !Memory.isAtomic() && Description.AddressSpace == 0 && Description.Size.isKnown() && + !Description.Size.isScalable() && Description.Size.minimumBytes() != 0; +} + +[[nodiscard]] bool isSupportedAtomicMemoryAccess(const MachineInstr &Instruction, + AtomicOperation Operation, + std::uint64_t Bytes) noexcept { + if (Instruction.memOperands().size() != 1 || (Bytes != 4 && Bytes != 8)) + return false; + const MachineMemOperand &Memory = *Instruction.memOperands().front(); + const MachineMemOperandDesc &Description = Memory.description(); + return Description.AtomicKind == Operation && Description.AddressSpace == 0 && + Description.Size.isKnown() && !Description.Size.isScalable() && + Description.Size.minimumBytes() == Bytes && Description.Align.bytes() >= Bytes && + Description.SyncScope.value() == 0; +} + +[[nodiscard]] std::optional +memoryAccessBytes(const MachineInstr &Instruction) noexcept { + if (Instruction.memOperands().size() != 1) + return std::nullopt; + const MachineMemOperandDesc &Description = Instruction.memOperands().front()->description(); + if (!Description.Size.isKnown() || Description.Size.isScalable()) + return std::nullopt; + return Description.Size.minimumBytes(); +} + +[[nodiscard]] bool memoryMatchesType(const MachineInstr &Instruction, LLT Type) noexcept { + const auto Bytes = memoryAccessBytes(Instruction); + if (!Bytes || (!isSupportedIntegerType(Type) && !isRVFloatType(Type))) + return false; + if (isRVFloatType(Type)) + return Type.scalarBitWidth() == 32 ? *Bytes == 4 : *Bytes == 8; + const std::uint64_t ValueBytes = (Type.scalarBitWidth() + 7U) / 8U; + return (*Bytes == 1 || *Bytes == 2 || *Bytes == 4 || *Bytes == 8) && *Bytes <= ValueBytes; +} + +[[nodiscard]] Expected requiredRegisterClass(const MachineFunction &Function, + Register Reg, + const TargetDescription &Target) { + if (!Reg.isVirtual()) + return makeUnexpected(ErrorCode::InvalidArgument, + "RISC-V register-class selection requires a virtual register"); + auto Info = Function.registerInfo().virtualRegisterInfo(Reg); + if (!Info) + return std::unexpected(Info.error()); + if (!Info->get().Type) { + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V selection requires a type for every virtual register"); + } + const VirtualRegisterInfo &RegisterInfo = Info->get(); + const LLT Type = RegisterInfo.Type.value_or(LLT{}); + RegisterClassID RequiredClass; + if (isScalarGPRType(Type)) { + RequiredClass = RegisterClassID::fromValue(static_cast(RegisterClass::GPR)); + } else if (Type.isFloat() && Type.scalarBitWidth() == 32) { + RequiredClass = RegisterClassID::fromValue(static_cast(RegisterClass::FPR32)); + } else if (Type.isFloat() && Type.scalarBitWidth() == 64) { + RequiredClass = RegisterClassID::fromValue(static_cast(RegisterClass::FPR64)); + } else { + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V RV64GC has no register class for type '" + Type.toString() + "'"); + } + + if (Target.registers().registerClass(RequiredClass) == nullptr) { + return makeUnexpected(ErrorCode::InternalError, + "RISC-V target description is missing a required register class"); + } + if (RegisterInfo.Bank && !Target.registers().bankCoversClass(*RegisterInfo.Bank, RequiredClass)) { + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V virtual register bank cannot cover its required class"); + } + if (RegisterInfo.RegClass && + !Target.registers().isSubclass(*RegisterInfo.RegClass, RequiredClass)) { + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V virtual register has a conflicting register class"); + } + return RequiredClass; +} + +[[nodiscard]] Expected appendImmediate(MachineInstr &Instruction, std::int64_t Value) { + return Instruction.addOperand(MachineOperand::immediate(Value)); +} + +[[nodiscard]] Expected insertPhysicalRegister(MachineInstr &Instruction, std::size_t Index, + PhysicalRegister Reg, + RegisterOperandFlag Flag) { + auto Operand = MachineOperand::registerOperand(Register::fromPhysical(Reg), {Flag}); + if (!Operand) + return std::unexpected(Operand.error()); + return Instruction.insertOperand(Index, std::move(*Operand)); +} + +[[nodiscard]] Expected insertVirtualUse(MachineInstr &Instruction, std::size_t Index, + Register Reg) { + auto Operand = MachineOperand::registerOperand(Reg, {RegisterOperandFlag::Use}); + if (!Operand) + return std::unexpected(Operand.error()); + return Instruction.insertOperand(Index, std::move(*Operand)); +} + +[[nodiscard]] Expected createTemporary(MachineFunction &Function, LLT Type, + const TargetDescription &Target) { + auto Temporary = Function.registerInfo().createVirtualRegister(Type); + if (!Temporary) + return std::unexpected(Temporary.error()); + auto RequiredClass = requiredRegisterClass(Function, *Temporary, Target); + if (!RequiredClass) + return std::unexpected(RequiredClass.error()); + const RegisterBank BankKind = Type.isFloat() ? RegisterBank::FPR : RegisterBank::GPR; + auto Bank = Function.registerInfo().setRegisterBank( + *Temporary, RegisterBankID::fromValue(static_cast(BankKind))); + if (!Bank) + return std::unexpected(Bank.error()); + auto Class = Function.registerInfo().setRegisterClass(*Temporary, *RequiredClass); + if (!Class) + return std::unexpected(Class.error()); + return *Temporary; +} + +[[nodiscard]] Expected signedImmediate(const MachineOperand &Operand) { + if (const std::int64_t *Value = Operand.getIf()) + return *Value; + const auto *Integer = Operand.getIf(); + if (Integer == nullptr || Integer->Value.bitWidth() == 0 || Integer->Value.bitWidth() > 64) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V constant selection requires an immediate no wider than 64 bits"); + std::uint64_t Raw = Integer->Value.words().empty() ? 0 : Integer->Value.words().front(); + const std::uint32_t Width = Integer->Value.bitWidth(); + if (Width < 64 && (Raw & (std::uint64_t{1} << (Width - 1U))) != 0) + Raw |= ~std::uint64_t{0} << Width; + return std::bit_cast(Raw); +} + +class RISCVLegalizerInfo final : public TargetLegalizerInfo { +public: + [[nodiscard]] Expected classify(const MachineInstr &Instruction, + const MachineFunction &Function, + const TargetDescription &) const override { + if (!isGenericOpcode(Instruction.opcode())) + return LegalizationAction::Legal; + + const GenericOpcode Generic = static_cast(Instruction.opcode().value()); + const std::optional Type = resultType(Function, Instruction); + switch (Generic) { + case GenericOpcode::G_BR: + return LegalizationAction::Legal; + case GenericOpcode::G_BRCOND: { + const std::optional Condition = operandType(Function, Instruction, 0); + return Condition && isI1Type(*Condition) ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_PHI: { + if (!Type || (!isSupportedIntegerType(*Type) && !isRVFloatType(*Type))) + return LegalizationAction::Unsupported; + for (std::size_t Index = 1; Index < Instruction.operandCount(); Index += 2) { + const std::optional Incoming = operandType(Function, Instruction, Index); + if (!Incoming || *Incoming != *Type) + return LegalizationAction::Unsupported; + } + return LegalizationAction::Legal; + } + case GenericOpcode::G_IMPLICIT_DEF: + return Type && (isSupportedIntegerType(*Type) || isRVFloatType(*Type)) + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + case GenericOpcode::G_STORE: + if (Instruction.operandCount() == 0) + return LegalizationAction::Unsupported; + if (const std::optional ValueType = registerType(Function, *Instruction.operandAt(0)); + ValueType) { + const std::optional AddressType = operandType(Function, Instruction, 1); + if (!AddressType || !AddressType->isPointer() || AddressType->addressSpace() != 0 || + AddressType->scalarBitWidth() != 64) { + return LegalizationAction::Unsupported; + } + if (Instruction.memOperands().size() == 1 && + Instruction.memOperands().front()->isAtomic()) { + const std::uint64_t Bytes = ValueType->scalarBitWidth() / 8U; + return isSupportedIntegerType(*ValueType) && + isSupportedAtomicMemoryAccess(Instruction, AtomicOperation::Store, Bytes) + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + if ((isSupportedIntegerType(*ValueType) || isRVFloatType(*ValueType)) && + isSupportedNonAtomicMemoryAccess(Instruction) && + memoryMatchesType(Instruction, *ValueType)) { + return LegalizationAction::Legal; + } + } + return LegalizationAction::Unsupported; + case GenericOpcode::G_SEXTLOAD: + case GenericOpcode::G_ZEXTLOAD: { + const auto Bytes = memoryAccessBytes(Instruction); + const std::optional AddressType = operandType(Function, Instruction, 1); + return Type && Bytes && AddressType && isSupportedIntegerType(*Type) && + AddressType->isPointer() && AddressType->addressSpace() == 0 && + AddressType->scalarBitWidth() == 64 && + *Bytes < ((Type->scalarBitWidth() + 7U) / 8U) && + (*Bytes == 1 || *Bytes == 2 || *Bytes == 4) && + isSupportedNonAtomicMemoryAccess(Instruction) + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_FPEXTLOAD: { + const auto Bytes = memoryAccessBytes(Instruction); + const std::optional AddressType = operandType(Function, Instruction, 1); + return Type && Bytes && AddressType && Type->isFloat() && Type->scalarBitWidth() == 64 && + AddressType->isPointer() && AddressType->addressSpace() == 0 && + AddressType->scalarBitWidth() == 64 && *Bytes == 4 && + isSupportedNonAtomicMemoryAccess(Instruction) + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_FPTRUNCSTORE: { + const std::optional ValueType = operandType(Function, Instruction, 0); + const std::optional AddressType = operandType(Function, Instruction, 1); + const auto Bytes = memoryAccessBytes(Instruction); + return ValueType && AddressType && Bytes && ValueType->isFloat() && + ValueType->scalarBitWidth() == 64 && AddressType->isPointer() && + AddressType->addressSpace() == 0 && AddressType->scalarBitWidth() == 64 && + *Bytes == 4 && isSupportedNonAtomicMemoryAccess(Instruction) + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_LOAD: + if (Type) { + const std::optional AddressType = operandType(Function, Instruction, 1); + if (!AddressType || !AddressType->isPointer() || AddressType->addressSpace() != 0 || + AddressType->scalarBitWidth() != 64) { + return LegalizationAction::Unsupported; + } + if (Instruction.memOperands().size() == 1 && + Instruction.memOperands().front()->isAtomic()) { + const std::uint64_t Bytes = Type->scalarBitWidth() / 8U; + return isSupportedIntegerType(*Type) && + isSupportedAtomicMemoryAccess(Instruction, AtomicOperation::Load, Bytes) + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + if ((isSupportedIntegerType(*Type) || isRVFloatType(*Type)) && + isSupportedNonAtomicMemoryAccess(Instruction) && + memoryMatchesType(Instruction, *Type)) { + return LegalizationAction::Legal; + } + } + return LegalizationAction::Unsupported; + case GenericOpcode::G_FADD: + case GenericOpcode::G_FSUB: + case GenericOpcode::G_FMUL: + case GenericOpcode::G_FDIV: { + const std::optional Left = operandType(Function, Instruction, 1); + const std::optional Right = operandType(Function, Instruction, 2); + return Type && Left && Right && isRVFloatType(*Type) && *Left == *Type && *Right == *Type + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_FMA: { + const std::optional First = operandType(Function, Instruction, 1); + const std::optional Second = operandType(Function, Instruction, 2); + const std::optional Third = operandType(Function, Instruction, 3); + return Type && First && Second && Third && isRVFloatType(*Type) && *First == *Type && + *Second == *Type && *Third == *Type + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_FNEG: + case GenericOpcode::G_FABS: + case GenericOpcode::G_FSQRT: { + const std::optional Source = operandType(Function, Instruction, 1); + return Type && Source && isRVFloatType(*Type) && *Source == *Type + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_FCOPYSIGN: { + const std::optional Left = operandType(Function, Instruction, 1); + const std::optional Right = operandType(Function, Instruction, 2); + return Type && Left && Right && isRVFloatType(*Type) && *Left == *Type && *Right == *Type + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_FCONSTANT: { + const MachineOperand *Immediate = Instruction.operandAt(1); + const auto *Floating = + Immediate == nullptr ? nullptr : Immediate->getIf(); + const std::optional Semantics = Type ? Type->floatSemantics() : std::nullopt; + return Type && isRVFloatType(*Type) && Floating != nullptr && Semantics && + Floating->Bits.semantics() == *Semantics + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_ASSERT_SEXT: + case GenericOpcode::G_ASSERT_ZEXT: + case GenericOpcode::G_ASSERT_ALIGN: { + const std::optional Source = operandType(Function, Instruction, 1); + return Type && Source && *Type == *Source && isScalarGPRType(*Type) && + Instruction.operandCount() == 3 + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_FREEZE: + case GenericOpcode::G_CONSTANT_FOLD_BARRIER: { + const std::optional Source = operandType(Function, Instruction, 1); + return Type && Source && *Type == *Source && + (isSupportedIntegerType(*Type) || isRVFloatType(*Type)) + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_PTRTOINT: { + const std::optional Source = operandType(Function, Instruction, 1); + return Type && Source && Type->isInteger() && Type->scalarBitWidth() == 64 && + Source->isPointer() && Source->scalarBitWidth() == 64 + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_INTTOPTR: { + const std::optional Source = operandType(Function, Instruction, 1); + return Type && Source && Type->isPointer() && Type->scalarBitWidth() == 64 && + Source->isInteger() && Source->scalarBitWidth() == 64 + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_BITCAST: { + const std::optional Source = operandType(Function, Instruction, 1); + const bool IntegerFloatPair = + Type && Source && Type->scalarBitWidth() == Source->scalarBitWidth() && + ((Type->isFloat() && Source->isInteger()) || (Type->isInteger() && Source->isFloat())) && + (Type->scalarBitWidth() == 32 || Type->scalarBitWidth() == 64); + return IntegerFloatPair ? LegalizationAction::Legal : LegalizationAction::Unsupported; + } + case GenericOpcode::G_ANYEXT: + case GenericOpcode::G_SEXT: + case GenericOpcode::G_ZEXT: { + const std::optional Source = operandType(Function, Instruction, 1); + return Type && Source && isScalarIntegerExtension(*Type, *Source) + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_TRUNC: { + const std::optional Source = operandType(Function, Instruction, 1); + return Type && Source && isSupportedIntegerScalar(*Type) && + isSupportedIntegerScalar(*Source) && Type->scalarBitWidth() != 0 && + Source->scalarBitWidth() >= Type->scalarBitWidth() + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_SEXT_INREG: { + const std::optional Source = operandType(Function, Instruction, 1); + const MachineOperand *WidthOperand = Instruction.operandAt(2); + const auto Width = + WidthOperand == nullptr + ? Expected(std::unexpected( + Error(ErrorCode::InvalidArgument, "missing in-register extension width"))) + : signedImmediate(*WidthOperand); + return Type && Source && *Type == *Source && isScalarGPRType(*Type) && Width && *Width > 0 && + static_cast(*Width) <= Type->scalarBitWidth() + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_ICMP: { + const MachineOperand *Predicate = Instruction.operandAt(1); + const auto *IntegerPredicate = + Predicate == nullptr ? nullptr : Predicate->getIf(); + const std::optional Left = operandType(Function, Instruction, 2); + const std::optional Right = operandType(Function, Instruction, 3); + return Type && isI1Type(*Type) && IntegerPredicate != nullptr && + IntegerPredicate->integerPredicate() != nullptr && Left && Right && + *Left == *Right && isSupportedIntegerType(*Left) + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_FCMP: { + const MachineOperand *Predicate = Instruction.operandAt(1); + const auto *FloatingPredicate = + Predicate == nullptr ? nullptr : Predicate->getIf(); + const std::optional Left = operandType(Function, Instruction, 2); + const std::optional Right = operandType(Function, Instruction, 3); + return Type && isI1Type(*Type) && FloatingPredicate != nullptr && + FloatingPredicate->floatingPredicate() != nullptr && Left && Right && + *Left == *Right && isRVFloatType(*Left) + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_FPEXT: + case GenericOpcode::G_FPTRUNC: { + const std::optional Source = operandType(Function, Instruction, 1); + const bool IsExtension = Generic == GenericOpcode::G_FPEXT; + return Type && Source && isRVFloatType(*Type) && isRVFloatType(*Source) && + ((IsExtension && Source->scalarBitWidth() == 32 && + Type->scalarBitWidth() == 64) || + (!IsExtension && Source->scalarBitWidth() == 64 && + Type->scalarBitWidth() == 32)) + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_FPTOSI: + case GenericOpcode::G_FPTOUI: { + const std::optional Source = operandType(Function, Instruction, 1); + return Type && Source && isSupportedIntegerType(*Type) && isRVFloatType(*Source) + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_SITOFP: + case GenericOpcode::G_UITOFP: { + const std::optional Source = operandType(Function, Instruction, 1); + return Type && Source && isRVFloatType(*Type) && isSupportedIntegerType(*Source) + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_FMINNUM: + case GenericOpcode::G_FMAXNUM: { + const std::optional Left = operandType(Function, Instruction, 1); + const std::optional Right = operandType(Function, Instruction, 2); + return Type && Left && Right && isRVFloatType(*Type) && *Left == *Type && *Right == *Type + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_SELECT: { + const std::optional Condition = operandType(Function, Instruction, 1); + const std::optional TrueValue = operandType(Function, Instruction, 2); + const std::optional FalseValue = operandType(Function, Instruction, 3); + return Type && Condition && TrueValue && FalseValue && isI1Type(*Condition) && + *TrueValue == *Type && *FalseValue == *Type && + (isSupportedIntegerType(*Type) || isRVFloatType(*Type)) + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_PTRMASK: { + const std::optional Pointer = operandType(Function, Instruction, 1); + const std::optional Mask = operandType(Function, Instruction, 2); + return Type && Pointer && Mask && Type->isPointer() && Type->addressSpace() == 0 && + Type->scalarBitWidth() == 64 && *Pointer == *Type && + isSupportedIntegerScalar(*Mask) && Mask->scalarBitWidth() == 64 + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_ABS: { + const std::optional Source = operandType(Function, Instruction, 1); + return Type && Source && *Source == *Type && isSupportedIntegerScalar(*Type) + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_SDIVREM: + case GenericOpcode::G_UDIVREM: { + const std::optional Remainder = operandType(Function, Instruction, 1); + const std::optional Left = operandType(Function, Instruction, 2); + const std::optional Right = operandType(Function, Instruction, 3); + return Type && Remainder && Left && Right && isSupportedIntegerScalar(*Type) && + *Remainder == *Type && *Left == *Type && *Right == *Type + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_SMULH: + case GenericOpcode::G_UMULH: { + const std::optional Left = operandType(Function, Instruction, 1); + const std::optional Right = operandType(Function, Instruction, 2); + return Type && Left && Right && isNativeIntegerScalar(*Type) && *Left == *Type && + *Right == *Type + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_SMIN: + case GenericOpcode::G_SMAX: + case GenericOpcode::G_UMIN: + case GenericOpcode::G_UMAX: { + const std::optional Left = operandType(Function, Instruction, 1); + const std::optional Right = operandType(Function, Instruction, 2); + return Type && Left && Right && isSupportedIntegerScalar(*Type) && *Left == *Type && + *Right == *Type + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_UADDO: + case GenericOpcode::G_USUBO: + case GenericOpcode::G_SADDO: + case GenericOpcode::G_SSUBO: + case GenericOpcode::G_UMULO: + case GenericOpcode::G_SMULO: { + const std::optional Overflow = operandType(Function, Instruction, 1); + const std::optional Left = operandType(Function, Instruction, 2); + const std::optional Right = operandType(Function, Instruction, 3); + return Type && Overflow && Left && Right && isNativeIntegerScalar(*Type) && + isI1Type(*Overflow) && *Left == *Type && *Right == *Type + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_UADDE: + case GenericOpcode::G_USUBE: + case GenericOpcode::G_SADDE: + case GenericOpcode::G_SSUBE: { + const std::optional CarryOut = operandType(Function, Instruction, 1); + const std::optional Left = operandType(Function, Instruction, 2); + const std::optional Right = operandType(Function, Instruction, 3); + const std::optional CarryIn = operandType(Function, Instruction, 4); + return Type && CarryOut && Left && Right && CarryIn && isNativeIntegerScalar(*Type) && + isI1Type(*CarryOut) && isI1Type(*CarryIn) && *Left == *Type && *Right == *Type + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_ATOMICRMW_XCHG: + case GenericOpcode::G_ATOMICRMW_ADD: + case GenericOpcode::G_ATOMICRMW_SUB: + case GenericOpcode::G_ATOMICRMW_AND: + case GenericOpcode::G_ATOMICRMW_OR: + case GenericOpcode::G_ATOMICRMW_XOR: + case GenericOpcode::G_ATOMICRMW_MAX: + case GenericOpcode::G_ATOMICRMW_MIN: + case GenericOpcode::G_ATOMICRMW_UMAX: + case GenericOpcode::G_ATOMICRMW_UMIN: { + const std::optional Address = operandType(Function, Instruction, 1); + const std::optional Value = operandType(Function, Instruction, 2); + const std::uint64_t Bytes = Type ? Type->scalarBitWidth() / 8U : 0; + return Type && Address && Value && isNativeIntegerScalar(*Type) && *Value == *Type && + Address->isPointer() && Address->addressSpace() == 0 && + Address->scalarBitWidth() == 64 && + isSupportedAtomicMemoryAccess(Instruction, AtomicOperation::ReadModifyWrite, + Bytes) + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_FENCE: { + const MachineOperand *OrderingOperand = Instruction.operandAt(0); + const MachineOperand *ScopeOperand = Instruction.operandAt(1); + const auto Ordering = OrderingOperand == nullptr + ? Expected(std::unexpected( + Error(ErrorCode::InvalidArgument, "missing fence ordering"))) + : signedImmediate(*OrderingOperand); + const auto Scope = ScopeOperand == nullptr + ? Expected(std::unexpected( + Error(ErrorCode::InvalidArgument, "missing fence scope"))) + : signedImmediate(*ScopeOperand); + return Ordering && Scope && *Ordering >= 4 && *Ordering <= 7 && *Scope == 0 + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_BRINDIRECT: + return operandType(Function, Instruction, 0) + .transform([](LLT Value) { + return Value.isPointer() && Value.addressSpace() == 0 && + Value.scalarBitWidth() == 64; + }) + .value_or(false) + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + case GenericOpcode::G_TRAP: + case GenericOpcode::G_DEBUGTRAP: + return Instruction.operandCount() == 0 ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + case GenericOpcode::G_UBSANTRAP: + return Instruction.operandCount() == 1 ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + case GenericOpcode::G_CONSTANT: { + const MachineOperand *Immediate = Instruction.operandAt(1); + return Type && isSupportedIntegerType(*Type) && Immediate != nullptr && + (Immediate->kind() == MachineOperand::Kind::Immediate || + Immediate->kind() == MachineOperand::Kind::IntegerImmediate) + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_ADD: + case GenericOpcode::G_SUB: + case GenericOpcode::G_MUL: + case GenericOpcode::G_SDIV: + case GenericOpcode::G_UDIV: + case GenericOpcode::G_SREM: + case GenericOpcode::G_UREM: + case GenericOpcode::G_AND: + case GenericOpcode::G_OR: + case GenericOpcode::G_XOR: + case GenericOpcode::G_SHL: + case GenericOpcode::G_LSHR: + case GenericOpcode::G_ASHR: { + const std::optional Left = operandType(Function, Instruction, 1); + const std::optional Right = operandType(Function, Instruction, 2); + return Type && Left && Right && isSupportedIntegerScalar(*Type) && *Left == *Type && + *Right == *Type + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + case GenericOpcode::G_PTR_ADD: { + const std::optional Pointer = operandType(Function, Instruction, 1); + const std::optional Offset = operandType(Function, Instruction, 2); + return Type && Pointer && Offset && Type->isPointer() && Type->addressSpace() == 0 && + Type->scalarBitWidth() == 64 && *Pointer == *Type && Offset->isInteger() && + Offset->scalarBitWidth() == 64 + ? LegalizationAction::Legal + : LegalizationAction::Unsupported; + } + default: + break; + } + + // Keep the unsupported policy mechanically synchronized with the complete + // target-independent opcode inventory. Supported cases return above. + switch (Generic) { +#define RCC_CORE_OPCODE(Name) +#define RCC_GENERIC_OPCODE(Name) case GenericOpcode::Name: +#include "rcc/CodeGen/GenericOpcode.def" +#undef RCC_GENERIC_OPCODE +#undef RCC_CORE_OPCODE + return LegalizationAction::Unsupported; + case GenericOpcode::Invalid: + case GenericOpcode::CORE_PSEUDO_END: + case GenericOpcode::G_GENERIC_END: + default: + break; + } + return makeUnexpected(ErrorCode::InternalError, + "RISC-V legalizer received an invalid generic opcode value"); + } + + [[nodiscard]] Expected lower(MachineInstr &, MachineFunction &, const TargetDescription &, + LegalizationAction) const override { + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V legalizer has no lowering rule for this instruction"); + } +}; + +class RISCVRegisterBankInfo final : public TargetRegisterBankInfo { +public: + [[nodiscard]] Expected selectBank(const MachineFunction &, Register, + const VirtualRegisterInfo &Info, + const MachineInstr *, + const TargetDescription &) const override { + if (!Info.Type) { + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V register-bank selection requires a virtual-register type"); + } + if (isScalarGPRType(*Info.Type)) { + return RegisterBankID::fromValue(static_cast(RegisterBank::GPR)); + } + if (isRVFloatType(*Info.Type)) + return RegisterBankID::fromValue(static_cast(RegisterBank::FPR)); + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V RV64GC has no register bank for virtual register type '" + + Info.Type->toString() + "'"); + } + + [[nodiscard]] Expected + constrainInstruction(MachineInstr &Instruction, MachineFunction &Function, + const TargetDescription &Target) const override { + std::vector> PlannedClasses; + for (const MachineOperand &Operand : Instruction.operands()) { + const Register *Reg = Operand.getIf(); + if (Reg == nullptr || !Reg->isVirtual()) + continue; + auto Required = requiredRegisterClass(Function, *Reg, Target); + if (!Required) + return std::unexpected(Required.error()); + if (std::ranges::find_if(PlannedClasses, [Reg](const auto &Entry) { + return Entry.first == *Reg; + }) == PlannedClasses.end()) + PlannedClasses.emplace_back(*Reg, *Required); + } + for (const auto &[Reg, Required] : PlannedClasses) { + auto Info = Function.registerInfo().virtualRegisterInfo(Reg); + if (!Info) + return std::unexpected(Info.error()); + if (Info->get().RegClass) + continue; + auto SetClass = Function.registerInfo().setRegisterClass(Reg, Required); + if (!SetClass) + return std::unexpected(SetClass.error()); + } + return {}; + } + + [[nodiscard]] Expected validateInstruction(const MachineInstr &Instruction, + const MachineFunction &Function, + const TargetDescription &Target) const override { + for (const MachineOperand &Operand : Instruction.operands()) { + const Register *Reg = Operand.getIf(); + if (Reg == nullptr || !Reg->isVirtual()) + continue; + auto Required = requiredRegisterClass(Function, *Reg, Target); + if (!Required) + return std::unexpected(Required.error()); + } + return {}; + } +}; + +class RISCVInstructionSelector final : public TargetInstructionSelector { +public: + [[nodiscard]] Expected canSelect(const MachineInstr &Instruction, + const MachineFunction &Function, + const TargetDescription &Target) const override { + if (!isGenericOpcode(Instruction.opcode())) + return {}; + const GenericOpcode Generic = static_cast(Instruction.opcode().value()); + const std::optional Type = resultType(Function, Instruction); + const auto Integer = [Type] { return Type && isSupportedIntegerType(*Type); }; + const auto Float = [Type] { return Type && isRVFloatType(*Type); }; + auto Classification = legalizerInfo().classify(Instruction, Function, Target); + if (!Classification) + return std::unexpected(Classification.error()); + if (*Classification != LegalizationAction::Legal) { + std::string Message = "RISC-V selector cannot select opcode '" + + std::string(getGenericOpcodeName(Generic)) + "'"; + if (Type) + Message += " with result type '" + Type->toString() + "'"; + Message += ": the legalization rule explicitly marks this combination unsupported"; + return makeUnexpected(ErrorCode::Unsupported, std::move(Message)); + } + switch (Generic) { + case GenericOpcode::G_ADD: + case GenericOpcode::G_SUB: + case GenericOpcode::G_MUL: + case GenericOpcode::G_SDIV: + case GenericOpcode::G_UDIV: + case GenericOpcode::G_SREM: + case GenericOpcode::G_UREM: + case GenericOpcode::G_AND: + case GenericOpcode::G_OR: + case GenericOpcode::G_XOR: + case GenericOpcode::G_SHL: + case GenericOpcode::G_LSHR: + case GenericOpcode::G_ASHR: + case GenericOpcode::G_PTR_ADD: + case GenericOpcode::G_PTRMASK: + case GenericOpcode::G_CONSTANT: + if (!Integer()) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V selector requires an i1-i64 integer or 64-bit pointer value"); + if (Generic == GenericOpcode::G_CONSTANT) { + const MachineOperand *Immediate = Instruction.operandAt(1); + if (Immediate == nullptr || (Immediate->kind() != MachineOperand::Kind::Immediate && + Immediate->kind() != MachineOperand::Kind::IntegerImmediate)) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V integer constant requires an immediate operand"); + } + return {}; + case GenericOpcode::G_LOAD: + case GenericOpcode::G_SEXTLOAD: + case GenericOpcode::G_ZEXTLOAD: + case GenericOpcode::G_FPEXTLOAD: + case GenericOpcode::G_FPTRUNCSTORE: + case GenericOpcode::G_STORE: + case GenericOpcode::G_BR: + case GenericOpcode::G_BRCOND: + return {}; + case GenericOpcode::G_FADD: + case GenericOpcode::G_FSUB: + case GenericOpcode::G_FMUL: + case GenericOpcode::G_FDIV: + case GenericOpcode::G_FMA: + case GenericOpcode::G_FSQRT: + case GenericOpcode::G_FNEG: + case GenericOpcode::G_FABS: + case GenericOpcode::G_FCOPYSIGN: + if (!Float()) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V selector requires f32 or f64 floating values"); + return {}; + case GenericOpcode::G_FCONSTANT: { + const MachineOperand *Immediate = Instruction.operandAt(1); + const auto *Floating = + Immediate == nullptr ? nullptr : Immediate->getIf(); + const std::optional Semantics = Type ? Type->floatSemantics() : std::nullopt; + if (!Float() || Floating == nullptr || !Semantics || Floating->Bits.semantics() != *Semantics) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V selector requires a matching floating constant"); + return {}; + } + case GenericOpcode::G_ANYEXT: + case GenericOpcode::G_ASSERT_SEXT: + case GenericOpcode::G_ASSERT_ZEXT: + case GenericOpcode::G_ASSERT_ALIGN: + case GenericOpcode::G_FREEZE: + case GenericOpcode::G_CONSTANT_FOLD_BARRIER: + case GenericOpcode::G_PTRTOINT: + case GenericOpcode::G_INTTOPTR: + case GenericOpcode::G_BITCAST: + case GenericOpcode::G_TRUNC: + case GenericOpcode::G_SEXT: + case GenericOpcode::G_SEXT_INREG: + case GenericOpcode::G_ZEXT: + case GenericOpcode::G_ICMP: + case GenericOpcode::G_FCMP: + case GenericOpcode::G_FPEXT: + case GenericOpcode::G_FPTRUNC: + case GenericOpcode::G_FPTOSI: + case GenericOpcode::G_FPTOUI: + case GenericOpcode::G_SITOFP: + case GenericOpcode::G_UITOFP: + case GenericOpcode::G_FMINNUM: + case GenericOpcode::G_FMAXNUM: + case GenericOpcode::G_SDIVREM: + case GenericOpcode::G_UDIVREM: + case GenericOpcode::G_SMULH: + case GenericOpcode::G_UMULH: + case GenericOpcode::G_SMIN: + case GenericOpcode::G_SMAX: + case GenericOpcode::G_UMIN: + case GenericOpcode::G_UMAX: + case GenericOpcode::G_SELECT: + case GenericOpcode::G_ABS: + case GenericOpcode::G_UADDO: + case GenericOpcode::G_USUBO: + case GenericOpcode::G_SADDO: + case GenericOpcode::G_SSUBO: + case GenericOpcode::G_UMULO: + case GenericOpcode::G_SMULO: + case GenericOpcode::G_UADDE: + case GenericOpcode::G_USUBE: + case GenericOpcode::G_SADDE: + case GenericOpcode::G_SSUBE: + case GenericOpcode::G_ATOMICRMW_XCHG: + case GenericOpcode::G_ATOMICRMW_ADD: + case GenericOpcode::G_ATOMICRMW_SUB: + case GenericOpcode::G_ATOMICRMW_AND: + case GenericOpcode::G_ATOMICRMW_OR: + case GenericOpcode::G_ATOMICRMW_XOR: + case GenericOpcode::G_ATOMICRMW_MAX: + case GenericOpcode::G_ATOMICRMW_MIN: + case GenericOpcode::G_ATOMICRMW_UMAX: + case GenericOpcode::G_ATOMICRMW_UMIN: + case GenericOpcode::G_FENCE: + case GenericOpcode::G_BRINDIRECT: + case GenericOpcode::G_TRAP: + case GenericOpcode::G_DEBUGTRAP: + case GenericOpcode::G_UBSANTRAP: + case GenericOpcode::G_PHI: + case GenericOpcode::G_IMPLICIT_DEF: + return {}; + default: + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V instruction selector has no rule for generic opcode"); + } + } + + [[nodiscard]] Expected select(MachineInstr &Instruction, MachineFunction &Function, + const TargetDescription &Target) const override { + auto Match = canSelect(Instruction, Function, Target); + if (!Match) + return std::unexpected(Match.error()); + const GenericOpcode Generic = static_cast(Instruction.opcode().value()); + const std::optional Type = resultType(Function, Instruction); + + const auto SelectDirect = [&Instruction](InstructionOpcode Selected) { + return Instruction.setOpcode(targetOpcode(Selected)); + }; + switch (Generic) { + case GenericOpcode::G_ADD: + return selectIntegerBinary(Instruction, Type, InstructionOpcode::ADD, + InstructionOpcode::ADDW); + case GenericOpcode::G_SUB: + return selectIntegerBinary(Instruction, Type, InstructionOpcode::SUB, + InstructionOpcode::SUBW); + case GenericOpcode::G_MUL: + return selectIntegerBinary(Instruction, Type, InstructionOpcode::MUL, + InstructionOpcode::MULW); + case GenericOpcode::G_SDIV: + return selectNormalizedIntegerBinary(Instruction, Function, Target, Type, + InstructionOpcode::DIV, InstructionOpcode::DIVW, true, + true); + case GenericOpcode::G_UDIV: + return selectNormalizedIntegerBinary(Instruction, Function, Target, Type, + InstructionOpcode::DIVU, InstructionOpcode::DIVUW, false, + true); + case GenericOpcode::G_SREM: + return selectNormalizedIntegerBinary(Instruction, Function, Target, Type, + InstructionOpcode::REM, InstructionOpcode::REMW, true, + true); + case GenericOpcode::G_UREM: + return selectNormalizedIntegerBinary(Instruction, Function, Target, Type, + InstructionOpcode::REMU, InstructionOpcode::REMUW, false, + true); + case GenericOpcode::G_SDIVREM: + return selectDivRem(Instruction, Function, Target, Type, true); + case GenericOpcode::G_UDIVREM: + return selectDivRem(Instruction, Function, Target, Type, false); + case GenericOpcode::G_AND: + return SelectDirect(InstructionOpcode::AND); + case GenericOpcode::G_OR: + return SelectDirect(InstructionOpcode::OR); + case GenericOpcode::G_XOR: + return SelectDirect(InstructionOpcode::XOR); + case GenericOpcode::G_SHL: + return selectIntegerBinary(Instruction, Type, InstructionOpcode::SLL, + InstructionOpcode::SLLW); + case GenericOpcode::G_LSHR: + return selectNormalizedIntegerBinary(Instruction, Function, Target, Type, + InstructionOpcode::SRL, InstructionOpcode::SRLW, false, + false); + case GenericOpcode::G_ASHR: + return selectNormalizedIntegerBinary(Instruction, Function, Target, Type, + InstructionOpcode::SRA, InstructionOpcode::SRAW, true, + false); + case GenericOpcode::G_PTR_ADD: + return SelectDirect(InstructionOpcode::ADD); + case GenericOpcode::G_PTRMASK: + return SelectDirect(InstructionOpcode::AND); + case GenericOpcode::G_LOAD: + return selectLoad(Instruction, Type); + case GenericOpcode::G_SEXTLOAD: + return selectExtLoad(Instruction, Type, true); + case GenericOpcode::G_ZEXTLOAD: + return selectExtLoad(Instruction, Type, false); + case GenericOpcode::G_FPEXTLOAD: + return selectFloatExtLoad(Instruction, Function, Target); + case GenericOpcode::G_STORE: + return selectStore(Instruction, Function); + case GenericOpcode::G_FPTRUNCSTORE: + return selectFloatTruncStore(Instruction, Function, Target); + case GenericOpcode::G_BR: + return selectBranch(Instruction); + case GenericOpcode::G_BRCOND: + return selectConditionalBranch(Instruction); + case GenericOpcode::G_CONSTANT: + return selectIntegerConstant(Instruction, Function, Target, Type); + case GenericOpcode::G_FCONSTANT: + return selectFloatingConstant(Instruction, Function, Target, Type); + case GenericOpcode::G_ASSERT_SEXT: + case GenericOpcode::G_ASSERT_ZEXT: + case GenericOpcode::G_ASSERT_ALIGN: + if (Instruction.operandCount() == 3) { + auto EraseImmediate = Instruction.eraseOperand(2); + if (!EraseImmediate) + return std::unexpected(EraseImmediate.error()); + } + return Instruction.setOpcode(opcode(GenericOpcode::COPY)); + case GenericOpcode::G_FREEZE: + case GenericOpcode::G_CONSTANT_FOLD_BARRIER: + case GenericOpcode::G_PTRTOINT: + case GenericOpcode::G_INTTOPTR: + return Instruction.setOpcode(opcode(GenericOpcode::COPY)); + case GenericOpcode::G_BITCAST: + return selectBitcast(Instruction, Function); + case GenericOpcode::G_FADD: + return selectRoundingBinary(Instruction, Type, InstructionOpcode::FADD_S, + InstructionOpcode::FADD_D); + case GenericOpcode::G_FSUB: + return selectRoundingBinary(Instruction, Type, InstructionOpcode::FSUB_S, + InstructionOpcode::FSUB_D); + case GenericOpcode::G_FMUL: + return selectRoundingBinary(Instruction, Type, InstructionOpcode::FMUL_S, + InstructionOpcode::FMUL_D); + case GenericOpcode::G_FDIV: + return selectRoundingBinary(Instruction, Type, InstructionOpcode::FDIV_S, + InstructionOpcode::FDIV_D); + case GenericOpcode::G_FMA: + return selectRoundingTernary(Instruction, Type, InstructionOpcode::FMADD_S, + InstructionOpcode::FMADD_D); + case GenericOpcode::G_FSQRT: + return selectRoundingUnary(Instruction, Type, InstructionOpcode::FSQRT_S, + InstructionOpcode::FSQRT_D); + case GenericOpcode::G_FNEG: + return selectSignInjection(Instruction, Type, InstructionOpcode::FSGNJN_S, + InstructionOpcode::FSGNJN_D); + case GenericOpcode::G_FABS: + return selectSignInjection(Instruction, Type, InstructionOpcode::FSGNJX_S, + InstructionOpcode::FSGNJX_D); + case GenericOpcode::G_FCOPYSIGN: + return selectFloatingMinMax(Instruction, Type, InstructionOpcode::FSGNJ_S, + InstructionOpcode::FSGNJ_D); + case GenericOpcode::G_ANYEXT: + case GenericOpcode::G_SEXT: + case GenericOpcode::G_ZEXT: + return selectIntegerExtension(Instruction, Function, Target, Generic, Type); + case GenericOpcode::G_TRUNC: + return Instruction.setOpcode(opcode(GenericOpcode::COPY)); + case GenericOpcode::G_SEXT_INREG: + return selectSignExtendInRegister(Instruction, Function, Target, Type); + case GenericOpcode::G_ICMP: + return selectIntegerCompare(Instruction, Function, Target); + case GenericOpcode::G_FCMP: + return selectFloatingCompare(Instruction, Function, Target); + case GenericOpcode::G_FPEXT: + return selectRoundingUnary(Instruction, Type, InstructionOpcode::FCVT_D_S, + InstructionOpcode::FCVT_D_S); + case GenericOpcode::G_FPTRUNC: + return selectRoundingUnary(Instruction, Type, InstructionOpcode::FCVT_S_D, + InstructionOpcode::FCVT_S_D); + case GenericOpcode::G_FPTOSI: + case GenericOpcode::G_FPTOUI: + return selectFloatToInteger(Instruction, Function, Type, Generic == GenericOpcode::G_FPTOUI); + case GenericOpcode::G_SITOFP: + case GenericOpcode::G_UITOFP: + return selectIntegerToFloat(Instruction, Function, Target, Type, + Generic == GenericOpcode::G_UITOFP); + case GenericOpcode::G_FMINNUM: + return selectFloatingMinMax(Instruction, Type, InstructionOpcode::FMIN_S, + InstructionOpcode::FMIN_D); + case GenericOpcode::G_FMAXNUM: + return selectFloatingMinMax(Instruction, Type, InstructionOpcode::FMAX_S, + InstructionOpcode::FMAX_D); + case GenericOpcode::G_SMULH: + return selectMultiplyHigh(Instruction, Function, Target, Type, true); + case GenericOpcode::G_UMULH: + return selectMultiplyHigh(Instruction, Function, Target, Type, false); + case GenericOpcode::G_SMIN: + case GenericOpcode::G_SMAX: + case GenericOpcode::G_UMIN: + case GenericOpcode::G_UMAX: + return selectIntegerMinMax(Instruction, Function, Target, Type, Generic); + case GenericOpcode::G_SELECT: + return selectValue(Instruction, Function, Target, Type); + case GenericOpcode::G_ABS: + return selectAbsoluteValue(Instruction, Function, Target, Type); + case GenericOpcode::G_UADDO: + case GenericOpcode::G_USUBO: + case GenericOpcode::G_SADDO: + case GenericOpcode::G_SSUBO: + case GenericOpcode::G_UMULO: + case GenericOpcode::G_SMULO: + return selectOverflow(Instruction, Function, Target, Type, Generic); + case GenericOpcode::G_UADDE: + case GenericOpcode::G_USUBE: + case GenericOpcode::G_SADDE: + case GenericOpcode::G_SSUBE: + return selectExtendedOverflow(Instruction, Function, Target, Type, Generic); + case GenericOpcode::G_ATOMICRMW_XCHG: + case GenericOpcode::G_ATOMICRMW_ADD: + case GenericOpcode::G_ATOMICRMW_SUB: + case GenericOpcode::G_ATOMICRMW_AND: + case GenericOpcode::G_ATOMICRMW_OR: + case GenericOpcode::G_ATOMICRMW_XOR: + case GenericOpcode::G_ATOMICRMW_MAX: + case GenericOpcode::G_ATOMICRMW_MIN: + case GenericOpcode::G_ATOMICRMW_UMAX: + case GenericOpcode::G_ATOMICRMW_UMIN: + return selectAtomicRMW(Instruction, Function, Target, Type, Generic); + case GenericOpcode::G_FENCE: + return selectFence(Instruction); + case GenericOpcode::G_BRINDIRECT: + return selectIndirectBranch(Instruction); + case GenericOpcode::G_TRAP: + case GenericOpcode::G_DEBUGTRAP: + case GenericOpcode::G_UBSANTRAP: + return selectTrap(Instruction); + case GenericOpcode::G_PHI: + return Instruction.setOpcode(opcode(GenericOpcode::PHI)); + case GenericOpcode::G_IMPLICIT_DEF: + return Instruction.setOpcode(opcode(GenericOpcode::IMPLICIT_DEF)); + case GenericOpcode::COPY: + case GenericOpcode::IMPLICIT_DEF: + case GenericOpcode::PHI: + case GenericOpcode::REG_SEQUENCE: + case GenericOpcode::INSERT_SUBREG: + case GenericOpcode::EXTRACT_SUBREG: + case GenericOpcode::SUBREG_TO_REG: + case GenericOpcode::KILL: + case GenericOpcode::BUNDLE: + case GenericOpcode::INLINEASM: + case GenericOpcode::CORE_PSEUDO_END: + break; + default: + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V instruction selector has no rule for generic opcode"); + } + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V instruction selector has no rule for generic opcode"); + } + +private: + [[nodiscard]] static Expected replaceImmediate(MachineInstr &Instruction, std::size_t Index, + std::int64_t Value) { + if (Instruction.operandAt(Index) == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V instruction is missing an immediate operand"); + auto Erased = Instruction.eraseOperand(Index); + if (!Erased) + return std::unexpected(Erased.error()); + return Instruction.insertOperand(Index, MachineOperand::immediate(Value)); + } + + [[nodiscard]] static Expected + buildBinaryAfter(MachineIRBuilder &Builder, MachineInstr &Anchor, InstructionOpcode Opcode, + Register Def, Register Left, Register Right) { + auto Point = Builder.setInsertPointAfter(Anchor); + if (!Point) + return std::unexpected(Point.error()); + auto Pending = Builder.buildInstr(targetOpcode(Opcode)); + if (!Pending) + return std::unexpected(Pending.error()); + auto DefResult = Pending->addDef(Def); + if (!DefResult) + return std::unexpected(DefResult.error()); + auto LeftResult = Pending->addUse(Left); + if (!LeftResult) + return std::unexpected(LeftResult.error()); + auto RightResult = Pending->addUse(Right); + if (!RightResult) + return std::unexpected(RightResult.error()); + auto Built = Pending->build(); + if (!Built) + return std::unexpected(Built.error()); + return *Built; + } + + [[nodiscard]] static Expected + buildImmediateAfter(MachineIRBuilder &Builder, MachineInstr &Anchor, InstructionOpcode Opcode, + Register Def, Register Source, std::int64_t Value) { + auto Point = Builder.setInsertPointAfter(Anchor); + if (!Point) + return std::unexpected(Point.error()); + auto Pending = Builder.buildInstr(targetOpcode(Opcode)); + if (!Pending) + return std::unexpected(Pending.error()); + auto DefResult = Pending->addDef(Def); + if (!DefResult) + return std::unexpected(DefResult.error()); + auto SourceResult = Pending->addUse(Source); + if (!SourceResult) + return std::unexpected(SourceResult.error()); + auto ImmediateResult = Pending->addImmediate(Value); + if (!ImmediateResult) + return std::unexpected(ImmediateResult.error()); + auto Built = Pending->build(); + if (!Built) + return std::unexpected(Built.error()); + return *Built; + } + + [[nodiscard]] static Expected + buildImmediateBefore(MachineIRBuilder &Builder, MachineInstr &Anchor, InstructionOpcode Opcode, + Register Def, Register Source, std::int64_t Value) { + auto Point = Builder.setInsertPointBefore(Anchor); + if (!Point) + return std::unexpected(Point.error()); + auto Pending = Builder.buildInstr(targetOpcode(Opcode)); + if (!Pending) + return std::unexpected(Pending.error()); + auto DefResult = Pending->addDef(Def); + if (!DefResult) + return std::unexpected(DefResult.error()); + auto SourceResult = Pending->addUse(Source); + if (!SourceResult) + return std::unexpected(SourceResult.error()); + auto ImmediateResult = Pending->addImmediate(Value); + if (!ImmediateResult) + return std::unexpected(ImmediateResult.error()); + auto Built = Pending->build(); + if (!Built) + return std::unexpected(Built.error()); + return *Built; + } + + [[nodiscard]] static Expected + normalizeIntegerBefore(MachineInstr &Anchor, MachineFunction &Function, + const TargetDescription &Target, Register Source, LLT Type, bool Signed) { + if (!isSupportedIntegerScalar(Type)) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V integer normalization requires a scalar i1-i64 type"); + const std::uint32_t Bits = Type.scalarBitWidth(); + if (Bits == 64) + return Source; + + MachineIRBuilder Builder(Function, Target); + if (Signed && Bits == 32) { + auto Result = createTemporary(Function, Type, Target); + if (!Result) + return std::unexpected(Result.error()); + auto Extended = + buildImmediateBefore(Builder, Anchor, InstructionOpcode::ADDIW, *Result, Source, 0); + if (!Extended) + return std::unexpected(Extended.error()); + return *Result; + } + + const std::uint32_t ContainerBits = Bits < 32 ? 32 : 64; + const std::int64_t Shift = static_cast(ContainerBits - Bits); + auto Shifted = createTemporary(Function, Type, Target); + auto Result = createTemporary(Function, Type, Target); + if (!Shifted || !Result) + return std::unexpected(!Shifted ? Shifted.error() : Result.error()); + const InstructionOpcode ShiftLeft = + ContainerBits == 32 ? InstructionOpcode::SLLIW : InstructionOpcode::SLLI; + const InstructionOpcode ShiftRight = + Signed ? (ContainerBits == 32 ? InstructionOpcode::SRAIW : InstructionOpcode::SRAI) + : (ContainerBits == 32 ? InstructionOpcode::SRLIW : InstructionOpcode::SRLI); + auto Left = buildImmediateBefore(Builder, Anchor, ShiftLeft, *Shifted, Source, Shift); + if (!Left) + return std::unexpected(Left.error()); + auto Right = buildImmediateBefore(Builder, Anchor, ShiftRight, *Result, *Shifted, Shift); + if (!Right) + return std::unexpected(Right.error()); + return *Result; + } + + [[nodiscard]] static Expected buildUnaryAfter(MachineIRBuilder &Builder, + MachineInstr &Anchor, + InstructionOpcode Opcode, + Register Def, Register Source) { + auto Point = Builder.setInsertPointAfter(Anchor); + if (!Point) + return std::unexpected(Point.error()); + auto Pending = Builder.buildInstr(targetOpcode(Opcode)); + if (!Pending) + return std::unexpected(Pending.error()); + auto DefResult = Pending->addDef(Def); + if (!DefResult) + return std::unexpected(DefResult.error()); + auto SourceResult = Pending->addUse(Source); + if (!SourceResult) + return std::unexpected(SourceResult.error()); + auto Built = Pending->build(); + if (!Built) + return std::unexpected(Built.error()); + return *Built; + } + + [[nodiscard]] static Expected + buildGPRSelectAfter(MachineIRBuilder &Builder, MachineInstr &Anchor, MachineFunction &Function, + const TargetDescription &Target, LLT ValueType, Register Destination, + Register Condition, Register TrueValue, Register FalseValue) { + auto IntegerType = LLT::makeInteger(ValueType.scalarBitWidth()); + if (!IntegerType) + return std::unexpected(IntegerType.error()); + auto Mask = createTemporary(Function, *IntegerType, Target); + auto Difference = createTemporary(Function, *IntegerType, Target); + auto MaskedDifference = createTemporary(Function, *IntegerType, Target); + if (!Mask || !Difference || !MaskedDifference) { + return std::unexpected( + (!Mask ? Mask.error() : (!Difference ? Difference.error() : MaskedDifference.error()))); + } + auto Zero = physicalRegister(PhysicalRegisterID::X0); + if (!Zero) + return std::unexpected(Zero.error()); + const InstructionOpcode Subtract = + ValueType.scalarBitWidth() == 32 ? InstructionOpcode::SUBW : InstructionOpcode::SUB; + auto MaskInstruction = buildBinaryAfter(Builder, Anchor, Subtract, *Mask, + Register::fromPhysical(*Zero), Condition); + if (!MaskInstruction) + return std::unexpected(MaskInstruction.error()); + auto DifferenceInstruction = buildBinaryAfter( + Builder, **MaskInstruction, InstructionOpcode::XOR, *Difference, TrueValue, FalseValue); + if (!DifferenceInstruction) + return std::unexpected(DifferenceInstruction.error()); + auto MaskInstructionResult = + buildBinaryAfter(Builder, **DifferenceInstruction, InstructionOpcode::AND, + *MaskedDifference, *Difference, *Mask); + if (!MaskInstructionResult) + return std::unexpected(MaskInstructionResult.error()); + return buildBinaryAfter(Builder, **MaskInstructionResult, InstructionOpcode::XOR, Destination, + FalseValue, *MaskedDifference); + } + + [[nodiscard]] static Expected selectDivRem(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target, + std::optional Type, bool Signed) { + const MachineOperand *RemainderOperand = Instruction.operandAt(1); + const MachineOperand *LeftOperand = Instruction.operandAt(2); + const MachineOperand *RightOperand = Instruction.operandAt(3); + const Register *Remainder = + RemainderOperand == nullptr ? nullptr : RemainderOperand->getIf(); + const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf(); + const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf(); + if (!Type || Remainder == nullptr || Left == nullptr || Right == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V division-with-remainder has invalid operands"); + const Register RemainderRegister = *Remainder; + auto LeftRegister = normalizeIntegerBefore(Instruction, Function, Target, *Left, *Type, Signed); + auto RightRegister = + normalizeIntegerBefore(Instruction, Function, Target, *Right, *Type, Signed); + if (!LeftRegister || !RightRegister) + return std::unexpected(!LeftRegister ? LeftRegister.error() : RightRegister.error()); + auto RemoveRemainder = Instruction.eraseOperand(1); + if (!RemoveRemainder) + return std::unexpected(RemoveRemainder.error()); + auto SetLeft = Instruction.operandAt(1)->setRegister(*LeftRegister); + if (!SetLeft) + return std::unexpected(SetLeft.error()); + auto SetRight = Instruction.operandAt(2)->setRegister(*RightRegister); + if (!SetRight) + return std::unexpected(SetRight.error()); + const bool IsWord = Type->scalarBitWidth() <= 32; + const InstructionOpcode Division = + IsWord ? (Signed ? InstructionOpcode::DIVW : InstructionOpcode::DIVUW) + : (Signed ? InstructionOpcode::DIV : InstructionOpcode::DIVU); + const InstructionOpcode RemainderOpcode = + IsWord ? (Signed ? InstructionOpcode::REMW : InstructionOpcode::REMUW) + : (Signed ? InstructionOpcode::REM : InstructionOpcode::REMU); + auto SetDivision = Instruction.setOpcode(targetOpcode(Division)); + if (!SetDivision) + return std::unexpected(SetDivision.error()); + MachineIRBuilder Builder(Function, Target); + auto BuiltRemainder = buildBinaryAfter(Builder, Instruction, RemainderOpcode, RemainderRegister, + *LeftRegister, *RightRegister); + if (!BuiltRemainder) + return std::unexpected(BuiltRemainder.error()); + return {}; + } + + [[nodiscard]] static Expected selectMultiplyHigh(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target, + std::optional Type, bool Signed) { + if (!Type) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V high multiplication has no result type"); + if (Type->scalarBitWidth() == 64) { + return Instruction.setOpcode( + targetOpcode(Signed ? InstructionOpcode::MULH : InstructionOpcode::MULHU)); + } + + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const MachineOperand *LeftOperand = Instruction.operandAt(1); + const MachineOperand *RightOperand = Instruction.operandAt(2); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); + const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf(); + const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf(); + if (Destination == nullptr || Left == nullptr || Right == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V high multiplication has invalid register operands"); + const Register DestinationRegister = *Destination; + const Register RightRegister = *Right; + auto I64 = LLT::makeInteger(64); + if (!I64) + return std::unexpected(I64.error()); + auto ExtendedLeft = createTemporary(Function, *I64, Target); + auto ExtendedRight = createTemporary(Function, *I64, Target); + auto Product = createTemporary(Function, *I64, Target); + if (!ExtendedLeft || !ExtendedRight || !Product) { + return std::unexpected(!ExtendedLeft + ? ExtendedLeft.error() + : (!ExtendedRight ? ExtendedRight.error() : Product.error())); + } + + auto RemoveRight = Instruction.eraseOperand(2); + if (!RemoveRight) + return std::unexpected(RemoveRight.error()); + MachineIRBuilder Builder(Function, Target); + MachineInstr *Anchor = &Instruction; + if (Signed) { + auto SetDestination = Instruction.operandAt(0)->setRegister(*ExtendedLeft); + if (!SetDestination) + return std::unexpected(SetDestination.error()); + auto Immediate = appendImmediate(Instruction, 0); + if (!Immediate) + return std::unexpected(Immediate.error()); + auto SetOpcode = Instruction.setOpcode(targetOpcode(InstructionOpcode::ADDIW)); + if (!SetOpcode) + return std::unexpected(SetOpcode.error()); + auto ExtendRight = buildImmediateAfter(Builder, *Anchor, InstructionOpcode::ADDIW, + *ExtendedRight, RightRegister, 0); + if (!ExtendRight) + return std::unexpected(ExtendRight.error()); + Anchor = *ExtendRight; + } else { + auto ShiftedLeft = createTemporary(Function, *I64, Target); + auto ShiftedRight = createTemporary(Function, *I64, Target); + if (!ShiftedLeft || !ShiftedRight) + return std::unexpected(!ShiftedLeft ? ShiftedLeft.error() : ShiftedRight.error()); + auto SetDestination = Instruction.operandAt(0)->setRegister(*ShiftedLeft); + if (!SetDestination) + return std::unexpected(SetDestination.error()); + auto Immediate = appendImmediate(Instruction, 32); + if (!Immediate) + return std::unexpected(Immediate.error()); + auto SetOpcode = Instruction.setOpcode(targetOpcode(InstructionOpcode::SLLI)); + if (!SetOpcode) + return std::unexpected(SetOpcode.error()); + auto ExtendLeft = buildImmediateAfter(Builder, *Anchor, InstructionOpcode::SRLI, + *ExtendedLeft, *ShiftedLeft, 32); + if (!ExtendLeft) + return std::unexpected(ExtendLeft.error()); + auto ShiftRight = buildImmediateAfter(Builder, **ExtendLeft, InstructionOpcode::SLLI, + *ShiftedRight, RightRegister, 32); + if (!ShiftRight) + return std::unexpected(ShiftRight.error()); + auto ExtendRight = buildImmediateAfter(Builder, **ShiftRight, InstructionOpcode::SRLI, + *ExtendedRight, *ShiftedRight, 32); + if (!ExtendRight) + return std::unexpected(ExtendRight.error()); + Anchor = *ExtendRight; + } + auto Multiply = buildBinaryAfter(Builder, *Anchor, InstructionOpcode::MUL, *Product, + *ExtendedLeft, *ExtendedRight); + if (!Multiply) + return std::unexpected(Multiply.error()); + auto Extract = buildImmediateAfter(Builder, **Multiply, + Signed ? InstructionOpcode::SRAI : InstructionOpcode::SRLI, + DestinationRegister, *Product, 32); + if (!Extract) + return std::unexpected(Extract.error()); + return {}; + } + + [[nodiscard]] static Expected selectIntegerMinMax(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target, + std::optional Type, + GenericOpcode Generic) { + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const MachineOperand *LeftOperand = Instruction.operandAt(1); + const MachineOperand *RightOperand = Instruction.operandAt(2); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); + const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf(); + const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf(); + if (!Type || Destination == nullptr || Left == nullptr || Right == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V integer min/max has invalid operands"); + auto I1 = LLT::makeInteger(1); + if (!I1) + return std::unexpected(I1.error()); + auto Condition = createTemporary(Function, *I1, Target); + if (!Condition) + return std::unexpected(Condition.error()); + const Register DestinationRegister = *Destination; + const Register LeftRegister = *Left; + const Register RightRegister = *Right; + const bool Unsigned = Generic == GenericOpcode::G_UMIN || Generic == GenericOpcode::G_UMAX; + auto NormalizedLeft = + normalizeIntegerBefore(Instruction, Function, Target, LeftRegister, *Type, !Unsigned); + auto NormalizedRight = + normalizeIntegerBefore(Instruction, Function, Target, RightRegister, *Type, !Unsigned); + if (!NormalizedLeft || !NormalizedRight) + return std::unexpected(!NormalizedLeft ? NormalizedLeft.error() : NormalizedRight.error()); + auto SetCondition = Instruction.operandAt(0)->setRegister(*Condition); + if (!SetCondition) + return std::unexpected(SetCondition.error()); + auto SetLeft = Instruction.operandAt(1)->setRegister(*NormalizedLeft); + if (!SetLeft) + return std::unexpected(SetLeft.error()); + auto SetRight = Instruction.operandAt(2)->setRegister(*NormalizedRight); + if (!SetRight) + return std::unexpected(SetRight.error()); + auto SetCompare = Instruction.setOpcode( + targetOpcode(Unsigned ? InstructionOpcode::SLTU : InstructionOpcode::SLT)); + if (!SetCompare) + return std::unexpected(SetCompare.error()); + const bool IsMaximum = Generic == GenericOpcode::G_SMAX || Generic == GenericOpcode::G_UMAX; + const Register TrueValue = IsMaximum ? RightRegister : LeftRegister; + const Register FalseValue = IsMaximum ? LeftRegister : RightRegister; + MachineIRBuilder Builder(Function, Target); + auto Selected = buildGPRSelectAfter(Builder, Instruction, Function, Target, *Type, + DestinationRegister, *Condition, TrueValue, FalseValue); + if (!Selected) + return std::unexpected(Selected.error()); + return {}; + } + + [[nodiscard]] static Expected selectValue(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target, + std::optional Type) { + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const MachineOperand *ConditionOperand = Instruction.operandAt(1); + const MachineOperand *TrueOperand = Instruction.operandAt(2); + const MachineOperand *FalseOperand = Instruction.operandAt(3); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); + const Register *Condition = + ConditionOperand == nullptr ? nullptr : ConditionOperand->getIf(); + const Register *TrueValue = TrueOperand == nullptr ? nullptr : TrueOperand->getIf(); + const Register *FalseValue = + FalseOperand == nullptr ? nullptr : FalseOperand->getIf(); + if (!Type || Destination == nullptr || Condition == nullptr || TrueValue == nullptr || + FalseValue == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V value selection has invalid operands"); + } + const Register DestinationRegister = *Destination; + const Register TrueRegister = *TrueValue; + const Register FalseRegister = *FalseValue; + auto I1 = LLT::makeInteger(1); + if (!I1) + return std::unexpected(I1.error()); + auto NormalizedCondition = createTemporary(Function, *I1, Target); + if (!NormalizedCondition) + return std::unexpected(NormalizedCondition.error()); + auto SetCondition = Instruction.operandAt(0)->setRegister(*NormalizedCondition); + if (!SetCondition) + return std::unexpected(SetCondition.error()); + auto EraseFalse = Instruction.eraseOperand(3); + if (!EraseFalse) + return std::unexpected(EraseFalse.error()); + auto EraseTrue = Instruction.eraseOperand(2); + if (!EraseTrue) + return std::unexpected(EraseTrue.error()); + auto Mask = appendImmediate(Instruction, 1); + if (!Mask) + return std::unexpected(Mask.error()); + auto SetNormalize = Instruction.setOpcode(targetOpcode(InstructionOpcode::ANDI)); + if (!SetNormalize) + return std::unexpected(SetNormalize.error()); + + MachineIRBuilder Builder(Function, Target); + if (!Type->isFloat()) { + auto Selected = + buildGPRSelectAfter(Builder, Instruction, Function, Target, *Type, DestinationRegister, + *NormalizedCondition, TrueRegister, FalseRegister); + if (!Selected) + return std::unexpected(Selected.error()); + return {}; + } + + auto IntegerType = LLT::makeInteger(Type->scalarBitWidth()); + if (!IntegerType) + return std::unexpected(IntegerType.error()); + auto IntegerTrue = createTemporary(Function, *IntegerType, Target); + auto IntegerFalse = createTemporary(Function, *IntegerType, Target); + auto IntegerResult = createTemporary(Function, *IntegerType, Target); + if (!IntegerTrue || !IntegerFalse || !IntegerResult) { + return std::unexpected(!IntegerTrue + ? IntegerTrue.error() + : (!IntegerFalse ? IntegerFalse.error() : IntegerResult.error())); + } + const InstructionOpcode ToInteger = + Type->scalarBitWidth() == 32 ? InstructionOpcode::FMV_X_W : InstructionOpcode::FMV_X_D; + const InstructionOpcode ToFloat = + Type->scalarBitWidth() == 32 ? InstructionOpcode::FMV_W_X : InstructionOpcode::FMV_D_X; + auto MoveTrue = buildUnaryAfter(Builder, Instruction, ToInteger, *IntegerTrue, TrueRegister); + if (!MoveTrue) + return std::unexpected(MoveTrue.error()); + auto MoveFalse = buildUnaryAfter(Builder, **MoveTrue, ToInteger, *IntegerFalse, FalseRegister); + if (!MoveFalse) + return std::unexpected(MoveFalse.error()); + auto Selected = + buildGPRSelectAfter(Builder, **MoveFalse, Function, Target, *IntegerType, *IntegerResult, + *NormalizedCondition, *IntegerTrue, *IntegerFalse); + if (!Selected) + return std::unexpected(Selected.error()); + auto MoveResult = + buildUnaryAfter(Builder, **Selected, ToFloat, DestinationRegister, *IntegerResult); + if (!MoveResult) + return std::unexpected(MoveResult.error()); + return {}; + } + + [[nodiscard]] static Expected selectAbsoluteValue(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target, + std::optional Type) { + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const MachineOperand *SourceOperand = Instruction.operandAt(1); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); + const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); + if (!Type || Destination == nullptr || Source == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V absolute value has invalid operands"); + const Register DestinationRegister = *Destination; + const Register SourceRegister = *Source; + auto NormalizedSource = + normalizeIntegerBefore(Instruction, Function, Target, SourceRegister, *Type, true); + if (!NormalizedSource) + return std::unexpected(NormalizedSource.error()); + auto SignMask = createTemporary(Function, *Type, Target); + auto Flipped = createTemporary(Function, *Type, Target); + if (!SignMask || !Flipped) + return std::unexpected(!SignMask ? SignMask.error() : Flipped.error()); + auto SetMask = Instruction.operandAt(0)->setRegister(*SignMask); + if (!SetMask) + return std::unexpected(SetMask.error()); + auto SetSource = Instruction.operandAt(1)->setRegister(*NormalizedSource); + if (!SetSource) + return std::unexpected(SetSource.error()); + const std::uint32_t ContainerBits = Type->scalarBitWidth() <= 32 ? 32 : 64; + auto Shift = appendImmediate(Instruction, ContainerBits - 1U); + if (!Shift) + return std::unexpected(Shift.error()); + const bool IsWord = Type->scalarBitWidth() <= 32; + auto SetShift = Instruction.setOpcode( + targetOpcode(IsWord ? InstructionOpcode::SRAIW : InstructionOpcode::SRAI)); + if (!SetShift) + return std::unexpected(SetShift.error()); + MachineIRBuilder Builder(Function, Target); + auto Flip = buildBinaryAfter(Builder, Instruction, InstructionOpcode::XOR, *Flipped, + *NormalizedSource, *SignMask); + if (!Flip) + return std::unexpected(Flip.error()); + auto Result = + buildBinaryAfter(Builder, **Flip, IsWord ? InstructionOpcode::SUBW : InstructionOpcode::SUB, + DestinationRegister, *Flipped, *SignMask); + if (!Result) + return std::unexpected(Result.error()); + return {}; + } + + [[nodiscard]] static Expected> + buildSignedOverflowAfter(MachineIRBuilder &Builder, MachineInstr &Anchor, + MachineFunction &Function, const TargetDescription &Target, LLT Type, + Register Result, Register Left, Register Right, bool IsAdd) { + auto FirstDifference = createTemporary(Function, Type, Target); + auto SecondDifference = createTemporary(Function, Type, Target); + auto SignBits = createTemporary(Function, Type, Target); + auto I1 = LLT::makeInteger(1); + if (!I1) + return std::unexpected(I1.error()); + auto Overflow = createTemporary(Function, *I1, Target); + if (!FirstDifference || !SecondDifference || !SignBits || !Overflow) { + return std::unexpected(!FirstDifference ? FirstDifference.error() + : !SecondDifference ? SecondDifference.error() + : !SignBits ? SignBits.error() + : Overflow.error()); + } + const Register FirstLeft = IsAdd ? Result : Left; + const Register FirstRight = IsAdd ? Left : Right; + const Register SecondLeft = Result; + const Register SecondRight = IsAdd ? Right : Left; + auto First = buildBinaryAfter(Builder, Anchor, InstructionOpcode::XOR, *FirstDifference, + FirstLeft, FirstRight); + if (!First) + return std::unexpected(First.error()); + auto Second = buildBinaryAfter(Builder, **First, InstructionOpcode::XOR, *SecondDifference, + SecondLeft, SecondRight); + if (!Second) + return std::unexpected(Second.error()); + auto Both = buildBinaryAfter(Builder, **Second, InstructionOpcode::AND, *SignBits, + *FirstDifference, *SecondDifference); + if (!Both) + return std::unexpected(Both.error()); + const InstructionOpcode Shift = + Type.scalarBitWidth() == 32 ? InstructionOpcode::SRLIW : InstructionOpcode::SRLI; + auto Extract = buildImmediateAfter(Builder, **Both, Shift, *Overflow, *SignBits, + Type.scalarBitWidth() - 1U); + if (!Extract) + return std::unexpected(Extract.error()); + return std::pair{*Overflow, *Extract}; + } + + [[nodiscard]] static Expected + selectOverflow(MachineInstr &Instruction, MachineFunction &Function, + const TargetDescription &Target, std::optional Type, GenericOpcode Generic) { + const MachineOperand *ResultOperand = Instruction.operandAt(0); + const MachineOperand *OverflowOperand = Instruction.operandAt(1); + const MachineOperand *LeftOperand = Instruction.operandAt(2); + const MachineOperand *RightOperand = Instruction.operandAt(3); + const Register *Result = ResultOperand == nullptr ? nullptr : ResultOperand->getIf(); + const Register *Overflow = + OverflowOperand == nullptr ? nullptr : OverflowOperand->getIf(); + const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf(); + const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf(); + if (!Type || Result == nullptr || Overflow == nullptr || Left == nullptr || Right == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V overflow operation has invalid operands"); + } + const Register ResultRegister = *Result; + const Register OverflowRegister = *Overflow; + const Register LeftRegister = *Left; + const Register RightRegister = *Right; + auto RemoveOverflow = Instruction.eraseOperand(1); + if (!RemoveOverflow) + return std::unexpected(RemoveOverflow.error()); + const bool IsWord = Type->scalarBitWidth() <= 32; + const bool IsAdd = Generic == GenericOpcode::G_UADDO || Generic == GenericOpcode::G_SADDO; + const bool IsSub = Generic == GenericOpcode::G_USUBO || Generic == GenericOpcode::G_SSUBO; + const bool IsUnsigned = Generic == GenericOpcode::G_UADDO || + Generic == GenericOpcode::G_USUBO || Generic == GenericOpcode::G_UMULO; + const InstructionOpcode Arithmetic = + IsAdd ? (IsWord ? InstructionOpcode::ADDW : InstructionOpcode::ADD) + : IsSub ? (IsWord ? InstructionOpcode::SUBW : InstructionOpcode::SUB) + : (IsWord ? InstructionOpcode::MULW : InstructionOpcode::MUL); + auto SetArithmetic = Instruction.setOpcode(targetOpcode(Arithmetic)); + if (!SetArithmetic) + return std::unexpected(SetArithmetic.error()); + MachineIRBuilder Builder(Function, Target); + if (IsUnsigned && IsAdd) { + auto Carry = buildBinaryAfter(Builder, Instruction, InstructionOpcode::SLTU, OverflowRegister, + ResultRegister, LeftRegister); + if (!Carry) + return std::unexpected(Carry.error()); + return {}; + } + if (IsUnsigned && IsSub) { + auto Borrow = buildBinaryAfter(Builder, Instruction, InstructionOpcode::SLTU, + OverflowRegister, LeftRegister, RightRegister); + if (!Borrow) + return std::unexpected(Borrow.error()); + return {}; + } + if (IsAdd || IsSub) { + auto FirstDifference = createTemporary(Function, *Type, Target); + auto SecondDifference = createTemporary(Function, *Type, Target); + auto SignBits = createTemporary(Function, *Type, Target); + if (!FirstDifference || !SecondDifference || !SignBits) { + return std::unexpected( + !FirstDifference ? FirstDifference.error() + : (!SecondDifference ? SecondDifference.error() : SignBits.error())); + } + const Register FirstLeft = IsAdd ? ResultRegister : LeftRegister; + const Register FirstRight = IsAdd ? LeftRegister : RightRegister; + const Register SecondLeft = ResultRegister; + const Register SecondRight = IsAdd ? RightRegister : LeftRegister; + auto First = buildBinaryAfter(Builder, Instruction, InstructionOpcode::XOR, *FirstDifference, + FirstLeft, FirstRight); + if (!First) + return std::unexpected(First.error()); + auto Second = buildBinaryAfter(Builder, **First, InstructionOpcode::XOR, *SecondDifference, + SecondLeft, SecondRight); + if (!Second) + return std::unexpected(Second.error()); + auto Both = buildBinaryAfter(Builder, **Second, InstructionOpcode::AND, *SignBits, + *FirstDifference, *SecondDifference); + if (!Both) + return std::unexpected(Both.error()); + auto OverflowResult = buildImmediateAfter( + Builder, **Both, IsWord ? InstructionOpcode::SRLIW : InstructionOpcode::SRLI, + OverflowRegister, *SignBits, Type->scalarBitWidth() - 1U); + if (!OverflowResult) + return std::unexpected(OverflowResult.error()); + return {}; + } + + auto I64 = LLT::makeInteger(64); + if (!I64) + return std::unexpected(I64.error()); + auto Zero = physicalRegister(PhysicalRegisterID::X0); + if (!Zero) + return std::unexpected(Zero.error()); + if (!IsWord) { + auto High = createTemporary(Function, *I64, Target); + if (!High) + return std::unexpected(High.error()); + auto HighProduct = buildBinaryAfter( + Builder, Instruction, IsUnsigned ? InstructionOpcode::MULHU : InstructionOpcode::MULH, + *High, LeftRegister, RightRegister); + if (!HighProduct) + return std::unexpected(HighProduct.error()); + if (IsUnsigned) { + auto OverflowResult = + buildBinaryAfter(Builder, **HighProduct, InstructionOpcode::SLTU, OverflowRegister, + Register::fromPhysical(*Zero), *High); + if (!OverflowResult) + return std::unexpected(OverflowResult.error()); + return {}; + } + auto ExpectedHigh = createTemporary(Function, *I64, Target); + auto Difference = createTemporary(Function, *I64, Target); + if (!ExpectedHigh || !Difference) + return std::unexpected(!ExpectedHigh ? ExpectedHigh.error() : Difference.error()); + auto Sign = buildImmediateAfter(Builder, **HighProduct, InstructionOpcode::SRAI, + *ExpectedHigh, ResultRegister, 63); + if (!Sign) + return std::unexpected(Sign.error()); + auto Compare = buildBinaryAfter(Builder, **Sign, InstructionOpcode::XOR, *Difference, *High, + *ExpectedHigh); + if (!Compare) + return std::unexpected(Compare.error()); + auto OverflowResult = + buildBinaryAfter(Builder, **Compare, InstructionOpcode::SLTU, OverflowRegister, + Register::fromPhysical(*Zero), *Difference); + if (!OverflowResult) + return std::unexpected(OverflowResult.error()); + return {}; + } + + auto ExtendedLeft = createTemporary(Function, *I64, Target); + auto ExtendedRight = createTemporary(Function, *I64, Target); + auto FullProduct = createTemporary(Function, *I64, Target); + if (!ExtendedLeft || !ExtendedRight || !FullProduct) { + return std::unexpected(!ExtendedLeft + ? ExtendedLeft.error() + : (!ExtendedRight ? ExtendedRight.error() : FullProduct.error())); + } + MachineInstr *Anchor = &Instruction; + if (IsUnsigned) { + auto ShiftedLeft = createTemporary(Function, *I64, Target); + auto ShiftedRight = createTemporary(Function, *I64, Target); + if (!ShiftedLeft || !ShiftedRight) + return std::unexpected(!ShiftedLeft ? ShiftedLeft.error() : ShiftedRight.error()); + auto LeftShift = buildImmediateAfter(Builder, *Anchor, InstructionOpcode::SLLI, *ShiftedLeft, + LeftRegister, 32); + if (!LeftShift) + return std::unexpected(LeftShift.error()); + auto LeftExtend = buildImmediateAfter(Builder, **LeftShift, InstructionOpcode::SRLI, + *ExtendedLeft, *ShiftedLeft, 32); + if (!LeftExtend) + return std::unexpected(LeftExtend.error()); + auto RightShift = buildImmediateAfter(Builder, **LeftExtend, InstructionOpcode::SLLI, + *ShiftedRight, RightRegister, 32); + if (!RightShift) + return std::unexpected(RightShift.error()); + auto RightExtend = buildImmediateAfter(Builder, **RightShift, InstructionOpcode::SRLI, + *ExtendedRight, *ShiftedRight, 32); + if (!RightExtend) + return std::unexpected(RightExtend.error()); + Anchor = *RightExtend; + } else { + auto LeftExtend = buildImmediateAfter(Builder, *Anchor, InstructionOpcode::ADDIW, + *ExtendedLeft, LeftRegister, 0); + if (!LeftExtend) + return std::unexpected(LeftExtend.error()); + auto RightExtend = buildImmediateAfter(Builder, **LeftExtend, InstructionOpcode::ADDIW, + *ExtendedRight, RightRegister, 0); + if (!RightExtend) + return std::unexpected(RightExtend.error()); + Anchor = *RightExtend; + } + auto Product = buildBinaryAfter(Builder, *Anchor, InstructionOpcode::MUL, *FullProduct, + *ExtendedLeft, *ExtendedRight); + if (!Product) + return std::unexpected(Product.error()); + if (IsUnsigned) { + auto High = createTemporary(Function, *I64, Target); + if (!High) + return std::unexpected(High.error()); + auto ExtractHigh = + buildImmediateAfter(Builder, **Product, InstructionOpcode::SRLI, *High, *FullProduct, 32); + if (!ExtractHigh) + return std::unexpected(ExtractHigh.error()); + auto OverflowResult = + buildBinaryAfter(Builder, **ExtractHigh, InstructionOpcode::SLTU, OverflowRegister, + Register::fromPhysical(*Zero), *High); + if (!OverflowResult) + return std::unexpected(OverflowResult.error()); + return {}; + } + auto Difference = createTemporary(Function, *I64, Target); + if (!Difference) + return std::unexpected(Difference.error()); + auto Compare = buildBinaryAfter(Builder, **Product, InstructionOpcode::XOR, *Difference, + *FullProduct, ResultRegister); + if (!Compare) + return std::unexpected(Compare.error()); + auto OverflowResult = + buildBinaryAfter(Builder, **Compare, InstructionOpcode::SLTU, OverflowRegister, + Register::fromPhysical(*Zero), *Difference); + if (!OverflowResult) + return std::unexpected(OverflowResult.error()); + return {}; + } + + [[nodiscard]] static Expected selectExtendedOverflow(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target, + std::optional Type, + GenericOpcode Generic) { + const MachineOperand *ResultOperand = Instruction.operandAt(0); + const MachineOperand *CarryOutOperand = Instruction.operandAt(1); + const MachineOperand *LeftOperand = Instruction.operandAt(2); + const MachineOperand *RightOperand = Instruction.operandAt(3); + const MachineOperand *CarryInOperand = Instruction.operandAt(4); + const Register *Result = ResultOperand == nullptr ? nullptr : ResultOperand->getIf(); + const Register *CarryOut = + CarryOutOperand == nullptr ? nullptr : CarryOutOperand->getIf(); + const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf(); + const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf(); + const Register *CarryIn = + CarryInOperand == nullptr ? nullptr : CarryInOperand->getIf(); + if (!Type || Result == nullptr || CarryOut == nullptr || Left == nullptr || Right == nullptr || + CarryIn == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V extended overflow operation has invalid operands"); + } + const Register ResultRegister = *Result; + const Register CarryOutRegister = *CarryOut; + const Register LeftRegister = *Left; + const Register RightRegister = *Right; + const Register CarryInRegister = *CarryIn; + const bool IsAdd = Generic == GenericOpcode::G_UADDE || Generic == GenericOpcode::G_SADDE; + const bool IsUnsigned = Generic == GenericOpcode::G_UADDE || Generic == GenericOpcode::G_USUBE; + const bool IsWord = Type->scalarBitWidth() <= 32; + const InstructionOpcode Arithmetic = + IsAdd ? (IsWord ? InstructionOpcode::ADDW : InstructionOpcode::ADD) + : (IsWord ? InstructionOpcode::SUBW : InstructionOpcode::SUB); + auto Partial = createTemporary(Function, *Type, Target); + auto I1 = LLT::makeInteger(1); + if (!Partial || !I1) + return std::unexpected(!Partial ? Partial.error() : I1.error()); + auto NormalizedCarry = createTemporary(Function, *I1, Target); + if (!NormalizedCarry) + return std::unexpected(NormalizedCarry.error()); + auto SetPartial = Instruction.operandAt(0)->setRegister(*Partial); + if (!SetPartial) + return std::unexpected(SetPartial.error()); + auto RemoveCarryOut = Instruction.eraseOperand(1); + if (!RemoveCarryOut) + return std::unexpected(RemoveCarryOut.error()); + auto RemoveCarryIn = Instruction.eraseOperand(3); + if (!RemoveCarryIn) + return std::unexpected(RemoveCarryIn.error()); + auto SetArithmetic = Instruction.setOpcode(targetOpcode(Arithmetic)); + if (!SetArithmetic) + return std::unexpected(SetArithmetic.error()); + MachineIRBuilder Builder(Function, Target); + auto Normalize = buildImmediateAfter(Builder, Instruction, InstructionOpcode::ANDI, + *NormalizedCarry, CarryInRegister, 1); + if (!Normalize) + return std::unexpected(Normalize.error()); + auto Final = buildBinaryAfter(Builder, **Normalize, Arithmetic, ResultRegister, *Partial, + *NormalizedCarry); + if (!Final) + return std::unexpected(Final.error()); + + if (IsUnsigned) { + auto FirstCarry = createTemporary(Function, *I1, Target); + auto SecondCarry = createTemporary(Function, *I1, Target); + if (!FirstCarry || !SecondCarry) + return std::unexpected(!FirstCarry ? FirstCarry.error() : SecondCarry.error()); + auto First = + buildBinaryAfter(Builder, **Final, InstructionOpcode::SLTU, *FirstCarry, + IsAdd ? *Partial : LeftRegister, IsAdd ? LeftRegister : RightRegister); + if (!First) + return std::unexpected(First.error()); + auto Second = + buildBinaryAfter(Builder, **First, InstructionOpcode::SLTU, *SecondCarry, + IsAdd ? ResultRegister : *Partial, IsAdd ? *Partial : *NormalizedCarry); + if (!Second) + return std::unexpected(Second.error()); + auto Combined = buildBinaryAfter(Builder, **Second, InstructionOpcode::OR, CarryOutRegister, + *FirstCarry, *SecondCarry); + if (!Combined) + return std::unexpected(Combined.error()); + return {}; + } + + auto FirstOverflow = buildSignedOverflowAfter(Builder, **Final, Function, Target, *Type, + *Partial, LeftRegister, RightRegister, IsAdd); + if (!FirstOverflow) + return std::unexpected(FirstOverflow.error()); + auto SecondOverflow = + buildSignedOverflowAfter(Builder, *FirstOverflow->second, Function, Target, *Type, + ResultRegister, *Partial, *NormalizedCarry, IsAdd); + if (!SecondOverflow) + return std::unexpected(SecondOverflow.error()); + auto Combined = buildBinaryAfter(Builder, *SecondOverflow->second, InstructionOpcode::OR, + CarryOutRegister, FirstOverflow->first, SecondOverflow->first); + if (!Combined) + return std::unexpected(Combined.error()); + return {}; + } + + [[nodiscard]] static Expected + selectAtomicRMW(MachineInstr &Instruction, MachineFunction &Function, + const TargetDescription &Target, std::optional Type, GenericOpcode Generic) { + const MachineOperand *AddressOperand = Instruction.operandAt(1); + const MachineOperand *ValueOperand = Instruction.operandAt(2); + const Register *Address = + AddressOperand == nullptr ? nullptr : AddressOperand->getIf(); + const Register *Value = ValueOperand == nullptr ? nullptr : ValueOperand->getIf(); + if (!Type || Address == nullptr || Value == nullptr) + return makeUnexpected(ErrorCode::VerificationError, "RISC-V atomic RMW has invalid operands"); + Register SelectedValue = *Value; + const bool IsWord = Type->scalarBitWidth() == 32; + if (Generic == GenericOpcode::G_ATOMICRMW_SUB) { + auto Negated = createTemporary(Function, *Type, Target); + auto Zero = physicalRegister(PhysicalRegisterID::X0); + if (!Negated || !Zero) + return std::unexpected(!Negated ? Negated.error() : Zero.error()); + MachineIRBuilder Builder(Function, Target); + auto Point = Builder.setInsertPointBefore(Instruction); + if (!Point) + return std::unexpected(Point.error()); + auto Pending = Builder.buildInstr( + targetOpcode(IsWord ? InstructionOpcode::SUBW : InstructionOpcode::SUB)); + if (!Pending) + return std::unexpected(Pending.error()); + auto Def = Pending->addDef(*Negated); + if (!Def) + return std::unexpected(Def.error()); + auto Left = Pending->addUse(Register::fromPhysical(*Zero)); + if (!Left) + return std::unexpected(Left.error()); + auto Right = Pending->addUse(*Value); + if (!Right) + return std::unexpected(Right.error()); + auto Built = Pending->build(); + if (!Built) + return std::unexpected(Built.error()); + SelectedValue = *Negated; + } + const auto SelectOpcode = [Generic, IsWord]() -> InstructionOpcode { + switch (Generic) { + case GenericOpcode::G_ATOMICRMW_XCHG: + return IsWord ? InstructionOpcode::AMOSWAP_W : InstructionOpcode::AMOSWAP_D; + case GenericOpcode::G_ATOMICRMW_ADD: + case GenericOpcode::G_ATOMICRMW_SUB: + return IsWord ? InstructionOpcode::AMOADD_W : InstructionOpcode::AMOADD_D; + case GenericOpcode::G_ATOMICRMW_AND: + return IsWord ? InstructionOpcode::AMOAND_W : InstructionOpcode::AMOAND_D; + case GenericOpcode::G_ATOMICRMW_OR: + return IsWord ? InstructionOpcode::AMOOR_W : InstructionOpcode::AMOOR_D; + case GenericOpcode::G_ATOMICRMW_XOR: + return IsWord ? InstructionOpcode::AMOXOR_W : InstructionOpcode::AMOXOR_D; + case GenericOpcode::G_ATOMICRMW_MAX: + return IsWord ? InstructionOpcode::AMOMAX_W : InstructionOpcode::AMOMAX_D; + case GenericOpcode::G_ATOMICRMW_MIN: + return IsWord ? InstructionOpcode::AMOMIN_W : InstructionOpcode::AMOMIN_D; + case GenericOpcode::G_ATOMICRMW_UMAX: + return IsWord ? InstructionOpcode::AMOMAXU_W : InstructionOpcode::AMOMAXU_D; + case GenericOpcode::G_ATOMICRMW_UMIN: + return IsWord ? InstructionOpcode::AMOMINU_W : InstructionOpcode::AMOMINU_D; + default: + return InstructionOpcode::Invalid; + } + }; + const InstructionOpcode Selected = SelectOpcode(); + if (Selected == InstructionOpcode::Invalid) + return makeUnexpected(ErrorCode::InternalError, + "RISC-V atomic RMW selection reached an unhandled opcode"); + auto SetValue = Instruction.operandAt(1)->setRegister(SelectedValue); + if (!SetValue) + return std::unexpected(SetValue.error()); + auto SetAddress = Instruction.operandAt(2)->setRegister(*Address); + if (!SetAddress) + return std::unexpected(SetAddress.error()); + return Instruction.setOpcode(targetOpcode(Selected)); + } + + [[nodiscard]] static Expected selectIntegerExtension(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target, + GenericOpcode Generic, + std::optional Type) { + const std::optional SourceType = operandType(Function, Instruction, 1); + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const MachineOperand *SourceOperand = Instruction.operandAt(1); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); + const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); + if (!Type || !SourceType || Destination == nullptr || Source == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V integer extension has invalid register operands"); + const Register DestinationRegister = *Destination; + if (Generic == GenericOpcode::G_ANYEXT) + return Instruction.setOpcode(opcode(GenericOpcode::COPY)); + + const std::uint32_t DestinationBits = Type->scalarBitWidth(); + const std::uint32_t SourceBits = SourceType->scalarBitWidth(); + if (Generic == GenericOpcode::G_SEXT && DestinationBits == 64 && SourceBits == 32) { + auto ImmediateResult = appendImmediate(Instruction, 0); + if (!ImmediateResult) + return std::unexpected(ImmediateResult.error()); + return Instruction.setOpcode(targetOpcode(InstructionOpcode::ADDIW)); + } + + const std::uint32_t ContainerBits = DestinationBits <= 32 ? 32 : 64; + const std::int64_t Shift = static_cast(ContainerBits - SourceBits); + const InstructionOpcode ShiftLeft = + ContainerBits == 32 ? InstructionOpcode::SLLIW : InstructionOpcode::SLLI; + const InstructionOpcode ShiftRight = + Generic == GenericOpcode::G_SEXT + ? (ContainerBits == 32 ? InstructionOpcode::SRAIW : InstructionOpcode::SRAI) + : (ContainerBits == 32 ? InstructionOpcode::SRLIW : InstructionOpcode::SRLI); + auto Temporary = createTemporary(Function, *Type, Target); + if (!Temporary) + return std::unexpected(Temporary.error()); + auto SetDestination = Instruction.operandAt(0)->setRegister(*Temporary); + if (!SetDestination) + return std::unexpected(SetDestination.error()); + auto SetOpcode = Instruction.setOpcode(targetOpcode(ShiftLeft)); + if (!SetOpcode) + return std::unexpected(SetOpcode.error()); + auto ShiftImmediate = appendImmediate(Instruction, Shift); + if (!ShiftImmediate) + return std::unexpected(ShiftImmediate.error()); + + MachineIRBuilder Builder(Function, Target); + auto Shifted = buildImmediateAfter(Builder, Instruction, ShiftRight, DestinationRegister, + *Temporary, Shift); + if (!Shifted) + return std::unexpected(Shifted.error()); + return {}; + } + + [[nodiscard]] static Expected selectSignExtendInRegister(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target, + std::optional Type) { + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const MachineOperand *SourceOperand = Instruction.operandAt(1); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); + const Register *Source = SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); + std::optional Width; + if (const MachineOperand *WidthOperand = Instruction.operandAt(2); WidthOperand != nullptr) { + auto ParsedWidth = signedImmediate(*WidthOperand); + if (ParsedWidth) + Width = *ParsedWidth; + } + if (!Type || Destination == nullptr || Source == nullptr || !Width || *Width <= 0 || + static_cast(*Width) > Type->scalarBitWidth()) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V sign-extension-in-register has invalid operands"); + const Register DestinationRegister = *Destination; + if (static_cast(*Width) == Type->scalarBitWidth()) { + auto EraseWidth = Instruction.eraseOperand(2); + if (!EraseWidth) + return std::unexpected(EraseWidth.error()); + return Instruction.setOpcode(opcode(GenericOpcode::COPY)); + } + + const std::uint32_t TypeBits = Type->scalarBitWidth(); + if (TypeBits == 64 && *Width == 32) { + auto ImmediateResult = replaceImmediate(Instruction, 2, 0); + if (!ImmediateResult) + return std::unexpected(ImmediateResult.error()); + return Instruction.setOpcode(targetOpcode(InstructionOpcode::ADDIW)); + } + const std::uint32_t ContainerBits = TypeBits <= 32 ? 32 : 64; + const std::int64_t Shift = static_cast(ContainerBits) - *Width; + const InstructionOpcode ShiftLeft = + ContainerBits == 32 ? InstructionOpcode::SLLIW : InstructionOpcode::SLLI; + const InstructionOpcode ShiftRight = + ContainerBits == 32 ? InstructionOpcode::SRAIW : InstructionOpcode::SRAI; + auto Temporary = createTemporary(Function, *Type, Target); + if (!Temporary) + return std::unexpected(Temporary.error()); + auto SetDestination = Instruction.operandAt(0)->setRegister(*Temporary); + if (!SetDestination) + return std::unexpected(SetDestination.error()); + auto SetOpcode = Instruction.setOpcode(targetOpcode(ShiftLeft)); + if (!SetOpcode) + return std::unexpected(SetOpcode.error()); + auto ShiftImmediate = replaceImmediate(Instruction, 2, Shift); + if (!ShiftImmediate) + return std::unexpected(ShiftImmediate.error()); + + MachineIRBuilder Builder(Function, Target); + auto Shifted = buildImmediateAfter(Builder, Instruction, ShiftRight, DestinationRegister, + *Temporary, Shift); + if (!Shifted) + return std::unexpected(Shifted.error()); + return {}; + } + + [[nodiscard]] static Expected selectIntegerCompare(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target) { + const MachineOperand *PredicateMO = Instruction.operandAt(1); + const auto *Predicate = + PredicateMO == nullptr ? nullptr : PredicateMO->getIf(); + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const MachineOperand *LeftOperand = Instruction.operandAt(2); + const MachineOperand *RightOperand = Instruction.operandAt(3); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); + const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf(); + const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf(); + const std::optional LeftType = operandType(Function, Instruction, 2); + if (Predicate == nullptr || Predicate->integerPredicate() == nullptr || + Destination == nullptr || Left == nullptr || Right == nullptr || !LeftType) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V integer comparison has invalid operands"); + const IntegerPredicate Condition = *Predicate->integerPredicate(); + const Register DestinationRegister = *Destination; + const bool SignedOperands = Condition == IntegerPredicate::SignedGreaterThan || + Condition == IntegerPredicate::SignedGreaterEqual || + Condition == IntegerPredicate::SignedLessThan || + Condition == IntegerPredicate::SignedLessEqual; + auto LeftRegister = + normalizeIntegerBefore(Instruction, Function, Target, *Left, *LeftType, SignedOperands); + auto RightRegister = + normalizeIntegerBefore(Instruction, Function, Target, *Right, *LeftType, SignedOperands); + if (!LeftRegister || !RightRegister) + return std::unexpected(!LeftRegister ? LeftRegister.error() : RightRegister.error()); + auto ErasePredicate = Instruction.eraseOperand(1); + if (!ErasePredicate) + return std::unexpected(ErasePredicate.error()); + + const auto SetSources = [&Instruction](Register NewLeft, Register NewRight) -> Expected { + auto SetLeft = Instruction.operandAt(1)->setRegister(NewLeft); + if (!SetLeft) + return std::unexpected(SetLeft.error()); + return Instruction.operandAt(2)->setRegister(NewRight); + }; + const auto StrictOpcode = [](IntegerPredicate Value) -> std::optional { + switch (Value) { + case IntegerPredicate::SignedLessThan: + return InstructionOpcode::SLT; + case IntegerPredicate::UnsignedLessThan: + return InstructionOpcode::SLTU; + default: + return std::nullopt; + } + }; + + IntegerPredicate StrictCondition = Condition; + bool Swap = false; + bool Invert = false; + switch (Condition) { + case IntegerPredicate::Equal: + case IntegerPredicate::NotEqual: + break; + case IntegerPredicate::SignedGreaterThan: + StrictCondition = IntegerPredicate::SignedLessThan; + Swap = true; + break; + case IntegerPredicate::SignedGreaterEqual: + StrictCondition = IntegerPredicate::SignedLessThan; + Invert = true; + break; + case IntegerPredicate::SignedLessThan: + StrictCondition = IntegerPredicate::SignedLessThan; + break; + case IntegerPredicate::SignedLessEqual: + StrictCondition = IntegerPredicate::SignedLessThan; + Swap = true; + Invert = true; + break; + case IntegerPredicate::UnsignedGreaterThan: + StrictCondition = IntegerPredicate::UnsignedLessThan; + Swap = true; + break; + case IntegerPredicate::UnsignedGreaterEqual: + StrictCondition = IntegerPredicate::UnsignedLessThan; + Invert = true; + break; + case IntegerPredicate::UnsignedLessThan: + StrictCondition = IntegerPredicate::UnsignedLessThan; + break; + case IntegerPredicate::UnsignedLessEqual: + StrictCondition = IntegerPredicate::UnsignedLessThan; + Swap = true; + Invert = true; + break; + case IntegerPredicate::Count: + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V integer comparison predicate is invalid"); + } + + MachineIRBuilder Builder(Function, Target); + if (Condition == IntegerPredicate::Equal || Condition == IntegerPredicate::NotEqual) { + auto Temporary = createTemporary(Function, *LeftType, Target); + if (!Temporary) + return std::unexpected(Temporary.error()); + auto SetDestination = Instruction.operandAt(0)->setRegister(*Temporary); + if (!SetDestination) + return std::unexpected(SetDestination.error()); + auto Set = SetSources(*LeftRegister, *RightRegister); + if (!Set) + return std::unexpected(Set.error()); + auto SetOpcode = Instruction.setOpcode(targetOpcode(InstructionOpcode::XOR)); + if (!SetOpcode) + return std::unexpected(SetOpcode.error()); + if (Condition == IntegerPredicate::Equal) { + auto Result = buildImmediateAfter(Builder, Instruction, InstructionOpcode::SLTIU, + DestinationRegister, *Temporary, 1); + if (!Result) + return std::unexpected(Result.error()); + } else { + auto Zero = physicalRegister(PhysicalRegisterID::X0); + if (!Zero) + return std::unexpected(Zero.error()); + auto Result = + buildBinaryAfter(Builder, Instruction, InstructionOpcode::SLTU, DestinationRegister, + Register::fromPhysical(*Zero), *Temporary); + if (!Result) + return std::unexpected(Result.error()); + } + return {}; + } + + const std::optional OpcodeValue = StrictOpcode(StrictCondition); + if (!OpcodeValue) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V integer comparison predicate is invalid"); + if (Invert) { + auto Temporary = createTemporary(Function, *LeftType, Target); + if (!Temporary) + return std::unexpected(Temporary.error()); + auto SetDestination = Instruction.operandAt(0)->setRegister(*Temporary); + if (!SetDestination) + return std::unexpected(SetDestination.error()); + const Register Base = *Temporary; + if (Swap) { + auto Set = SetSources(*RightRegister, *LeftRegister); + if (!Set) + return std::unexpected(Set.error()); + } else { + auto Set = SetSources(*LeftRegister, *RightRegister); + if (!Set) + return std::unexpected(Set.error()); + } + auto SetOpcode = Instruction.setOpcode(targetOpcode(*OpcodeValue)); + if (!SetOpcode) + return std::unexpected(SetOpcode.error()); + auto Result = buildImmediateAfter(Builder, Instruction, InstructionOpcode::XORI, + DestinationRegister, Base, 1); + if (!Result) + return std::unexpected(Result.error()); + return {}; + } + if (Swap) { + auto Set = SetSources(*RightRegister, *LeftRegister); + if (!Set) + return std::unexpected(Set.error()); + } else { + auto Set = SetSources(*LeftRegister, *RightRegister); + if (!Set) + return std::unexpected(Set.error()); + } + return Instruction.setOpcode(targetOpcode(*OpcodeValue)); + } + + [[nodiscard]] static Expected selectFloatingCompare(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target) { + const MachineOperand *PredicateMO = Instruction.operandAt(1); + const auto *Predicate = + PredicateMO == nullptr ? nullptr : PredicateMO->getIf(); + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const MachineOperand *LeftOperand = Instruction.operandAt(2); + const MachineOperand *RightOperand = Instruction.operandAt(3); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); + const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf(); + const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf(); + const std::optional FloatType = operandType(Function, Instruction, 2); + const std::optional ResultType = resultType(Function, Instruction); + if (Predicate == nullptr || Predicate->floatingPredicate() == nullptr || + Destination == nullptr || Left == nullptr || Right == nullptr || !FloatType || !ResultType) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V floating comparison has invalid operands"); + const FloatingPredicate Condition = *Predicate->floatingPredicate(); + const Register DestinationRegister = *Destination; + const Register LeftRegister = *Left; + const Register RightRegister = *Right; + const InstructionOpcode EqualOpcode = + FloatType->scalarBitWidth() == 32 ? InstructionOpcode::FEQ_S : InstructionOpcode::FEQ_D; + const InstructionOpcode LessOpcode = + FloatType->scalarBitWidth() == 32 ? InstructionOpcode::FLT_S : InstructionOpcode::FLT_D; + const InstructionOpcode LessEqualOpcode = + FloatType->scalarBitWidth() == 32 ? InstructionOpcode::FLE_S : InstructionOpcode::FLE_D; + + auto ErasePredicate = Instruction.eraseOperand(1); + if (!ErasePredicate) + return std::unexpected(ErasePredicate.error()); + const auto SetBase = [&Instruction](InstructionOpcode OpcodeValue, Register Def, + Register NewLeft, Register NewRight) -> Expected { + auto SetDestination = Instruction.operandAt(0)->setRegister(Def); + if (!SetDestination) + return std::unexpected(SetDestination.error()); + auto SetLeft = Instruction.operandAt(1)->setRegister(NewLeft); + if (!SetLeft) + return std::unexpected(SetLeft.error()); + auto SetRight = Instruction.operandAt(2)->setRegister(NewRight); + if (!SetRight) + return std::unexpected(SetRight.error()); + return Instruction.setOpcode(targetOpcode(OpcodeValue)); + }; + const auto BuildInverted = [&](InstructionOpcode BaseOpcode, Register BaseLeft, + Register BaseRight) -> Expected { + auto Temporary = createTemporary(Function, *ResultType, Target); + if (!Temporary) + return std::unexpected(Temporary.error()); + auto Base = SetBase(BaseOpcode, *Temporary, BaseLeft, BaseRight); + if (!Base) + return std::unexpected(Base.error()); + MachineIRBuilder Builder(Function, Target); + auto Invert = buildImmediateAfter(Builder, Instruction, InstructionOpcode::XORI, + DestinationRegister, *Temporary, 1); + if (!Invert) + return std::unexpected(Invert.error()); + return {}; + }; + const auto BuildSelfAnd = [&](bool Invert) -> Expected { + auto First = createTemporary(Function, *ResultType, Target); + auto Second = createTemporary(Function, *ResultType, Target); + if (!First || !Second) + return std::unexpected((!First ? First.error() : Second.error())); + auto Base = SetBase(EqualOpcode, *First, LeftRegister, LeftRegister); + if (!Base) + return std::unexpected(Base.error()); + MachineIRBuilder Builder(Function, Target); + auto SecondInstruction = buildBinaryAfter(Builder, Instruction, EqualOpcode, *Second, + RightRegister, RightRegister); + if (!SecondInstruction) + return std::unexpected(SecondInstruction.error()); + Register CombinedDestination = DestinationRegister; + if (Invert) { + auto Temporary = createTemporary(Function, *ResultType, Target); + if (!Temporary) + return std::unexpected(Temporary.error()); + CombinedDestination = *Temporary; + } + auto Combined = buildBinaryAfter(Builder, **SecondInstruction, InstructionOpcode::AND, + CombinedDestination, *First, *Second); + if (!Combined) + return std::unexpected(Combined.error()); + if (!Invert) + return {}; + auto Inverted = buildImmediateAfter(Builder, **Combined, InstructionOpcode::XORI, + DestinationRegister, CombinedDestination, 1); + if (!Inverted) + return std::unexpected(Inverted.error()); + return {}; + }; + const auto BuildOrderedNotEqual = [&](bool Invert) -> Expected { + auto First = createTemporary(Function, *ResultType, Target); + auto Second = createTemporary(Function, *ResultType, Target); + if (!First || !Second) + return std::unexpected((!First ? First.error() : Second.error())); + auto Base = SetBase(LessOpcode, *First, LeftRegister, RightRegister); + if (!Base) + return std::unexpected(Base.error()); + MachineIRBuilder Builder(Function, Target); + auto SecondInstruction = + buildBinaryAfter(Builder, Instruction, LessOpcode, *Second, RightRegister, LeftRegister); + if (!SecondInstruction) + return std::unexpected(SecondInstruction.error()); + Register CombinedDestination = DestinationRegister; + if (Invert) { + auto Temporary = createTemporary(Function, *ResultType, Target); + if (!Temporary) + return std::unexpected(Temporary.error()); + CombinedDestination = *Temporary; + } + auto Combined = buildBinaryAfter(Builder, **SecondInstruction, InstructionOpcode::OR, + CombinedDestination, *First, *Second); + if (!Combined) + return std::unexpected(Combined.error()); + if (!Invert) + return {}; + auto Inverted = buildImmediateAfter(Builder, **Combined, InstructionOpcode::XORI, + DestinationRegister, CombinedDestination, 1); + if (!Inverted) + return std::unexpected(Inverted.error()); + return {}; + }; + + switch (Condition) { + case FloatingPredicate::AlwaysFalse: + case FloatingPredicate::AlwaysTrue: { + auto EraseRight = Instruction.eraseOperand(2); + if (!EraseRight) + return std::unexpected(EraseRight.error()); + auto EraseLeft = Instruction.eraseOperand(1); + if (!EraseLeft) + return std::unexpected(EraseLeft.error()); + auto Zero = physicalRegister(PhysicalRegisterID::X0); + if (!Zero) + return std::unexpected(Zero.error()); + auto InsertZero = insertPhysicalRegister(Instruction, 1, *Zero, RegisterOperandFlag::Use); + if (!InsertZero) + return std::unexpected(InsertZero.error()); + auto Immediate = + appendImmediate(Instruction, Condition == FloatingPredicate::AlwaysTrue ? 1 : 0); + if (!Immediate) + return std::unexpected(Immediate.error()); + return Instruction.setOpcode(targetOpcode(InstructionOpcode::ADDI)); + } + case FloatingPredicate::OrderedEqual: + return SetBase(EqualOpcode, DestinationRegister, LeftRegister, RightRegister); + case FloatingPredicate::OrderedGreaterThan: + return SetBase(LessOpcode, DestinationRegister, RightRegister, LeftRegister); + case FloatingPredicate::OrderedGreaterEqual: + return SetBase(LessEqualOpcode, DestinationRegister, RightRegister, LeftRegister); + case FloatingPredicate::OrderedLessThan: + return SetBase(LessOpcode, DestinationRegister, LeftRegister, RightRegister); + case FloatingPredicate::OrderedLessEqual: + return SetBase(LessEqualOpcode, DestinationRegister, LeftRegister, RightRegister); + case FloatingPredicate::OrderedNotEqual: + return BuildOrderedNotEqual(false); + case FloatingPredicate::Ordered: + return BuildSelfAnd(false); + case FloatingPredicate::Unordered: + return BuildSelfAnd(true); + case FloatingPredicate::UnorderedEqual: + return BuildOrderedNotEqual(true); + case FloatingPredicate::UnorderedGreaterThan: + return BuildInverted(LessEqualOpcode, LeftRegister, RightRegister); + case FloatingPredicate::UnorderedGreaterEqual: + return BuildInverted(LessOpcode, LeftRegister, RightRegister); + case FloatingPredicate::UnorderedLessThan: + return BuildInverted(LessEqualOpcode, RightRegister, LeftRegister); + case FloatingPredicate::UnorderedLessEqual: + return BuildInverted(LessOpcode, RightRegister, LeftRegister); + case FloatingPredicate::UnorderedNotEqual: + return BuildInverted(EqualOpcode, LeftRegister, RightRegister); + case FloatingPredicate::Count: + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V floating comparison predicate is invalid"); + } + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V floating comparison predicate is invalid"); + } + + [[nodiscard]] static Expected selectFloatToInteger(MachineInstr &Instruction, + MachineFunction &Function, + std::optional Type, bool Unsigned) { + const std::optional Source = operandType(Function, Instruction, 1); + if (!Type || !Source) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V floating-to-integer conversion has invalid types"); + auto Rounding = appendImmediate(Instruction, 1); + if (!Rounding) + return std::unexpected(Rounding.error()); + const bool IsWord = Type->scalarBitWidth() <= 32; + if (Source->scalarBitWidth() == 32) { + return Instruction.setOpcode(targetOpcode( + IsWord ? (Unsigned ? InstructionOpcode::FCVT_WU_S : InstructionOpcode::FCVT_W_S) + : (Unsigned ? InstructionOpcode::FCVT_LU_S : InstructionOpcode::FCVT_L_S))); + } + return Instruction.setOpcode(targetOpcode( + IsWord ? (Unsigned ? InstructionOpcode::FCVT_WU_D : InstructionOpcode::FCVT_W_D) + : (Unsigned ? InstructionOpcode::FCVT_LU_D : InstructionOpcode::FCVT_L_D))); + } + + [[nodiscard]] static Expected selectIntegerToFloat(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target, + std::optional Type, bool Unsigned) { + const std::optional Source = operandType(Function, Instruction, 1); + const MachineOperand *SourceOperand = Instruction.operandAt(1); + const Register *SourceRegister = + SourceOperand == nullptr ? nullptr : SourceOperand->getIf(); + if (!Type || !Source || SourceRegister == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V integer-to-floating conversion has invalid types"); + auto Normalized = + normalizeIntegerBefore(Instruction, Function, Target, *SourceRegister, *Source, !Unsigned); + if (!Normalized) + return std::unexpected(Normalized.error()); + auto SetSource = Instruction.operandAt(1)->setRegister(*Normalized); + if (!SetSource) + return std::unexpected(SetSource.error()); + auto Rounding = appendImmediate(Instruction, 0); + if (!Rounding) + return std::unexpected(Rounding.error()); + const bool IsWord = Source->scalarBitWidth() <= 32; + if (Type->scalarBitWidth() == 32) { + return Instruction.setOpcode(targetOpcode( + IsWord ? (Unsigned ? InstructionOpcode::FCVT_S_WU : InstructionOpcode::FCVT_S_W) + : (Unsigned ? InstructionOpcode::FCVT_S_LU : InstructionOpcode::FCVT_S_L))); + } + return Instruction.setOpcode(targetOpcode( + IsWord ? (Unsigned ? InstructionOpcode::FCVT_D_WU : InstructionOpcode::FCVT_D_W) + : (Unsigned ? InstructionOpcode::FCVT_D_LU : InstructionOpcode::FCVT_D_L))); + } + + [[nodiscard]] static Expected selectFloatingMinMax(MachineInstr &Instruction, + std::optional Type, + InstructionOpcode Single, + InstructionOpcode Double) { + if (Type && Type->isFloat() && Type->scalarBitWidth() == 32) + return Instruction.setOpcode(targetOpcode(Single)); + if (Type && Type->isFloat() && Type->scalarBitWidth() == 64) + return Instruction.setOpcode(targetOpcode(Double)); + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V cannot select this floating min/max type"); + } + + [[nodiscard]] static Expected selectFence(MachineInstr &Instruction) { + const auto Ordering = signedImmediate(*Instruction.operandAt(0)); + if (!Ordering) + return std::unexpected(Ordering.error()); + std::int64_t Pred = 0; + std::int64_t Succ = 0; + switch (*Ordering) { + case 4: + Pred = 1; + Succ = 3; + break; + case 5: + Pred = 3; + Succ = 2; + break; + case 6: + case 7: + Pred = 3; + Succ = 3; + break; + default: + return makeUnexpected(ErrorCode::Unsupported, "RISC-V fence ordering is unsupported"); + } + auto SetPred = replaceImmediate(Instruction, 0, Pred); + if (!SetPred) + return std::unexpected(SetPred.error()); + auto SetSucc = replaceImmediate(Instruction, 1, Succ); + if (!SetSucc) + return std::unexpected(SetSucc.error()); + return Instruction.setOpcode(targetOpcode(InstructionOpcode::FENCE)); + } + + [[nodiscard]] static Expected selectIndirectBranch(MachineInstr &Instruction) { + auto Zero = physicalRegister(PhysicalRegisterID::X0); + if (!Zero) + return std::unexpected(Zero.error()); + auto Inserted = insertPhysicalRegister(Instruction, 0, *Zero, RegisterOperandFlag::Define); + if (!Inserted) + return std::unexpected(Inserted.error()); + auto Immediate = appendImmediate(Instruction, 0); + if (!Immediate) + return std::unexpected(Immediate.error()); + return Instruction.setOpcode(targetOpcode(InstructionOpcode::JALR)); + } + + [[nodiscard]] static Expected selectTrap(MachineInstr &Instruction) { + while (Instruction.operandCount() != 0) { + auto Erased = Instruction.eraseOperand(Instruction.operandCount() - 1U); + if (!Erased) + return std::unexpected(Erased.error()); + } + return Instruction.setOpcode(targetOpcode(InstructionOpcode::EBREAK)); + } + + [[nodiscard]] static Expected selectBitcast(MachineInstr &Instruction, + MachineFunction &Function) { + const std::optional DestinationType = resultType(Function, Instruction); + const std::optional SourceType = operandType(Function, Instruction, 1); + if (!DestinationType || !SourceType) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V bitcast has invalid register types"); + if (DestinationType->isInteger() && SourceType->isFloat()) { + return Instruction.setOpcode(targetOpcode(SourceType->scalarBitWidth() == 32 + ? InstructionOpcode::FMV_X_W + : InstructionOpcode::FMV_X_D)); + } + if (DestinationType->isFloat() && SourceType->isInteger()) { + return Instruction.setOpcode(targetOpcode(DestinationType->scalarBitWidth() == 32 + ? InstructionOpcode::FMV_W_X + : InstructionOpcode::FMV_D_X)); + } + return makeUnexpected(ErrorCode::Unsupported, "RISC-V bitcast pair is unsupported"); + } + + [[nodiscard]] static Expected selectIntegerBinary(MachineInstr &Instruction, + std::optional Type, + InstructionOpcode DoubleWord, + InstructionOpcode Word) { + if (!Type || !isSupportedIntegerType(*Type)) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V integer instruction requires a supported scalar integer"); + const bool IsWord = Type->isInteger() && Type->scalarBitWidth() <= 32; + return Instruction.setOpcode(targetOpcode(IsWord ? Word : DoubleWord)); + } + + [[nodiscard]] static Expected + selectNormalizedIntegerBinary(MachineInstr &Instruction, MachineFunction &Function, + const TargetDescription &Target, std::optional Type, + InstructionOpcode DoubleWord, InstructionOpcode Word, bool Signed, + bool NormalizeRight) { + const MachineOperand *LeftOperand = Instruction.operandAt(1); + const MachineOperand *RightOperand = Instruction.operandAt(2); + const Register *Left = LeftOperand == nullptr ? nullptr : LeftOperand->getIf(); + const Register *Right = RightOperand == nullptr ? nullptr : RightOperand->getIf(); + if (!Type || !isSupportedIntegerScalar(*Type) || Left == nullptr || Right == nullptr) { + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V normalized integer operation has invalid operands"); + } + auto NormalizedLeft = + normalizeIntegerBefore(Instruction, Function, Target, *Left, *Type, Signed); + if (!NormalizedLeft) + return std::unexpected(NormalizedLeft.error()); + Register SelectedRight = *Right; + if (NormalizeRight) { + auto NormalizedRight = + normalizeIntegerBefore(Instruction, Function, Target, *Right, *Type, Signed); + if (!NormalizedRight) + return std::unexpected(NormalizedRight.error()); + SelectedRight = *NormalizedRight; + } + auto SetLeft = Instruction.operandAt(1)->setRegister(*NormalizedLeft); + if (!SetLeft) + return std::unexpected(SetLeft.error()); + auto SetRight = Instruction.operandAt(2)->setRegister(SelectedRight); + if (!SetRight) + return std::unexpected(SetRight.error()); + const bool IsWord = Type->scalarBitWidth() <= 32; + return Instruction.setOpcode(targetOpcode(IsWord ? Word : DoubleWord)); + } + + [[nodiscard]] static Expected selectLoad(MachineInstr &Instruction, + std::optional Type) { + if (!Type) + return makeUnexpected(ErrorCode::VerificationError, + "load result has no virtual-register type"); + const auto Bytes = memoryAccessBytes(Instruction); + if (!Bytes) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V load size does not match its result type"); + if (Instruction.memOperands().front()->isAtomic()) { + if (!isSupportedIntegerType(*Type) || (*Bytes != 4 && *Bytes != 8)) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V atomic load requires a 32-bit or 64-bit GPR value"); + return Instruction.setOpcode( + targetOpcode(*Bytes == 4 ? InstructionOpcode::LR_W : InstructionOpcode::LR_D)); + } + auto AppendOffset = appendImmediate(Instruction, 0); + if (!AppendOffset) + return std::unexpected(AppendOffset.error()); + if (isSupportedIntegerType(*Type)) { + const InstructionOpcode Selected = *Bytes == 1 ? InstructionOpcode::LB + : *Bytes == 2 ? InstructionOpcode::LH + : *Bytes == 4 ? InstructionOpcode::LW + : *Bytes == 8 ? InstructionOpcode::LD + : InstructionOpcode::Invalid; + if (Selected == InstructionOpcode::Invalid) + return makeUnexpected(ErrorCode::Unsupported, "RISC-V integer load width is unsupported"); + return Instruction.setOpcode(targetOpcode(Selected)); + } + if (Type->isFloat() && Type->scalarBitWidth() == 32) + return Instruction.setOpcode(targetOpcode(InstructionOpcode::FLW)); + if (Type->isFloat() && Type->scalarBitWidth() == 64) + return Instruction.setOpcode(targetOpcode(InstructionOpcode::FLD)); + return makeUnexpected(ErrorCode::Unsupported, "RISC-V cannot select this generic load type"); + } + + [[nodiscard]] static Expected selectExtLoad(MachineInstr &Instruction, + std::optional Type, bool Signed) { + if (!Type || !isSupportedIntegerType(*Type)) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V extension load requires an integer result"); + const auto Bytes = memoryAccessBytes(Instruction); + if (!Bytes) + return makeUnexpected(ErrorCode::Unsupported, "RISC-V extension load has an invalid width"); + const InstructionOpcode Selected = + *Bytes == 1 ? (Signed ? InstructionOpcode::LB : InstructionOpcode::LBU) + : *Bytes == 2 ? (Signed ? InstructionOpcode::LH : InstructionOpcode::LHU) + : *Bytes == 4 ? (Signed ? InstructionOpcode::LW : InstructionOpcode::LWU) + : InstructionOpcode::Invalid; + if (Selected == InstructionOpcode::Invalid) + return makeUnexpected(ErrorCode::Unsupported, "RISC-V extension load width is unsupported"); + auto Immediate = appendImmediate(Instruction, 0); + if (!Immediate) + return std::unexpected(Immediate.error()); + return Instruction.setOpcode(targetOpcode(Selected)); + } + + [[nodiscard]] static Expected selectFloatExtLoad(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target) { + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); + if (Destination == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V floating extension load has no destination register"); + const Register DestinationRegister = *Destination; + auto F32 = LLT::makeFloat(FloatSemantics::IEEESingle); + if (!F32) + return std::unexpected(F32.error()); + auto Temporary = createTemporary(Function, *F32, Target); + if (!Temporary) + return std::unexpected(Temporary.error()); + auto SetDestination = Instruction.operandAt(0)->setRegister(*Temporary); + if (!SetDestination) + return std::unexpected(SetDestination.error()); + auto Immediate = appendImmediate(Instruction, 0); + if (!Immediate) + return std::unexpected(Immediate.error()); + auto SetOpcode = Instruction.setOpcode(targetOpcode(InstructionOpcode::FLW)); + if (!SetOpcode) + return std::unexpected(SetOpcode.error()); + MachineIRBuilder Builder(Function, Target); + auto Conversion = buildImmediateAfter(Builder, Instruction, InstructionOpcode::FCVT_D_S, + DestinationRegister, *Temporary, 0); + if (!Conversion) + return std::unexpected(Conversion.error()); + return {}; + } + + [[nodiscard]] static Expected selectFloatTruncStore(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target) { + const MachineOperand *ValueOperand = Instruction.operandAt(0); + const Register *Value = ValueOperand == nullptr ? nullptr : ValueOperand->getIf(); + if (Value == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V floating truncation store has no value register"); + const Register ValueRegister = *Value; + auto F32 = LLT::makeFloat(FloatSemantics::IEEESingle); + if (!F32) + return std::unexpected(F32.error()); + auto Temporary = createTemporary(Function, *F32, Target); + if (!Temporary) + return std::unexpected(Temporary.error()); + MachineIRBuilder Builder(Function, Target); + auto Point = Builder.setInsertPointBefore(Instruction); + if (!Point) + return std::unexpected(Point.error()); + auto Conversion = Builder.buildInstr(targetOpcode(InstructionOpcode::FCVT_S_D)); + if (!Conversion) + return std::unexpected(Conversion.error()); + auto Def = Conversion->addDef(*Temporary); + if (!Def) + return std::unexpected(Def.error()); + auto Use = Conversion->addUse(ValueRegister); + if (!Use) + return std::unexpected(Use.error()); + auto Rounding = Conversion->addImmediate(0); + if (!Rounding) + return std::unexpected(Rounding.error()); + auto Built = Conversion->build(); + if (!Built) + return std::unexpected(Built.error()); + auto SetValue = Instruction.operandAt(0)->setRegister(*Temporary); + if (!SetValue) + return std::unexpected(SetValue.error()); + auto Immediate = appendImmediate(Instruction, 0); + if (!Immediate) + return std::unexpected(Immediate.error()); + return Instruction.setOpcode(targetOpcode(InstructionOpcode::FSW)); + } + + [[nodiscard]] static Expected selectStore(MachineInstr &Instruction, + MachineFunction &Function) { + const MachineOperand *Value = Instruction.operandAt(0); + const std::optional Type = + Value == nullptr ? std::nullopt : registerType(Function, *Value); + if (!Type) + return makeUnexpected(ErrorCode::VerificationError, + "store value has no virtual-register type"); + const auto Bytes = memoryAccessBytes(Instruction); + if (!Bytes) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V store size does not match its value type"); + if (Instruction.memOperands().front()->isAtomic()) { + if (!isSupportedIntegerType(*Type) || (*Bytes != 4 && *Bytes != 8)) + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V atomic store requires a 32-bit or 64-bit GPR value"); + auto Zero = physicalRegister(PhysicalRegisterID::X0); + if (!Zero) + return std::unexpected(Zero.error()); + auto Destination = insertPhysicalRegister(Instruction, 0, *Zero, RegisterOperandFlag::Define); + if (!Destination) + return std::unexpected(Destination.error()); + return Instruction.setOpcode( + targetOpcode(*Bytes == 4 ? InstructionOpcode::AMOSWAP_W : InstructionOpcode::AMOSWAP_D)); + } + auto AppendOffset = appendImmediate(Instruction, 0); + if (!AppendOffset) + return std::unexpected(AppendOffset.error()); + if (isSupportedIntegerType(*Type)) { + const InstructionOpcode Selected = *Bytes == 1 ? InstructionOpcode::SB + : *Bytes == 2 ? InstructionOpcode::SH + : *Bytes == 4 ? InstructionOpcode::SW + : *Bytes == 8 ? InstructionOpcode::SD + : InstructionOpcode::Invalid; + if (Selected == InstructionOpcode::Invalid) + return makeUnexpected(ErrorCode::Unsupported, "RISC-V integer store width is unsupported"); + return Instruction.setOpcode(targetOpcode(Selected)); + } + if (Type->isFloat() && Type->scalarBitWidth() == 32) + return Instruction.setOpcode(targetOpcode(InstructionOpcode::FSW)); + if (Type->isFloat() && Type->scalarBitWidth() == 64) + return Instruction.setOpcode(targetOpcode(InstructionOpcode::FSD)); + return makeUnexpected(ErrorCode::Unsupported, "RISC-V cannot select this generic store type"); + } + + [[nodiscard]] static Expected selectBranch(MachineInstr &Instruction) { + auto Zero = physicalRegister(PhysicalRegisterID::X0); + if (!Zero) + return std::unexpected(Zero.error()); + auto Inserted = insertPhysicalRegister(Instruction, 0, *Zero, RegisterOperandFlag::Define); + if (!Inserted) + return std::unexpected(Inserted.error()); + return Instruction.setOpcode(targetOpcode(InstructionOpcode::JAL)); + } + + [[nodiscard]] static Expected selectConditionalBranch(MachineInstr &Instruction) { + auto Zero = physicalRegister(PhysicalRegisterID::X0); + if (!Zero) + return std::unexpected(Zero.error()); + auto Inserted = insertPhysicalRegister(Instruction, 1, *Zero, RegisterOperandFlag::Use); + if (!Inserted) + return std::unexpected(Inserted.error()); + return Instruction.setOpcode(targetOpcode(InstructionOpcode::BNE)); + } + + [[nodiscard]] static Expected selectIntegerConstant(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target, + std::optional Type) { + if (!Type || !isSupportedIntegerType(*Type)) { + return makeUnexpected(ErrorCode::Unsupported, + "RISC-V constant selection requires an i1-i64 or pointer result"); + } + const MachineOperand *ImmediateOperand = Instruction.operandAt(1); + if (ImmediateOperand == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V generic constant has no immediate operand"); + auto ImmediateValue = signedImmediate(*ImmediateOperand); + if (!ImmediateValue) + return std::unexpected(ImmediateValue.error()); + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const Register *DestinationValue = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); + if (DestinationValue == nullptr || !DestinationValue->isVirtual()) { + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V generic constant must define a virtual register"); + } + const Register Destination = *DestinationValue; + const std::int64_t Value = *ImmediateValue; + const bool IsWord = Type->isInteger() && Type->scalarBitWidth() <= 32; + const InstructionOpcode AddImmediate = + IsWord ? InstructionOpcode::ADDIW : InstructionOpcode::ADDI; + const InstructionOpcode ShiftLeftImmediate = + IsWord ? InstructionOpcode::SLLIW : InstructionOpcode::SLLI; + if (fitsSigned(Value, 12)) { + auto Zero = physicalRegister(PhysicalRegisterID::X0); + if (!Zero) + return std::unexpected(Zero.error()); + auto Source = insertPhysicalRegister(Instruction, 1, *Zero, RegisterOperandFlag::Use); + if (!Source) + return std::unexpected(Source.error()); + return Instruction.setOpcode(targetOpcode(AddImmediate)); + } + + // Recursively decompose the value into signed 12-bit chunks. The + // quotient is computed from an unsigned bit pattern with explicit sign + // extension so both INT64_MIN and INT64_MAX remain well-defined. + std::uint64_t Remaining = static_cast(Value); + std::vector Digits; + while (!fitsSigned(std::bit_cast(Remaining), 12)) { + const std::uint64_t LowBits = Remaining & 0xFFFU; + std::int64_t Digit = static_cast(LowBits); + if (Digit >= 0x800) + Digit -= 0x1000; + Digits.push_back(Digit); + + const std::uint64_t Adjusted = Remaining - static_cast(Digit); + std::uint64_t Quotient = Adjusted >> 12U; + if ((Remaining & (std::uint64_t{1} << 63U)) != 0) + Quotient |= ~std::uint64_t{0} << 52U; + Remaining = Quotient; + } + const std::int64_t Base = std::bit_cast(Remaining); + + auto Temporary = createTemporary(Function, *Type, Target); + if (!Temporary) + return std::unexpected(Temporary.error()); + auto *Parent = Instruction.parent(); + if (Parent == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V constant selection received a detached instruction"); + auto SetOpcode = Instruction.setOpcode(targetOpcode(AddImmediate)); + if (!SetOpcode) + return std::unexpected(SetOpcode.error()); + auto SetDestination = Instruction.operandAt(0)->setRegister(*Temporary); + if (!SetDestination) + return std::unexpected(SetDestination.error()); + auto EraseImmediate = Instruction.eraseOperand(1); + if (!EraseImmediate) + return std::unexpected(EraseImmediate.error()); + auto Zero = physicalRegister(PhysicalRegisterID::X0); + if (!Zero) + return std::unexpected(Zero.error()); + auto AddSource = insertPhysicalRegister(Instruction, 1, *Zero, RegisterOperandFlag::Use); + if (!AddSource) + return std::unexpected(AddSource.error()); + auto AddBase = appendImmediate(Instruction, Base); + if (!AddBase) + return std::unexpected(AddBase.error()); + + MachineIRBuilder Builder(Function, Target); + auto InsertPoint = Builder.setInsertPointAfter(Instruction); + if (!InsertPoint) + return std::unexpected(InsertPoint.error()); + + Register Current = *Temporary; + for (std::size_t Reverse = Digits.size(); Reverse != 0; --Reverse) { + const std::size_t Index = Reverse - 1U; + auto ShiftResult = createTemporary(Function, *Type, Target); + if (!ShiftResult) + return std::unexpected(ShiftResult.error()); + auto Shift = Builder.buildInstr(targetOpcode(ShiftLeftImmediate)); + if (!Shift) + return std::unexpected(Shift.error()); + auto ShiftDef = Shift->addDef(*ShiftResult); + if (!ShiftDef) + return std::unexpected(ShiftDef.error()); + auto ShiftUse = Shift->addUse(Current); + if (!ShiftUse) + return std::unexpected(ShiftUse.error()); + auto ShiftAmount = Shift->addImmediate(12); + if (!ShiftAmount) + return std::unexpected(ShiftAmount.error()); + auto BuiltShift = Shift->build(); + if (!BuiltShift) + return std::unexpected(BuiltShift.error()); + + const bool IsFinal = Index == 0; + Register AddDestination; + if (IsFinal) { + AddDestination = Destination; + } else { + auto AddTemporary = createTemporary(Function, *Type, Target); + if (!AddTemporary) + return std::unexpected(AddTemporary.error()); + AddDestination = *AddTemporary; + } + auto Add = Builder.setInsertPointAfter(**BuiltShift); + if (!Add) + return std::unexpected(Add.error()); + auto AddInstruction = Builder.buildInstr(targetOpcode(AddImmediate)); + if (!AddInstruction) + return std::unexpected(AddInstruction.error()); + auto AddDef = AddInstruction->addDef(AddDestination); + if (!AddDef) + return std::unexpected(AddDef.error()); + auto AddUse = AddInstruction->addUse(*ShiftResult); + if (!AddUse) + return std::unexpected(AddUse.error()); + auto AddImmediateResult = AddInstruction->addImmediate(Digits[Index]); + if (!AddImmediateResult) + return std::unexpected(AddImmediateResult.error()); + auto BuiltAdd = AddInstruction->build(); + if (!BuiltAdd) + return std::unexpected(BuiltAdd.error()); + Current = AddDestination; + } + return {}; + } + + [[nodiscard]] static Expected selectFloatingConstant(MachineInstr &Instruction, + MachineFunction &Function, + const TargetDescription &Target, + std::optional Type) { + if (!Type || !isRVFloatType(*Type)) + return makeUnexpected(ErrorCode::Unsupported, "RISC-V floating constant requires f32 or f64"); + const MachineOperand *Immediate = Instruction.operandAt(1); + const auto *Floating = + Immediate == nullptr ? nullptr : Immediate->getIf(); + const std::optional Semantics = Type->floatSemantics(); + if (Floating == nullptr || !Semantics || Floating->Bits.semantics() != *Semantics) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V floating constant has mismatched bit semantics"); + const MachineOperand *DestinationOperand = Instruction.operandAt(0); + const Register *Destination = + DestinationOperand == nullptr ? nullptr : DestinationOperand->getIf(); + if (Destination == nullptr || !Destination->isVirtual()) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V floating constant must define a virtual register"); + const Register DestinationRegister = *Destination; + + const std::uint32_t BitWidth = Type->scalarBitWidth(); + const std::uint64_t Bits = Floating->Bits.words().empty() ? 0 : Floating->Bits.words().front(); + auto Integer = IntegerImmediate::fromUnsigned(64, Bits); + if (!Integer) + return std::unexpected(Integer.error()); + auto IntegerType = LLT::makeInteger(64); + if (!IntegerType) + return std::unexpected(IntegerType.error()); + auto Temporary = createTemporary(Function, *IntegerType, Target); + if (!Temporary) + return std::unexpected(Temporary.error()); + + auto SetOpcode = Instruction.setOpcode(opcode(GenericOpcode::G_CONSTANT)); + if (!SetOpcode) + return std::unexpected(SetOpcode.error()); + auto SetDestination = Instruction.operandAt(0)->setRegister(*Temporary); + if (!SetDestination) + return std::unexpected(SetDestination.error()); + auto EraseImmediate = Instruction.eraseOperand(1); + if (!EraseImmediate) + return std::unexpected(EraseImmediate.error()); + auto AddInteger = Instruction.addOperand(MachineOperand::integerImmediate(*Integer)); + if (!AddInteger) + return std::unexpected(AddInteger.error()); + auto IntegerSelection = selectIntegerConstant(Instruction, Function, Target, *IntegerType); + if (!IntegerSelection) + return std::unexpected(IntegerSelection.error()); + + MachineInstr *IntegerDefinition = nullptr; + for (MachineOperand &DefOperand : Function.registerInfo().defOperands(*Temporary)) { + IntegerDefinition = DefOperand.parent(); + break; + } + if (IntegerDefinition == nullptr) + return makeUnexpected(ErrorCode::InternalError, + "RISC-V floating constant lost its integer materialization"); + MachineIRBuilder Builder(Function, Target); + auto InsertPoint = Builder.setInsertPointAfter(*IntegerDefinition); + if (!InsertPoint) + return std::unexpected(InsertPoint.error()); + const InstructionOpcode Move = + BitWidth == 32 ? InstructionOpcode::FMV_W_X : InstructionOpcode::FMV_D_X; + auto MoveInstruction = Builder.buildInstr(targetOpcode(Move)); + if (!MoveInstruction) + return std::unexpected(MoveInstruction.error()); + auto MoveDef = MoveInstruction->addDef(DestinationRegister); + if (!MoveDef) + return std::unexpected(MoveDef.error()); + auto MoveUse = MoveInstruction->addUse(*Temporary); + if (!MoveUse) + return std::unexpected(MoveUse.error()); + auto BuiltMove = MoveInstruction->build(); + if (!BuiltMove) + return std::unexpected(BuiltMove.error()); + return {}; + } + + [[nodiscard]] static Expected selectRoundingBinary(MachineInstr &Instruction, + std::optional Type, + InstructionOpcode Single, + InstructionOpcode Double) { + auto RoundingMode = appendImmediate(Instruction, 0); + if (!RoundingMode) + return std::unexpected(RoundingMode.error()); + if (Type && Type->isFloat() && Type->scalarBitWidth() == 32) + return Instruction.setOpcode(targetOpcode(Single)); + if (Type && Type->isFloat() && Type->scalarBitWidth() == 64) + return Instruction.setOpcode(targetOpcode(Double)); + return makeUnexpected(ErrorCode::Unsupported, "RISC-V cannot select this floating binary type"); + } + + [[nodiscard]] static Expected selectRoundingTernary(MachineInstr &Instruction, + std::optional Type, + InstructionOpcode Single, + InstructionOpcode Double) { + return selectRoundingBinary(Instruction, Type, Single, Double); + } + + [[nodiscard]] static Expected selectRoundingUnary(MachineInstr &Instruction, + std::optional Type, + InstructionOpcode Single, + InstructionOpcode Double) { + return selectRoundingBinary(Instruction, Type, Single, Double); + } + + [[nodiscard]] static Expected selectSignInjection(MachineInstr &Instruction, + std::optional Type, + InstructionOpcode Single, + InstructionOpcode Double) { + const MachineOperand *Source = Instruction.operandAt(1); + const Register *SourceReg = Source == nullptr ? nullptr : Source->getIf(); + if (SourceReg == nullptr) + return makeUnexpected(ErrorCode::VerificationError, + "RISC-V floating sign operation requires a register source"); + auto Duplicate = insertVirtualUse(Instruction, 2, *SourceReg); + if (!Duplicate) + return std::unexpected(Duplicate.error()); + if (Type && Type->isFloat() && Type->scalarBitWidth() == 32) + return Instruction.setOpcode(targetOpcode(Single)); + if (Type && Type->isFloat() && Type->scalarBitWidth() == 64) + return Instruction.setOpcode(targetOpcode(Double)); + return makeUnexpected(ErrorCode::Unsupported, "RISC-V cannot select this floating unary type"); + } +}; + +} // namespace + +const TargetLegalizerInfo &legalizerInfo() noexcept { + static const RISCVLegalizerInfo Value; + return Value; +} + +const TargetRegisterBankInfo ®isterBankInfo() noexcept { + static const RISCVRegisterBankInfo Value; + return Value; +} + +const TargetInstructionSelector &instructionSelector() noexcept { + static const RISCVInstructionSelector Value; + return Value; +} + +} // namespace rcc::riscv diff --git a/backend/unittests/CodeGen/CMakeLists.txt b/backend/unittests/CodeGen/CMakeLists.txt index deb7a75..19beb2f 100644 --- a/backend/unittests/CodeGen/CMakeLists.txt +++ b/backend/unittests/CodeGen/CMakeLists.txt @@ -20,6 +20,7 @@ set( MachineMetadataTest.cpp MachineOperandTest.cpp MachinePassTest.cpp + MachinePipelinePassesTest.cpp MachineVerifierTest.cpp MachineVerifierTargetTest.cpp MachineValueTypesTest.cpp @@ -252,6 +253,17 @@ target_link_libraries( add_test(NAME RCCBackend.CodeGen.MachinePass COMMAND RCCBackendCodeGenMachinePassTests) +add_executable(RCCBackendCodeGenMachinePipelinePassesTests MachinePipelinePassesTest.cpp) +rcc_backend_configure_target(RCCBackendCodeGenMachinePipelinePassesTests) +target_link_libraries( + RCCBackendCodeGenMachinePipelinePassesTests + PRIVATE RCCBackend::CodeGen RCCBackend::RISCV RCCBackend::TestTarget +) +add_test( + NAME RCCBackend.CodeGen.MachinePipelinePasses + COMMAND RCCBackendCodeGenMachinePipelinePassesTests +) + add_executable(RCCBackendCodeGenMachineAnalysesTests MachineAnalysesTest.cpp) rcc_backend_configure_target(RCCBackendCodeGenMachineAnalysesTests) target_link_libraries( @@ -337,6 +349,7 @@ rcc_backend_register_format_files( "${CMAKE_CURRENT_SOURCE_DIR}/MachineMetadataTest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineOperandTest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachinePassTest.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachinePipelinePassesTest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineVerifierTest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineVerifierTargetTest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineValueTypesTest.cpp" @@ -365,6 +378,7 @@ rcc_backend_register_tidy_files( "${CMAKE_CURRENT_SOURCE_DIR}/MachineMetadataTest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineOperandTest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachinePassTest.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/MachinePipelinePassesTest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineVerifierTest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineVerifierTargetTest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/MachineValueTypesTest.cpp" diff --git a/backend/unittests/CodeGen/MachinePipelinePassesTest.cpp b/backend/unittests/CodeGen/MachinePipelinePassesTest.cpp new file mode 100644 index 0000000..0fe2e80 --- /dev/null +++ b/backend/unittests/CodeGen/MachinePipelinePassesTest.cpp @@ -0,0 +1,629 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/CodeGen/MachinePipelinePasses.h" + +#include "rcc/CodeGen/GenericOpcodes.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineIRBuilder.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineMemOperand.h" +#include "rcc/CodeGen/MachineRegisterInfo.h" +#include "rcc/Target/RISCV/RISCVCodeGen.h" +#include "rcc/Target/RISCV/RISCVTarget.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +class TestContext final { +public: + void expect(bool Condition, std::string_view Message) { + if (Condition) + return; + std::cerr << Message << '\n'; + ++Failures; + } + + [[nodiscard]] int result() const noexcept { + return Failures == 0 ? 0 : 1; + } + +private: + int Failures = 0; +}; + +struct FunctionFixture final { + std::unique_ptr Function; + rcc::MachineBasicBlock *Entry = nullptr; +}; + +class TogglingLegalizer final : public rcc::TargetLegalizerInfo { +public: + [[nodiscard]] rcc::Expected + classify(const rcc::MachineInstr &, const rcc::MachineFunction &, + const rcc::TargetDescription &) const override { + return rcc::LegalizationAction::Lower; + } + + [[nodiscard]] rcc::Expected lower(rcc::MachineInstr &Instruction, rcc::MachineFunction &, + const rcc::TargetDescription &, + rcc::LegalizationAction) const override { + const auto Generic = static_cast(Instruction.opcode().value()); + return Instruction.setOpcode(rcc::opcode(Generic == rcc::GenericOpcode::G_ADD + ? rcc::GenericOpcode::G_SUB + : rcc::GenericOpcode::G_ADD)); + } +}; + +class MutatingFailingSelector final : public rcc::TargetInstructionSelector { +public: + [[nodiscard]] rcc::Expected canSelect(const rcc::MachineInstr &, + const rcc::MachineFunction &, + const rcc::TargetDescription &) const override { + return {}; + } + + [[nodiscard]] rcc::Expected select(rcc::MachineInstr &Instruction, rcc::MachineFunction &, + const rcc::TargetDescription &) const override { + auto Mutation = Instruction.setOpcode(rcc::opcode(rcc::GenericOpcode::G_SUB)); + if (!Mutation) + return std::unexpected(Mutation.error()); + return rcc::makeUnexpected(rcc::ErrorCode::Unsupported, "test selector rejected its renderer"); + } +}; + +class RecordingSelector final : public rcc::TargetInstructionSelector { +public: + explicit RecordingSelector(const rcc::TargetInstructionSelector &Delegate) + : Delegate(&Delegate) {} + + [[nodiscard]] rcc::Expected canSelect(const rcc::MachineInstr &Instruction, + const rcc::MachineFunction &Function, + const rcc::TargetDescription &Target) const override { + return Delegate->canSelect(Instruction, Function, Target); + } + + [[nodiscard]] rcc::Expected select(rcc::MachineInstr &Instruction, + rcc::MachineFunction &Function, + const rcc::TargetDescription &Target) const override { + SelectedIDs.push_back(Instruction.stableID().value()); + return Delegate->select(Instruction, Function, Target); + } + + [[nodiscard]] const std::vector &selectedIDs() const noexcept { + return SelectedIDs; + } + +private: + const rcc::TargetInstructionSelector *Delegate; + mutable std::vector SelectedIDs; +}; + +class TwoStepSelector final : public rcc::TargetInstructionSelector { +public: + [[nodiscard]] rcc::Expected canSelect(const rcc::MachineInstr &, + const rcc::MachineFunction &, + const rcc::TargetDescription &) const override { + return {}; + } + + [[nodiscard]] rcc::Expected select(rcc::MachineInstr &Instruction, rcc::MachineFunction &, + const rcc::TargetDescription &) const override { + ++Selections; + if (Instruction.opcode() == rcc::opcode(rcc::GenericOpcode::G_ADD)) + return Instruction.setOpcode(rcc::opcode(rcc::GenericOpcode::G_SUB)); + if (Instruction.opcode() == rcc::opcode(rcc::GenericOpcode::G_SUB)) { + return Instruction.setOpcode( + rcc::Opcode::fromValue(static_cast(rcc::riscv::InstructionOpcode::ADD))); + } + return rcc::makeUnexpected(rcc::ErrorCode::Unsupported, + "two-step selector received an unexpected opcode"); + } + + [[nodiscard]] std::size_t selections() const noexcept { + return Selections; + } + +private: + mutable std::size_t Selections = 0; +}; + +[[nodiscard]] FunctionFixture createFixture(TestContext &Context, std::string_view Name, + std::uint64_t ID) { + auto Function = + rcc::MachineFunction::create(std::string(Name), rcc::MachineFunctionStableID::fromValue(ID)); + Context.expect(Function.has_value(), "machine pipeline function creation must succeed"); + if (!Function) + return {}; + auto Entry = (*Function)->createBasicBlock(); + Context.expect(Entry.has_value(), "machine pipeline block creation must succeed"); + if (!Entry || !(*Function)->appendBlock(**Entry)) + return {}; + return {std::move(*Function), *Entry}; +} + +[[nodiscard]] rcc::Register createRegister(TestContext &Context, rcc::MachineFunction &Function, + std::uint32_t Bits) { + auto Type = rcc::LLT::makeInteger(Bits); + Context.expect(Type.has_value(), "integer LLT construction must succeed"); + if (!Type) + return {}; + auto Reg = Function.registerInfo().createVirtualRegister(*Type); + Context.expect(Reg.has_value(), "virtual register construction must succeed"); + return Reg ? *Reg : rcc::Register{}; +} + +[[nodiscard]] rcc::Register createFloatRegister(TestContext &Context, + rcc::MachineFunction &Function) { + auto Type = rcc::LLT::makeFloat(rcc::FloatSemantics::IEEEDouble); + Context.expect(Type.has_value(), "floating LLT construction must succeed"); + if (!Type) + return {}; + auto Reg = Function.registerInfo().createVirtualRegister(*Type); + Context.expect(Reg.has_value(), "floating virtual register construction must succeed"); + return Reg ? *Reg : rcc::Register{}; +} + +[[nodiscard]] bool assignGPR(rcc::MachineFunction &Function, rcc::Register Reg) { + const auto Bank = + rcc::RegisterBankID::fromValue(static_cast(rcc::riscv::RegisterBank::GPR)); + const auto RegClass = + rcc::RegisterClassID::fromValue(static_cast(rcc::riscv::RegisterClass::GPR)); + return Function.registerInfo().setRegisterBank(Reg, Bank).has_value() && + Function.registerInfo().setRegisterClass(Reg, RegClass).has_value(); +} + +[[nodiscard]] bool buildConstant(TestContext &Context, rcc::MachineIRBuilder &Builder, + rcc::Register Destination, std::int64_t Value) { + auto Pending = Builder.buildInstr(rcc::opcode(rcc::GenericOpcode::G_CONSTANT)); + if (!Pending || !Pending->addDef(Destination) || !Pending->addImmediate(Value) || + !Pending->build()) { + Context.expect(false, "generic constant construction must succeed"); + return false; + } + return true; +} + +[[nodiscard]] bool buildAdd(TestContext &Context, rcc::MachineIRBuilder &Builder, + rcc::Register Destination, rcc::Register Left, rcc::Register Right) { + auto Pending = Builder.buildInstr(rcc::opcode(rcc::GenericOpcode::G_ADD)); + if (!Pending || !Pending->addDef(Destination) || !Pending->addUse(Left) || + !Pending->addUse(Right) || !Pending->build()) { + Context.expect(false, "generic add construction must succeed"); + return false; + } + return true; +} + +void testNullAndMissingContracts(TestContext &Context) { + auto Target = rcc::riscv::createTargetDescription(); + auto Fixture = createFixture(Context, "pipeline_contracts", 1); + Context.expect(Target.has_value() && Fixture.Function != nullptr && Fixture.Entry != nullptr, + "pipeline contract fixture must be complete"); + if (!Target || !Fixture.Function) + return; + + rcc::LegalizerPass NullPass(nullptr); + rcc::MachineAnalysisManager Analyses; + auto NullResult = NullPass.run(*Fixture.Function, *Target, Analyses); + Context.expect(!NullResult && NullResult.error().code() == rcc::ErrorCode::InvalidArgument, + "a null legalizer interface must be rejected"); + Context.expect(Fixture.Function->properties().empty(), + "a rejected pass must not publish function properties"); + + rcc::LegalizerPass ValidPass(rcc::riscv::legalizerInfo()); + auto MissingResult = ValidPass.run(*Fixture.Function, *Target, Analyses); + Context.expect(!MissingResult && + MissingResult.error().code() == rcc::ErrorCode::VerificationError, + "a pass with missing prerequisites must be rejected"); +} + +void testPipelineAndUnsupportedRollback(TestContext &Context) { + auto Target = rcc::riscv::createTargetDescription(); + auto Fixture = createFixture(Context, "pipeline_success", 2); + Context.expect(Target.has_value() && Fixture.Function != nullptr && Fixture.Entry != nullptr, + "pipeline success fixture must be complete"); + if (!Target || !Fixture.Function || Fixture.Entry == nullptr) + return; + + const rcc::Register Left = createRegister(Context, *Fixture.Function, 64); + const rcc::Register Right = createRegister(Context, *Fixture.Function, 64); + const rcc::Register Result = createRegister(Context, *Fixture.Function, 64); + rcc::MachineIRBuilder Builder(*Fixture.Function, *Target); + Context.expect(Builder.setInsertPoint(*Fixture.Entry).has_value(), + "pipeline insertion point must be valid"); + if (Left.isNoRegister() || Right.isNoRegister() || Result.isNoRegister() || + !buildConstant(Context, Builder, Left, 42) || !buildConstant(Context, Builder, Right, 1) || + !buildAdd(Context, Builder, Result, Left, Right)) + return; + + Fixture.Function->properties() = {rcc::MachineFunctionProperty::IsSSA, + rcc::MachineFunctionProperty::TracksLiveness}; + rcc::MachinePassManager Pipeline; + Context.expect(Pipeline.addPass(std::make_unique(rcc::riscv::legalizerInfo())) + .has_value(), + "legalizer must be accepted by the pass manager"); + Context.expect( + Pipeline + .addPass(std::make_unique(rcc::riscv::registerBankInfo())) + .has_value(), + "register-bank selector must be accepted by the pass manager"); + Context.expect( + Pipeline + .addPass(std::make_unique(rcc::riscv::instructionSelector())) + .has_value(), + "instruction selector must be accepted by the pass manager"); + rcc::MachineAnalysisManager Analyses; + auto Run = Pipeline.run(*Fixture.Function, *Target, Analyses); + Context.expect(Run.has_value(), "RV64GC generic pipeline must complete"); + Context.expect(Fixture.Function->properties().has(rcc::MachineFunctionProperty::Selected), + "successful instruction selection must publish Selected"); + Context.expect(Fixture.Function->firstBlock()->firstInstruction() != nullptr, + "selected instructions must remain attached to their block"); + for (const rcc::MachineInstr *Instruction : Fixture.Function->allInstructions()) + Context.expect(!rcc::isGenericOpcode(Instruction->opcode()), + "successful instruction selection must remove generic opcodes"); + + auto Unsupported = createFixture(Context, "pipeline_unsupported", 3); + Context.expect(Unsupported.Function != nullptr && Unsupported.Entry != nullptr, + "unsupported pipeline fixture must be complete"); + if (!Unsupported.Function || Unsupported.Entry == nullptr) + return; + const rcc::Register UnsupportedReg = createFloatRegister(Context, *Unsupported.Function); + const rcc::Register UnsupportedSource = createFloatRegister(Context, *Unsupported.Function); + rcc::MachineIRBuilder UnsupportedBuilder(*Unsupported.Function, *Target); + Context.expect(UnsupportedBuilder.setInsertPoint(*Unsupported.Entry).has_value(), + "unsupported pipeline insertion point must be valid"); + if (UnsupportedReg.isNoRegister() || UnsupportedSource.isNoRegister()) + return; + auto Pending = UnsupportedBuilder.buildInstr(rcc::opcode(rcc::GenericOpcode::G_FEXP)); + Context.expect(Pending.has_value(), "unsupported generic instruction must be constructible"); + if (!Pending) + return; + auto Built = Pending->addDef(UnsupportedReg); + Context.expect(Built.has_value(), "unsupported generic definition must be accepted"); + if (!Built) + return; + Context.expect(Pending->addUse(UnsupportedSource).has_value() && Pending->build().has_value(), + "unsupported floating operation must be constructible"); + Unsupported.Function->properties() = {rcc::MachineFunctionProperty::IsSSA, + rcc::MachineFunctionProperty::TracksLiveness}; + rcc::MachinePassManager OnlyLegalizer; + Context.expect( + OnlyLegalizer.addPass(std::make_unique(rcc::riscv::legalizerInfo())) + .has_value(), + "standalone legalizer must be accepted"); + rcc::MachineAnalysisManager UnsupportedAnalyses; + auto Failure = OnlyLegalizer.run(*Unsupported.Function, *Target, UnsupportedAnalyses); + Context.expect(!Failure && Failure.error().code() == rcc::ErrorCode::Unsupported, + "unsupported generic instructions must produce a target diagnostic"); + Context.expect(!Failure && Failure.error().message().find("G_FEXP") != std::string::npos, + "unsupported legalization diagnostics must name the rejected opcode"); + Context.expect( + Unsupported.Function->properties().has(rcc::MachineFunctionProperty::IsSSA) && + !Unsupported.Function->properties().has(rcc::MachineFunctionProperty::Legalized), + "failed legalizer execution must not publish Legalized"); +} + +void testBankConflict(TestContext &Context) { + auto Target = rcc::riscv::createTargetDescription(); + auto Fixture = createFixture(Context, "pipeline_bank_conflict", 4); + Context.expect(Target.has_value() && Fixture.Function != nullptr && Fixture.Entry != nullptr, + "bank conflict fixture must be complete"); + if (!Target || !Fixture.Function || Fixture.Entry == nullptr) + return; + const rcc::Register Value = createRegister(Context, *Fixture.Function, 64); + Context.expect( + Fixture.Function->registerInfo() + .setRegisterBank(Value, rcc::RegisterBankID::fromValue( + static_cast(rcc::riscv::RegisterBank::FPR))) + .has_value(), + "conflicting bank assignment must be constructible"); + Fixture.Function->properties() = {rcc::MachineFunctionProperty::Legalized}; + rcc::RegisterBankSelectPass Pass(rcc::riscv::registerBankInfo()); + rcc::MachineAnalysisManager Analyses; + auto Result = Pass.run(*Fixture.Function, *Target, Analyses); + Context.expect(!Result && Result.error().code() == rcc::ErrorCode::VerificationError, + "bank selection must reject a conflicting pre-existing bank"); + auto Info = Fixture.Function->registerInfo().virtualRegisterInfo(Value); + Context.expect(Info && Info->get().Bank, "failed bank selection must preserve a bank assignment"); + if (!Info || !Info->get().Bank) + return; + const rcc::RegisterBankID Bank = Info->get().Bank.value_or(rcc::RegisterBankID{}); + Context.expect(Bank == rcc::RegisterBankID::fromValue( + static_cast(rcc::riscv::RegisterBank::FPR)), + "failed bank selection must preserve the original bank assignment"); +} + +void testBankClassOnlyChange(TestContext &Context) { + auto Target = rcc::riscv::createTargetDescription(); + auto Fixture = createFixture(Context, "pipeline_bank_class", 5); + Context.expect(Target.has_value() && Fixture.Function != nullptr && Fixture.Entry != nullptr, + "bank class fixture must be complete"); + if (!Target || !Fixture.Function || Fixture.Entry == nullptr) + return; + const rcc::Register Value = createRegister(Context, *Fixture.Function, 64); + rcc::MachineIRBuilder Builder(*Fixture.Function, *Target); + if (Value.isNoRegister() || !Builder.setInsertPoint(*Fixture.Entry) || + !buildConstant(Context, Builder, Value, 7)) + return; + const auto GPR = + rcc::RegisterBankID::fromValue(static_cast(rcc::riscv::RegisterBank::GPR)); + Context.expect(Fixture.Function->registerInfo().setRegisterBank(Value, GPR).has_value(), + "pre-existing GPR bank must be assignable"); + Fixture.Function->properties() = {rcc::MachineFunctionProperty::Legalized}; + rcc::RegisterBankSelectPass Pass(rcc::riscv::registerBankInfo()); + rcc::MachineAnalysisManager Analyses; + auto Result = Pass.run(*Fixture.Function, *Target, Analyses); + Context.expect(Result.has_value() && Result->Changed && + Result->Changes.has(rcc::MachinePassChange::Registers), + "bank selection must report a class-only register change"); + auto Info = Fixture.Function->registerInfo().virtualRegisterInfo(Value); + Context.expect(Info && Info->get().Bank && Info->get().RegClass, + "bank selection must publish both bank and class"); + if (!Info || !Info->get().RegClass) + return; + Context.expect(Info->get().RegClass.value_or(rcc::RegisterClassID{}) == + rcc::RegisterClassID::fromValue( + static_cast(rcc::riscv::RegisterClass::GPR)), + "i64 GPR bank must select the GPR class"); +} + +void testLegalizerCycleDiagnostic(TestContext &Context) { + auto Target = rcc::riscv::createTargetDescription(); + auto Fixture = createFixture(Context, "pipeline_legalizer_cycle", 6); + Context.expect(Target.has_value() && Fixture.Function != nullptr && Fixture.Entry != nullptr, + "legalizer cycle fixture must be complete"); + if (!Target || !Fixture.Function || Fixture.Entry == nullptr) + return; + const rcc::Register Left = createRegister(Context, *Fixture.Function, 64); + const rcc::Register Right = createRegister(Context, *Fixture.Function, 64); + const rcc::Register Result = createRegister(Context, *Fixture.Function, 64); + rcc::MachineIRBuilder Builder(*Fixture.Function, *Target); + if (Left.isNoRegister() || Right.isNoRegister() || Result.isNoRegister() || + !Builder.setInsertPoint(*Fixture.Entry) || !buildAdd(Context, Builder, Result, Left, Right)) + return; + Fixture.Function->properties() = {rcc::MachineFunctionProperty::IsSSA, + rcc::MachineFunctionProperty::TracksLiveness}; + TogglingLegalizer Legalizer; + rcc::LegalizerPass StablePass(Legalizer); + rcc::MachineAnalysisManager Analyses; + auto ResultRun = StablePass.run(*Fixture.Function, *Target, Analyses); + Context.expect(!ResultRun && ResultRun.error().code() == rcc::ErrorCode::InternalError, + "repeated legalization states must report InternalError"); + Context.expect(Fixture.Entry->firstInstruction() != nullptr && + Fixture.Entry->firstInstruction()->opcode() == + rcc::opcode(rcc::GenericOpcode::G_ADD), + "failed legalization preflight must preserve the original instruction"); + Context.expect(!Fixture.Function->properties().has(rcc::MachineFunctionProperty::Legalized), + "a failed legalization cycle must not publish Legalized"); +} + +void testInstructionSelectionTransaction(TestContext &Context) { + auto Target = rcc::riscv::createTargetDescription(); + auto Fixture = createFixture(Context, "pipeline_selection_transaction", 8); + if (!Target || !Fixture.Function || Fixture.Entry == nullptr) + return; + const rcc::Register Left = createRegister(Context, *Fixture.Function, 64); + const rcc::Register Right = createRegister(Context, *Fixture.Function, 64); + const rcc::Register Result = createRegister(Context, *Fixture.Function, 64); + rcc::MachineIRBuilder Builder(*Fixture.Function, *Target); + if (Left.isNoRegister() || Right.isNoRegister() || Result.isNoRegister() || + !Builder.setInsertPoint(*Fixture.Entry) || !buildAdd(Context, Builder, Result, Left, Right) || + !assignGPR(*Fixture.Function, Left) || !assignGPR(*Fixture.Function, Right) || + !assignGPR(*Fixture.Function, Result)) { + return; + } + rcc::MachineInstr *Original = Fixture.Entry->firstInstruction(); + if (Original == nullptr) + return; + const rcc::Opcode OriginalOpcode = Original->opcode(); + const std::size_t OriginalOperandCount = Original->operandCount(); + Fixture.Function->properties() = {rcc::MachineFunctionProperty::Legalized, + rcc::MachineFunctionProperty::RegBankSelected}; + MutatingFailingSelector Selector; + rcc::InstructionSelectPass Pass(Selector); + rcc::MachineAnalysisManager Analyses; + auto Run = Pass.run(*Fixture.Function, *Target, Analyses); + Context.expect(!Run && Run.error().code() == rcc::ErrorCode::Unsupported, + "a renderer failure must be reported before publication"); + Context.expect(Original->opcode() == OriginalOpcode && + Original->operandCount() == OriginalOperandCount && + Fixture.Entry->instructionCount() == 1, + "a renderer failure must leave the original MachineIR unchanged"); +} + +void testInstructionSelectionOrder(TestContext &Context) { + auto Target = rcc::riscv::createTargetDescription(); + auto Fixture = createFixture(Context, "pipeline_selection_order", 9); + if (!Target || !Fixture.Function || Fixture.Entry == nullptr) + return; + const rcc::Register Left = createRegister(Context, *Fixture.Function, 64); + const rcc::Register Right = createRegister(Context, *Fixture.Function, 64); + const rcc::Register Result = createRegister(Context, *Fixture.Function, 64); + rcc::MachineIRBuilder Builder(*Fixture.Function, *Target); + if (Left.isNoRegister() || Right.isNoRegister() || Result.isNoRegister() || + !Builder.setInsertPoint(*Fixture.Entry) || !buildConstant(Context, Builder, Left, 6) || + !buildConstant(Context, Builder, Right, 7) || + !buildAdd(Context, Builder, Result, Left, Right) || !assignGPR(*Fixture.Function, Left) || + !assignGPR(*Fixture.Function, Right) || !assignGPR(*Fixture.Function, Result)) { + return; + } + std::vector LayoutIDs; + for (const rcc::MachineInstr *Instruction = Fixture.Entry->firstInstruction(); + Instruction != nullptr; Instruction = Instruction->next()) { + LayoutIDs.push_back(Instruction->stableID().value()); + } + Fixture.Function->properties() = {rcc::MachineFunctionProperty::Legalized, + rcc::MachineFunctionProperty::RegBankSelected}; + RecordingSelector Selector(rcc::riscv::instructionSelector()); + rcc::InstructionSelectPass Pass(Selector); + rcc::MachineAnalysisManager Analyses; + auto Run = Pass.run(*Fixture.Function, *Target, Analyses); + Context.expect(Run.has_value(), "recorded instruction selection must complete"); + std::ranges::reverse(LayoutIDs); + std::vector Expected = LayoutIDs; + Expected.insert(Expected.end(), LayoutIDs.begin(), LayoutIDs.end()); + Context.expect( + Selector.selectedIDs() == Expected, + "instruction selection must use stable block-postorder and reverse instruction order"); +} + +void testInstructionSelectionWorklist(TestContext &Context) { + auto Target = rcc::riscv::createTargetDescription(); + auto Fixture = createFixture(Context, "pipeline_selection_worklist", 10); + if (!Target || !Fixture.Function || Fixture.Entry == nullptr) + return; + const rcc::Register Left = createRegister(Context, *Fixture.Function, 64); + const rcc::Register Right = createRegister(Context, *Fixture.Function, 64); + const rcc::Register Result = createRegister(Context, *Fixture.Function, 64); + rcc::MachineIRBuilder Builder(*Fixture.Function, *Target); + if (Left.isNoRegister() || Right.isNoRegister() || Result.isNoRegister() || + !Builder.setInsertPoint(*Fixture.Entry) || !buildAdd(Context, Builder, Result, Left, Right) || + !assignGPR(*Fixture.Function, Left) || !assignGPR(*Fixture.Function, Right) || + !assignGPR(*Fixture.Function, Result)) { + return; + } + Fixture.Function->properties() = {rcc::MachineFunctionProperty::Legalized, + rcc::MachineFunctionProperty::RegBankSelected}; + TwoStepSelector Selector; + rcc::InstructionSelectPass Pass(Selector); + rcc::MachineAnalysisManager Analyses; + auto Run = Pass.run(*Fixture.Function, *Target, Analyses); + Context.expect(Run.has_value(), "a progressing multi-step selection must complete"); + Context.expect(Selector.selections() == 4, + "transaction preflight and publication must each select both generic states"); + Context.expect(Fixture.Entry->firstInstruction() != nullptr && + Fixture.Entry->firstInstruction()->opcode() == + rcc::Opcode::fromValue( + static_cast(rcc::riscv::InstructionOpcode::ADD)), + "the selection worklist must revisit a progressing generic root"); +} + +void testExtendedRV64GCPipeline(TestContext &Context) { + auto Target = rcc::riscv::createTargetDescription(); + auto Fixture = createFixture(Context, "pipeline_extended_rv64gc", 7); + Context.expect(Target.has_value() && Fixture.Function != nullptr && Fixture.Entry != nullptr, + "extended pipeline fixture must be complete"); + if (!Target || !Fixture.Function || Fixture.Entry == nullptr) + return; + auto I1 = rcc::LLT::makeInteger(1); + auto I64 = rcc::LLT::makeInteger(64); + auto Pointer = rcc::LLT::makePointer(0, 64); + if (!I1 || !I64 || !Pointer) + return; + const rcc::Register Left = createRegister(Context, *Fixture.Function, 64); + const rcc::Register Right = createRegister(Context, *Fixture.Function, 64); + const rcc::Register Condition = + Fixture.Function->registerInfo().createVirtualRegister(*I1).value_or(rcc::Register{}); + const rcc::Register Selected = createRegister(Context, *Fixture.Function, 64); + const rcc::Register Sum = createRegister(Context, *Fixture.Function, 64); + const rcc::Register Overflow = + Fixture.Function->registerInfo().createVirtualRegister(*I1).value_or(rcc::Register{}); + const rcc::Register Quotient = createRegister(Context, *Fixture.Function, 64); + const rcc::Register Remainder = createRegister(Context, *Fixture.Function, 64); + const rcc::Register Address = + Fixture.Function->registerInfo().createVirtualRegister(*Pointer).value_or(rcc::Register{}); + const rcc::Register AtomicResult = createRegister(Context, *Fixture.Function, 64); + const std::array Registers{Left, Right, Condition, Selected, Sum, + Overflow, Quotient, Remainder, Address, AtomicResult}; + if (std::ranges::any_of(Registers, [](rcc::Register Reg) { return Reg.isNoRegister(); })) + return; + rcc::MachineIRBuilder Builder(*Fixture.Function, *Target); + if (!Builder.setInsertPoint(*Fixture.Entry) || !buildConstant(Context, Builder, Left, 41) || + !buildConstant(Context, Builder, Right, 7)) { + return; + } + auto Compare = Builder.buildInstr(rcc::opcode(rcc::GenericOpcode::G_ICMP)); + if (!Compare || !Compare->addDef(Condition) || + !Compare->addIntegerPredicate(rcc::IntegerPredicate::UnsignedGreaterThan) || + !Compare->addUse(Left) || !Compare->addUse(Right) || !Compare->build()) { + return; + } + auto Select = Builder.buildInstr(rcc::opcode(rcc::GenericOpcode::G_SELECT)); + if (!Select || !Select->addDef(Selected) || !Select->addUse(Condition) || !Select->addUse(Left) || + !Select->addUse(Right) || !Select->build()) { + return; + } + auto AddOverflow = Builder.buildInstr(rcc::opcode(rcc::GenericOpcode::G_UADDO)); + if (!AddOverflow || !AddOverflow->addDef(Sum) || !AddOverflow->addDef(Overflow) || + !AddOverflow->addUse(Left) || !AddOverflow->addUse(Right) || !AddOverflow->build()) { + return; + } + auto DivRem = Builder.buildInstr(rcc::opcode(rcc::GenericOpcode::G_SDIVREM)); + if (!DivRem || !DivRem->addDef(Quotient) || !DivRem->addDef(Remainder) || !DivRem->addUse(Left) || + !DivRem->addUse(Right) || !DivRem->build()) { + return; + } + auto AddressDefinition = Builder.buildInstr(rcc::opcode(rcc::GenericOpcode::G_IMPLICIT_DEF)); + if (!AddressDefinition || !AddressDefinition->addDef(Address) || !AddressDefinition->build()) + return; + auto Size = rcc::MemorySize::fixed(8); + auto Align = rcc::Alignment::create(8); + if (!Size || !Align) + return; + rcc::MachineMemOperandDesc Description; + Description.Accesses = {rcc::MemoryAccessFlag::Load, rcc::MemoryAccessFlag::Store}; + Description.AtomicKind = rcc::AtomicOperation::ReadModifyWrite; + Description.SuccessOrdering = rcc::AtomicOrdering::SequentiallyConsistent; + Description.Size = *Size; + Description.Align = *Align; + Description.Base = rcc::MemoryBase::stack(); + auto Memory = Fixture.Function->createMemOperand(std::move(Description)); + if (!Memory) + return; + auto Atomic = Builder.buildInstr(rcc::opcode(rcc::GenericOpcode::G_ATOMICRMW_ADD)); + if (!Atomic || !Atomic->addDef(AtomicResult) || !Atomic->addUse(Address) || + !Atomic->addUse(Selected) || !Atomic->addMemOperand(**Memory) || !Atomic->build()) { + return; + } + + Fixture.Function->properties() = {rcc::MachineFunctionProperty::IsSSA, + rcc::MachineFunctionProperty::TracksLiveness}; + rcc::MachinePassManager Pipeline; + if (!Pipeline.addPass(std::make_unique(rcc::riscv::legalizerInfo())) || + !Pipeline.addPass( + std::make_unique(rcc::riscv::registerBankInfo())) || + !Pipeline.addPass( + std::make_unique(rcc::riscv::instructionSelector()))) { + Context.expect(false, "extended RV64GC pipeline construction must succeed"); + return; + } + rcc::MachineAnalysisManager Analyses; + auto Run = Pipeline.run(*Fixture.Function, *Target, Analyses); + Context.expect(Run.has_value(), "extended RV64GC pipeline must complete"); + Context.expect(Run && Fixture.Function->properties().has(rcc::MachineFunctionProperty::Selected), + "extended pipeline must publish Selected"); + for (const rcc::MachineInstr *Instruction : Fixture.Function->allInstructions()) { + Context.expect(!rcc::isGenericOpcode(Instruction->opcode()), + "extended pipeline must eliminate every generic opcode"); + Context.expect( + Target->validateInstruction(*Instruction, Fixture.Function->registerInfo()).has_value(), + "extended pipeline output must satisfy target descriptors"); + } +} + +} // namespace + +int main() { + TestContext Context; + testNullAndMissingContracts(Context); + testPipelineAndUnsupportedRollback(Context); + testBankConflict(Context); + testBankClassOnlyChange(Context); + testLegalizerCycleDiagnostic(Context); + testInstructionSelectionTransaction(Context); + testInstructionSelectionOrder(Context); + testInstructionSelectionWorklist(Context); + testExtendedRV64GCPipeline(Context); + return Context.result(); +} diff --git a/backend/unittests/Target/CMakeLists.txt b/backend/unittests/Target/CMakeLists.txt index aa45b90..98c4426 100644 --- a/backend/unittests/Target/CMakeLists.txt +++ b/backend/unittests/Target/CMakeLists.txt @@ -28,10 +28,16 @@ rcc_backend_configure_target(RCCBackendRISCVCallLoweringTests) target_link_libraries(RCCBackendRISCVCallLoweringTests PRIVATE RCCBackend::RISCV) add_test(NAME RCCBackendRISCVCallLoweringTests COMMAND RCCBackendRISCVCallLoweringTests) +add_executable(RCCBackendRISCVCodeGenTests RISCVCodeGenTest.cpp) +rcc_backend_configure_target(RCCBackendRISCVCodeGenTests) +target_link_libraries(RCCBackendRISCVCodeGenTests PRIVATE RCCBackend::RISCV RCCBackend::CodeGen) +add_test(NAME RCCBackendRISCVCodeGenTests COMMAND RCCBackendRISCVCodeGenTests) + rcc_backend_register_format_files( "${CMAKE_CURRENT_SOURCE_DIR}/DescriptionGeneratorTest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVCallLoweringTest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVTargetTest.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RISCVCodeGenTest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/TestTarget.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/TestTarget.h" ) @@ -39,5 +45,6 @@ rcc_backend_register_tidy_files( "${CMAKE_CURRENT_SOURCE_DIR}/DescriptionGeneratorTest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVCallLoweringTest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/RISCVTargetTest.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/RISCVCodeGenTest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/TestTarget.cpp" ) diff --git a/backend/unittests/Target/RISCVCodeGenTest.cpp b/backend/unittests/Target/RISCVCodeGenTest.cpp new file mode 100644 index 0000000..efb8eda --- /dev/null +++ b/backend/unittests/Target/RISCVCodeGenTest.cpp @@ -0,0 +1,1108 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rcc/Target/RISCV/RISCVCodeGen.h" + +#include "rcc/CodeGen/GenericOpcodes.h" +#include "rcc/CodeGen/MachineBasicBlock.h" +#include "rcc/CodeGen/MachineFunction.h" +#include "rcc/CodeGen/MachineIRBuilder.h" +#include "rcc/CodeGen/MachineInstr.h" +#include "rcc/CodeGen/MachineMemOperand.h" +#include "rcc/Target/RISCV/RISCVTarget.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace { + +class TestContext final { +public: + void expect(bool Condition, std::string_view Message) { + if (Condition) + return; + std::cerr << Message << '\n'; + ++Failures; + } + + [[nodiscard]] int result() const noexcept { + return Failures == 0 ? 0 : 1; + } + +private: + int Failures = 0; +}; + +struct InstructionFixture final { + std::unique_ptr Function; + rcc::MachineInstr *Instruction = nullptr; + rcc::Register Result; +}; + +[[nodiscard]] InstructionFixture createBinary(TestContext &Context, + const rcc::TargetDescription &Target, + rcc::GenericOpcode Generic, std::uint32_t Bits) { + auto Function = + rcc::MachineFunction::create("riscv_codegen", rcc::MachineFunctionStableID::fromValue(Bits)); + Context.expect(Function.has_value(), "RISC-V codegen function creation must succeed"); + if (!Function) + return {}; + auto Block = (*Function)->createBasicBlock(); + if (!Block || !(*Function)->appendBlock(**Block)) + return {}; + auto Type = rcc::LLT::makeInteger(Bits); + if (!Type) + return {}; + auto Left = (*Function)->registerInfo().createVirtualRegister(*Type); + auto Right = (*Function)->registerInfo().createVirtualRegister(*Type); + auto Result = (*Function)->registerInfo().createVirtualRegister(*Type); + if (!Left || !Right || !Result) + return {}; + rcc::MachineIRBuilder Builder(**Function, Target); + if (!Builder.setInsertPoint(**Block)) + return {}; + auto Pending = Builder.buildInstr(rcc::opcode(Generic)); + if (!Pending || !Pending->addDef(*Result) || !Pending->addUse(*Left) || !Pending->addUse(*Right)) + return {}; + auto Built = Pending->build(); + if (!Built) + return {}; + return {std::move(*Function), *Built, *Result}; +} + +[[nodiscard]] bool constrainRegister(rcc::MachineFunction &Function, rcc::Register Reg, + rcc::LLT Type) { + const auto Bank = Type.isFloat() ? rcc::RegisterBankID::fromValue( + static_cast(rcc::riscv::RegisterBank::FPR)) + : rcc::RegisterBankID::fromValue( + static_cast(rcc::riscv::RegisterBank::GPR)); + const auto Class = Type.isFloat() + ? rcc::RegisterClassID::fromValue(static_cast( + Type.scalarBitWidth() == 32 ? rcc::riscv::RegisterClass::FPR32 + : rcc::riscv::RegisterClass::FPR64)) + : rcc::RegisterClassID::fromValue( + static_cast(rcc::riscv::RegisterClass::GPR)); + return Function.registerInfo().setRegisterBank(Reg, Bank).has_value() && + Function.registerInfo().setRegisterClass(Reg, Class).has_value(); +} + +[[nodiscard]] bool constrainVirtualRegisters(rcc::MachineFunction &Function) { + const auto Registers = Function.registerInfo().virtualRegisters(); + for (std::size_t Index = 0; Index < Registers.size(); ++Index) { + const rcc::VirtualRegisterInfo &Info = Registers[Index]; + if (Info.IsTombstone) + continue; + auto Reg = rcc::Register::makeVirtual(Index); + if (!Reg || !Info.Type || !constrainRegister(Function, *Reg, *Info.Type)) + return false; + } + return true; +} + +[[nodiscard]] bool validatesSelectedFunction(const rcc::TargetDescription &Target, + const rcc::MachineFunction &Function) { + for (const rcc::MachineInstr *Instruction : Function.allInstructions()) { + if (rcc::isGenericOpcode(Instruction->opcode()) || + !Target.validateInstruction(*Instruction, Function.registerInfo())) { + return false; + } + } + return Target.validateRegisterInfo(Function.registerInfo()).has_value(); +} + +[[nodiscard]] std::unique_ptr +createUnaryFunction(TestContext &Context, const rcc::TargetDescription &Target, + rcc::GenericOpcode Generic, rcc::LLT DestinationType, rcc::LLT SourceType, + rcc::Register *DestinationOut = nullptr, rcc::Register *SourceOut = nullptr) { + auto Function = rcc::MachineFunction::create( + "riscv_unary", + rcc::MachineFunctionStableID::fromValue(200 + static_cast(Generic))); + Context.expect(Function.has_value(), "unary test function creation must succeed"); + if (!Function) + return nullptr; + auto Block = (*Function)->createBasicBlock(); + if (!Block || !(*Function)->appendBlock(**Block)) + return nullptr; + auto Destination = (*Function)->registerInfo().createVirtualRegister(DestinationType); + auto Source = (*Function)->registerInfo().createVirtualRegister(SourceType); + if (!Destination || !Source) + return nullptr; + if (!constrainRegister(**Function, *Destination, DestinationType) || + !constrainRegister(**Function, *Source, SourceType)) + return nullptr; + rcc::MachineIRBuilder Builder(**Function, Target); + if (!Builder.setInsertPoint(**Block)) + return nullptr; + auto Pending = Builder.buildInstr(rcc::opcode(Generic)); + if (!Pending || !Pending->addDef(*Destination) || !Pending->addUse(*Source) || !Pending->build()) + return nullptr; + if (DestinationOut != nullptr) + *DestinationOut = *Destination; + if (SourceOut != nullptr) + *SourceOut = *Source; + return std::move(*Function); +} + +[[nodiscard]] std::unique_ptr +createCompareFunction(TestContext &Context, const rcc::TargetDescription &Target, + rcc::GenericOpcode Generic, rcc::LLT ValueType, bool Floating, + rcc::IntegerPredicate IntegerCondition, + rcc::FloatingPredicate FloatingCondition) { + auto Function = rcc::MachineFunction::create( + "riscv_compare", + rcc::MachineFunctionStableID::fromValue(500 + static_cast(Generic))); + Context.expect(Function.has_value(), "comparison test function creation must succeed"); + if (!Function) + return nullptr; + auto Block = (*Function)->createBasicBlock(); + if (!Block || !(*Function)->appendBlock(**Block)) + return nullptr; + auto ResultType = rcc::LLT::makeInteger(1); + if (!ResultType) + return nullptr; + auto Result = (*Function)->registerInfo().createVirtualRegister(*ResultType); + auto Left = (*Function)->registerInfo().createVirtualRegister(ValueType); + auto Right = (*Function)->registerInfo().createVirtualRegister(ValueType); + if (!Result || !Left || !Right || !constrainRegister(**Function, *Result, *ResultType) || + !constrainRegister(**Function, *Left, ValueType) || + !constrainRegister(**Function, *Right, ValueType)) + return nullptr; + rcc::MachineIRBuilder Builder(**Function, Target); + if (!Builder.setInsertPoint(**Block)) + return nullptr; + auto Pending = Builder.buildInstr(rcc::opcode(Generic)); + if (!Pending || !Pending->addDef(*Result)) + return nullptr; + if (Floating) { + if (!Pending->addFloatingPredicate(FloatingCondition)) + return nullptr; + } else if (!Pending->addIntegerPredicate(IntegerCondition)) { + return nullptr; + } + if (!Pending->addUse(*Left) || !Pending->addUse(*Right) || !Pending->build()) + return nullptr; + return std::move(*Function); +} + +[[nodiscard]] std::unique_ptr +createFloatBinaryFunction(TestContext &Context, const rcc::TargetDescription &Target, + rcc::GenericOpcode Generic, rcc::LLT Type) { + auto Function = rcc::MachineFunction::create( + "riscv_float_binary", + rcc::MachineFunctionStableID::fromValue(700 + static_cast(Generic))); + Context.expect(Function.has_value(), "floating binary test function creation must succeed"); + if (!Function) + return nullptr; + auto Block = (*Function)->createBasicBlock(); + if (!Block || !(*Function)->appendBlock(**Block)) + return nullptr; + auto Left = (*Function)->registerInfo().createVirtualRegister(Type); + auto Right = (*Function)->registerInfo().createVirtualRegister(Type); + auto Result = (*Function)->registerInfo().createVirtualRegister(Type); + if (!Left || !Right || !Result || !constrainRegister(**Function, *Left, Type) || + !constrainRegister(**Function, *Right, Type) || !constrainRegister(**Function, *Result, Type)) + return nullptr; + rcc::MachineIRBuilder Builder(**Function, Target); + if (!Builder.setInsertPoint(**Block)) + return nullptr; + auto Pending = Builder.buildInstr(rcc::opcode(Generic)); + if (!Pending || !Pending->addDef(*Result) || !Pending->addUse(*Left) || + !Pending->addUse(*Right) || !Pending->build()) + return nullptr; + return std::move(*Function); +} + +[[nodiscard]] std::unique_ptr createRegisterOnlyFunction( + TestContext &Context, const rcc::TargetDescription &Target, rcc::GenericOpcode Generic, + std::initializer_list DefinitionTypes, std::initializer_list UseTypes) { + auto Function = rcc::MachineFunction::create( + "riscv_register_only", + rcc::MachineFunctionStableID::fromValue(1000 + static_cast(Generic))); + Context.expect(Function.has_value(), "register-only test function creation must succeed"); + if (!Function) + return nullptr; + auto Block = (*Function)->createBasicBlock(); + if (!Block || !(*Function)->appendBlock(**Block)) + return nullptr; + rcc::MachineIRBuilder Builder(**Function, Target); + if (!Builder.setInsertPoint(**Block)) + return nullptr; + auto Pending = Builder.buildInstr(rcc::opcode(Generic)); + if (!Pending) + return nullptr; + for (rcc::LLT Type : DefinitionTypes) { + auto Reg = (*Function)->registerInfo().createVirtualRegister(Type); + if (!Reg || !Pending->addDef(*Reg)) + return nullptr; + } + for (rcc::LLT Type : UseTypes) { + auto Reg = (*Function)->registerInfo().createVirtualRegister(Type); + if (!Reg || !Pending->addUse(*Reg)) + return nullptr; + } + if (!Pending->build() || !constrainVirtualRegisters(**Function)) + return nullptr; + return std::move(*Function); +} + +void testClassification(TestContext &Context) { + auto Target = rcc::riscv::createTargetDescription(); + Context.expect(Target.has_value(), "RV64GC target must be available for codegen tests"); + if (!Target) + return; + auto Add32 = createBinary(Context, *Target, rcc::GenericOpcode::G_ADD, 32); + auto Add64 = createBinary(Context, *Target, rcc::GenericOpcode::G_ADD, 64); + Context.expect(Add32.Instruction != nullptr && Add64.Instruction != nullptr, + "RISC-V integer test instructions must be constructible"); + if (Add32.Instruction == nullptr || Add64.Instruction == nullptr) + return; + auto Add32Action = + rcc::riscv::legalizerInfo().classify(*Add32.Instruction, *Add32.Function, *Target); + auto Add64Action = + rcc::riscv::legalizerInfo().classify(*Add64.Instruction, *Add64.Function, *Target); + Context.expect(Add32Action && *Add32Action == rcc::LegalizationAction::Legal, + "RV64GC legalizer must accept i32 arithmetic"); + Context.expect(Add64Action && *Add64Action == rcc::LegalizationAction::Legal, + "RV64GC legalizer must accept i64 arithmetic"); + Context.expect(rcc::riscv::instructionSelector() + .canSelect(*Add32.Instruction, *Add32.Function, *Target) + .has_value(), + "RV64GC selector must pre-match i32 arithmetic"); + Context.expect(rcc::riscv::instructionSelector() + .canSelect(*Add64.Instruction, *Add64.Function, *Target) + .has_value(), + "RV64GC selector must pre-match i64 arithmetic"); + Context.expect(rcc::riscv::instructionSelector() + .select(*Add32.Instruction, *Add32.Function, *Target) + .has_value(), + "RV64GC selector must lower i32 arithmetic"); + Context.expect(Add32.Instruction->opcode() == rcc::Opcode::fromValue(static_cast( + rcc::riscv::InstructionOpcode::ADDW)), + "i32 G_ADD must select ADDW"); + Context.expect(rcc::riscv::instructionSelector() + .select(*Add64.Instruction, *Add64.Function, *Target) + .has_value(), + "RV64GC selector must lower i64 arithmetic"); + Context.expect(Add64.Instruction->opcode() == rcc::Opcode::fromValue(static_cast( + rcc::riscv::InstructionOpcode::ADD)), + "i64 G_ADD must select ADD"); + + auto FloatFunction = rcc::MachineFunction::create("riscv_float_constants", + rcc::MachineFunctionStableID::fromValue(3)); + Context.expect(FloatFunction.has_value(), "floating constant function creation must succeed"); + if (!FloatFunction) + return; + auto FloatBlock = (*FloatFunction)->createBasicBlock(); + Context.expect(FloatBlock.has_value(), "floating constant block creation must succeed"); + if (!FloatBlock || !(*FloatFunction)->appendBlock(**FloatBlock)) + return; + auto F32 = rcc::LLT::makeFloat(rcc::FloatSemantics::IEEESingle); + auto F64 = rcc::LLT::makeFloat(rcc::FloatSemantics::IEEEDouble); + Context.expect(F32.has_value() && F64.has_value(), + "RV64GC floating constant types must be constructible"); + if (!F32 || !F64) + return; + auto F32Result = (*FloatFunction)->registerInfo().createVirtualRegister(*F32); + auto F64Result = (*FloatFunction)->registerInfo().createVirtualRegister(*F64); + Context.expect(F32Result.has_value() && F64Result.has_value(), + "RV64GC floating constant registers must be constructible"); + if (!F32Result || !F64Result) + return; + Context.expect((*FloatFunction) + ->registerInfo() + .setRegisterBank(*F32Result, + rcc::RegisterBankID::fromValue(static_cast( + rcc::riscv::RegisterBank::FPR))) + .has_value() && + (*FloatFunction) + ->registerInfo() + .setRegisterClass( + *F32Result, rcc::RegisterClassID::fromValue(static_cast( + rcc::riscv::RegisterClass::FPR32))) + .has_value(), + "f32 destination must have an FPR32 bank and class"); + Context.expect((*FloatFunction) + ->registerInfo() + .setRegisterBank(*F64Result, + rcc::RegisterBankID::fromValue(static_cast( + rcc::riscv::RegisterBank::FPR))) + .has_value() && + (*FloatFunction) + ->registerInfo() + .setRegisterClass( + *F64Result, rcc::RegisterClassID::fromValue(static_cast( + rcc::riscv::RegisterClass::FPR64))) + .has_value(), + "f64 destination must have an FPR64 bank and class"); + rcc::MachineIRBuilder FloatBuilder(**FloatFunction, *Target); + if (!FloatBuilder.setInsertPoint(**FloatBlock)) + return; + const std::array SingleWords{0x3f800000U}; + const std::array DoubleWords{0x3ff0000000000000ULL}; + auto SingleBits = rcc::FloatingPointBits::create(rcc::FloatSemantics::IEEESingle, SingleWords); + auto DoubleBits = rcc::FloatingPointBits::create(rcc::FloatSemantics::IEEEDouble, DoubleWords); + Context.expect(SingleBits.has_value() && DoubleBits.has_value(), + "floating constant payloads must be constructible"); + if (!SingleBits || !DoubleBits) + return; + auto SinglePending = FloatBuilder.buildInstr(rcc::opcode(rcc::GenericOpcode::G_FCONSTANT)); + auto DoublePending = FloatBuilder.buildInstr(rcc::opcode(rcc::GenericOpcode::G_FCONSTANT)); + Context.expect(SinglePending.has_value() && DoublePending.has_value(), + "floating constant instructions must be constructible"); + if (!SinglePending || !DoublePending) + return; + Context.expect(SinglePending->addDef(*F32Result).has_value() && + SinglePending->addFloatingImmediate(*SingleBits).has_value() && + SinglePending->build().has_value(), + "f32 floating constant must be buildable"); + Context.expect(DoublePending->addDef(*F64Result).has_value() && + DoublePending->addFloatingImmediate(*DoubleBits).has_value() && + DoublePending->build().has_value(), + "f64 floating constant must be buildable"); + if (FloatFunction->get()->firstBlock()->firstInstruction() == nullptr) + return; + auto Instructions = FloatFunction->get()->allInstructions(); + auto It = Instructions.begin(); + rcc::MachineInstr *F32Instruction = It == Instructions.end() ? nullptr : *It; + if (It != Instructions.end()) + ++It; + rcc::MachineInstr *F64Instruction = It == Instructions.end() ? nullptr : *It; + Context.expect(F32Instruction != nullptr && F64Instruction != nullptr, + "floating constant instructions must remain in layout order"); + if (F32Instruction == nullptr || F64Instruction == nullptr) + return; + auto F32Action = rcc::riscv::legalizerInfo().classify(*F32Instruction, **FloatFunction, *Target); + auto F64Action = rcc::riscv::legalizerInfo().classify(*F64Instruction, **FloatFunction, *Target); + Context.expect(F32Action && *F32Action == rcc::LegalizationAction::Legal, + "RV64GC legalizer must accept f32 constants"); + Context.expect(F64Action && *F64Action == rcc::LegalizationAction::Legal, + "RV64GC legalizer must accept f64 constants"); + auto F32Selection = + rcc::riscv::instructionSelector().select(*F32Instruction, **FloatFunction, *Target); + auto F64Selection = + rcc::riscv::instructionSelector().select(*F64Instruction, **FloatFunction, *Target); + Context.expect(F32Selection.has_value(), "RV64GC selector must lower f32 constants"); + Context.expect(F64Selection.has_value(), "RV64GC selector must lower f64 constants"); + bool HasFMVWX = false; + bool HasFMVDX = false; + for (const rcc::MachineInstr *Instruction : FloatFunction->get()->allInstructions()) { + HasFMVWX |= + Instruction->opcode() == + rcc::Opcode::fromValue(static_cast(rcc::riscv::InstructionOpcode::FMV_W_X)); + HasFMVDX |= + Instruction->opcode() == + rcc::Opcode::fromValue(static_cast(rcc::riscv::InstructionOpcode::FMV_D_X)); + } + Context.expect(HasFMVWX, "f32 constants must use FMV.W.X materialization"); + Context.expect(HasFMVDX, "f64 constants must use FMV.D.X materialization"); +} + +void testNarrowIntegerSelection(TestContext &Context) { + auto Target = rcc::riscv::createTargetDescription(); + if (!Target) + return; + struct BinaryCase final { + rcc::GenericOpcode Generic; + std::uint32_t Bits; + rcc::riscv::InstructionOpcode RequiredOpcode; + }; + const std::array Cases{ + BinaryCase{rcc::GenericOpcode::G_ADD, 1, rcc::riscv::InstructionOpcode::ADDW}, + BinaryCase{rcc::GenericOpcode::G_UDIV, 8, rcc::riscv::InstructionOpcode::DIVUW}, + BinaryCase{rcc::GenericOpcode::G_ASHR, 16, rcc::riscv::InstructionOpcode::SRAW}, + BinaryCase{rcc::GenericOpcode::G_SMIN, 24, rcc::riscv::InstructionOpcode::SLT}, + }; + for (const BinaryCase &TestCase : Cases) { + auto Fixture = createBinary(Context, *Target, TestCase.Generic, TestCase.Bits); + if (!Fixture.Function || Fixture.Instruction == nullptr || + !constrainVirtualRegisters(*Fixture.Function)) { + Context.expect(false, "narrow integer fixture must be constructible"); + continue; + } + auto Action = + rcc::riscv::legalizerInfo().classify(*Fixture.Instruction, *Fixture.Function, *Target); + Context.expect(Action && *Action == rcc::LegalizationAction::Legal, + "i1-i24 scalar operations must be legal on RV64GC GPR containers"); + auto Selection = + rcc::riscv::instructionSelector().select(*Fixture.Instruction, *Fixture.Function, *Target); + Context.expect(Selection.has_value(), "narrow integer operation must select"); + bool FoundRequiredOpcode = false; + for (const rcc::MachineInstr *Instruction : Fixture.Function->allInstructions()) { + FoundRequiredOpcode |= + Instruction->opcode() == + rcc::Opcode::fromValue(static_cast(TestCase.RequiredOpcode)); + } + Context.expect(FoundRequiredOpcode, + "narrow integer selection must use the required RV64 instruction"); + Context.expect(Selection && validatesSelectedFunction(*Target, *Fixture.Function), + "narrow integer selection must satisfy target descriptors"); + } + + auto I7 = rcc::LLT::makeInteger(7); + auto I24 = rcc::LLT::makeInteger(24); + if (!I7 || !I24) + return; + auto Compare = createCompareFunction(Context, *Target, rcc::GenericOpcode::G_ICMP, *I7, false, + rcc::IntegerPredicate::SignedLessThan, + rcc::FloatingPredicate::AlwaysFalse); + if (Compare && Compare->firstBlock()->firstInstruction() != nullptr) { + rcc::MachineInstr *Instruction = Compare->firstBlock()->firstInstruction(); + auto Action = rcc::riscv::legalizerInfo().classify(*Instruction, *Compare, *Target); + auto Selection = rcc::riscv::instructionSelector().select(*Instruction, *Compare, *Target); + Context.expect(Action && *Action == rcc::LegalizationAction::Legal && Selection, + "signed i7 comparisons must normalize and select"); + Context.expect(Selection && validatesSelectedFunction(*Target, *Compare), + "selected i7 comparison must satisfy target descriptors"); + } + + auto Extension = createUnaryFunction(Context, *Target, rcc::GenericOpcode::G_SEXT, *I24, *I7); + if (Extension && Extension->firstBlock()->firstInstruction() != nullptr) { + rcc::MachineInstr *Instruction = Extension->firstBlock()->firstInstruction(); + auto Action = rcc::riscv::legalizerInfo().classify(*Instruction, *Extension, *Target); + auto Selection = rcc::riscv::instructionSelector().select(*Instruction, *Extension, *Target); + Context.expect(Action && *Action == rcc::LegalizationAction::Legal && Selection, + "i7-to-i24 sign extension must select through a 32-bit container"); + Context.expect(Selection && validatesSelectedFunction(*Target, *Extension), + "selected narrow extension must satisfy target descriptors"); + } + + auto UnsupportedMulHigh = createBinary(Context, *Target, rcc::GenericOpcode::G_SMULH, 16); + if (UnsupportedMulHigh.Function && UnsupportedMulHigh.Instruction != nullptr) { + auto Action = rcc::riscv::legalizerInfo().classify(*UnsupportedMulHigh.Instruction, + *UnsupportedMulHigh.Function, *Target); + Context.expect(Action && *Action == rcc::LegalizationAction::Unsupported, + "non-native multiply-high widths must be explicitly unsupported"); + } +} + +void testScalarSelection(TestContext &Context) { + auto Target = rcc::riscv::createTargetDescription(); + Context.expect(Target.has_value(), "RV64GC target must be available for scalar selection"); + if (!Target) + return; + + auto I32 = rcc::LLT::makeInteger(32); + auto I64 = rcc::LLT::makeInteger(64); + auto F32 = rcc::LLT::makeFloat(rcc::FloatSemantics::IEEESingle); + auto F64 = rcc::LLT::makeFloat(rcc::FloatSemantics::IEEEDouble); + auto I1 = rcc::LLT::makeInteger(1); + Context.expect(I32 && I64 && F32 && F64 && I1, "scalar test types must be constructible"); + if (!I32 || !I64 || !F32 || !F64 || !I1) + return; + + auto AnyExt = createUnaryFunction(Context, *Target, rcc::GenericOpcode::G_ANYEXT, *I64, *I32); + auto Sext = createUnaryFunction(Context, *Target, rcc::GenericOpcode::G_SEXT, *I64, *I32); + auto Zext = createUnaryFunction(Context, *Target, rcc::GenericOpcode::G_ZEXT, *I64, *I32); + auto Trunc = createUnaryFunction(Context, *Target, rcc::GenericOpcode::G_TRUNC, *I32, *I64); + for (const auto &Entry : {std::pair{AnyExt.get(), rcc::GenericOpcode::G_ANYEXT}, + std::pair{Sext.get(), rcc::GenericOpcode::G_SEXT}, + std::pair{Zext.get(), rcc::GenericOpcode::G_ZEXT}, + std::pair{Trunc.get(), rcc::GenericOpcode::G_TRUNC}}) { + Context.expect(Entry.first != nullptr, "integer extension test instruction must exist"); + if (Entry.first == nullptr) + continue; + auto *Instruction = Entry.first->firstBlock()->firstInstruction(); + auto Selection = rcc::riscv::instructionSelector().select(*Instruction, *Entry.first, *Target); + const std::string_view Name = Entry.second == rcc::GenericOpcode::G_ANYEXT ? "anyext" + : Entry.second == rcc::GenericOpcode::G_SEXT ? "sext" + : Entry.second == rcc::GenericOpcode::G_ZEXT ? "zext" + : "trunc"; + Context.expect(Selection.has_value(), + std::string("RV64GC ") + std::string(Name) + " must select"); + if (!Selection) + continue; + if (Entry.second == rcc::GenericOpcode::G_ANYEXT || Entry.second == rcc::GenericOpcode::G_TRUNC) + Context.expect(Instruction->opcode() == rcc::opcode(rcc::GenericOpcode::COPY), + "anyext/trunc must select COPY"); + else + Context.expect(!rcc::isGenericOpcode(Instruction->opcode()), + "integer extension must not remain generic"); + } + if (Zext) { + Context.expect(Zext->allInstructions().size() == 2, + "zero extension must emit a two-instruction RV64 sequence"); + if (Zext->allInstructions().size() == 2) { + auto It = Zext->allInstructions().begin(); + Context.expect((*It)->opcode() == rcc::Opcode::fromValue(static_cast( + rcc::riscv::InstructionOpcode::SLLI)), + "zero extension must shift left first"); + ++It; + Context.expect((*It)->opcode() == rcc::Opcode::fromValue(static_cast( + rcc::riscv::InstructionOpcode::SRLI)), + "zero extension must shift right logically second"); + } + } + + auto FPExt = createUnaryFunction(Context, *Target, rcc::GenericOpcode::G_FPEXT, *F64, *F32); + auto FPTrunc = createUnaryFunction(Context, *Target, rcc::GenericOpcode::G_FPTRUNC, *F32, *F64); + auto FPToSI = createUnaryFunction(Context, *Target, rcc::GenericOpcode::G_FPTOSI, *I64, *F32); + auto SIToFP = createUnaryFunction(Context, *Target, rcc::GenericOpcode::G_SITOFP, *F64, *I32); + auto Bitcast = createUnaryFunction(Context, *Target, rcc::GenericOpcode::G_BITCAST, *I32, *F32); + if (FPExt) { + auto *Instruction = FPExt->firstBlock()->firstInstruction(); + Context.expect( + rcc::riscv::instructionSelector().select(*Instruction, *FPExt, *Target).has_value(), + "fpext must select"); + Context.expect(Instruction->opcode() == rcc::Opcode::fromValue(static_cast( + rcc::riscv::InstructionOpcode::FCVT_D_S)), + "fpext must select FCVT.D.S"); + } + if (FPTrunc) { + auto *Instruction = FPTrunc->firstBlock()->firstInstruction(); + Context.expect( + rcc::riscv::instructionSelector().select(*Instruction, *FPTrunc, *Target).has_value(), + "fptrunc must select"); + Context.expect(Instruction->opcode() == rcc::Opcode::fromValue(static_cast( + rcc::riscv::InstructionOpcode::FCVT_S_D)), + "fptrunc must select FCVT.S.D"); + } + if (FPToSI) { + auto *Instruction = FPToSI->firstBlock()->firstInstruction(); + Context.expect( + rcc::riscv::instructionSelector().select(*Instruction, *FPToSI, *Target).has_value(), + "fptosi must select"); + Context.expect(Instruction->opcode() == rcc::Opcode::fromValue(static_cast( + rcc::riscv::InstructionOpcode::FCVT_L_S)), + "fptosi must select FCVT.L.S"); + } + if (SIToFP) { + auto *Instruction = SIToFP->firstBlock()->firstInstruction(); + Context.expect( + rcc::riscv::instructionSelector().select(*Instruction, *SIToFP, *Target).has_value(), + "sitofp must select"); + Context.expect(Instruction->opcode() == rcc::Opcode::fromValue(static_cast( + rcc::riscv::InstructionOpcode::FCVT_D_W)), + "sitofp must select FCVT.D.W"); + } + if (Bitcast) { + auto *Instruction = Bitcast->firstBlock()->firstInstruction(); + Context.expect( + rcc::riscv::instructionSelector().select(*Instruction, *Bitcast, *Target).has_value(), + "float-to-integer bitcast must select"); + Context.expect(Instruction->opcode() == rcc::Opcode::fromValue(static_cast( + rcc::riscv::InstructionOpcode::FMV_X_W)), + "float-to-integer bitcast must select FMV.X.W"); + } +} + +void testComparisonsAndControl(TestContext &Context) { + auto Target = rcc::riscv::createTargetDescription(); + Context.expect(Target.has_value(), "RV64GC target must be available for comparison selection"); + if (!Target) + return; + auto I64 = rcc::LLT::makeInteger(64); + auto F32 = rcc::LLT::makeFloat(rcc::FloatSemantics::IEEESingle); + auto Pointer = rcc::LLT::makePointer(0, 64); + if (!I64 || !F32 || !Pointer) + return; + + auto Equal = + createCompareFunction(Context, *Target, rcc::GenericOpcode::G_ICMP, *I64, false, + rcc::IntegerPredicate::Equal, rcc::FloatingPredicate::AlwaysFalse); + auto LessEqual = createCompareFunction(Context, *Target, rcc::GenericOpcode::G_ICMP, *I64, false, + rcc::IntegerPredicate::SignedLessEqual, + rcc::FloatingPredicate::AlwaysFalse); + if (Equal) { + auto *Instruction = Equal->firstBlock()->firstInstruction(); + auto Selection = rcc::riscv::instructionSelector().select(*Instruction, *Equal, *Target); + Context.expect(Selection.has_value(), "integer equality must select"); + Context.expect(Equal->allInstructions().size() == 2, + "integer equality must emit XOR and zero-test instructions"); + if (Equal->allInstructions().size() == 2) { + auto It = Equal->allInstructions().begin(); + Context.expect((*It)->opcode() == rcc::Opcode::fromValue(static_cast( + rcc::riscv::InstructionOpcode::XOR)), + "integer equality must start with XOR"); + ++It; + Context.expect((*It)->opcode() == rcc::Opcode::fromValue(static_cast( + rcc::riscv::InstructionOpcode::SLTIU)), + "integer equality must finish with SLTIU"); + } + } + if (LessEqual) { + auto *Instruction = LessEqual->firstBlock()->firstInstruction(); + Context.expect( + rcc::riscv::instructionSelector().select(*Instruction, *LessEqual, *Target).has_value(), + "signed less-equal must select"); + Context.expect(LessEqual->allInstructions().size() == 2, + "signed less-equal must emit compare and inversion"); + } + + auto Ordered = + createCompareFunction(Context, *Target, rcc::GenericOpcode::G_FCMP, *F32, true, + rcc::IntegerPredicate::Equal, rcc::FloatingPredicate::OrderedEqual); + auto UnorderedEqual = + createCompareFunction(Context, *Target, rcc::GenericOpcode::G_FCMP, *F32, true, + rcc::IntegerPredicate::Equal, rcc::FloatingPredicate::UnorderedEqual); + if (Ordered) { + auto *Instruction = Ordered->firstBlock()->firstInstruction(); + Context.expect( + rcc::riscv::instructionSelector().select(*Instruction, *Ordered, *Target).has_value(), + "ordered floating equality must select"); + Context.expect(Instruction->opcode() == rcc::Opcode::fromValue(static_cast( + rcc::riscv::InstructionOpcode::FEQ_S)), + "ordered floating equality must select FEQ.S"); + } + if (UnorderedEqual) { + auto *Instruction = UnorderedEqual->firstBlock()->firstInstruction(); + Context.expect(rcc::riscv::instructionSelector() + .select(*Instruction, *UnorderedEqual, *Target) + .has_value(), + "unordered floating equality must select"); + Context.expect(UnorderedEqual->allInstructions().size() == 4, + "unordered floating equality must emit ordered compare sequence"); + } + + auto FMin = createFloatBinaryFunction(Context, *Target, rcc::GenericOpcode::G_FMINNUM, *F32); + if (FMin) { + auto *Instruction = FMin->firstBlock()->firstInstruction(); + Context.expect( + rcc::riscv::instructionSelector().select(*Instruction, *FMin, *Target).has_value(), + "floating minnum must select"); + Context.expect(Instruction->opcode() == rcc::Opcode::fromValue(static_cast( + rcc::riscv::InstructionOpcode::FMIN_S)), + "floating minnum must select FMIN.S"); + } + + auto ControlFunction = + rcc::MachineFunction::create("riscv_control", rcc::MachineFunctionStableID::fromValue(900)); + Context.expect(ControlFunction.has_value(), "control test function creation must succeed"); + if (!ControlFunction) + return; + auto Block = (*ControlFunction)->createBasicBlock(); + if (!Block || !(*ControlFunction)->appendBlock(**Block)) + return; + auto Address = (*ControlFunction)->registerInfo().createVirtualRegister(*Pointer); + if (!Address || !constrainRegister(**ControlFunction, *Address, *Pointer)) + return; + rcc::MachineIRBuilder Builder(**ControlFunction, *Target); + if (!Builder.setInsertPoint(**Block)) + return; + auto Indirect = Builder.buildInstr(rcc::opcode(rcc::GenericOpcode::G_BRINDIRECT)); + auto Fence = Builder.buildInstr(rcc::opcode(rcc::GenericOpcode::G_FENCE)); + auto Trap = Builder.buildInstr(rcc::opcode(rcc::GenericOpcode::G_TRAP)); + if (!Indirect || !Fence || !Trap || !Indirect->addUse(*Address) || !Indirect->build() || + !Fence->addImmediate(7) || !Fence->addImmediate(0) || !Fence->build() || !Trap->build()) + return; + auto *IndirectInstruction = (*ControlFunction)->firstBlock()->firstInstruction(); + Context.expect(rcc::riscv::instructionSelector() + .select(*IndirectInstruction, **ControlFunction, *Target) + .has_value(), + "indirect branch must select"); + Context.expect(IndirectInstruction->opcode() == rcc::Opcode::fromValue(static_cast( + rcc::riscv::InstructionOpcode::JALR)), + "indirect branch must select JALR"); + auto *FenceInstruction = IndirectInstruction->next(); + Context.expect(rcc::riscv::instructionSelector() + .select(*FenceInstruction, **ControlFunction, *Target) + .has_value(), + "fence must select"); + Context.expect(FenceInstruction->opcode() == rcc::Opcode::fromValue(static_cast( + rcc::riscv::InstructionOpcode::FENCE)), + "fence must select FENCE"); + auto *TrapInstruction = FenceInstruction->next(); + Context.expect(rcc::riscv::instructionSelector() + .select(*TrapInstruction, **ControlFunction, *Target) + .has_value(), + "trap must select"); + Context.expect(TrapInstruction->opcode() == rcc::Opcode::fromValue(static_cast( + rcc::riscv::InstructionOpcode::EBREAK)), + "trap must select EBREAK"); +} + +void testCompositeScalarSelection(TestContext &Context) { + auto Target = rcc::riscv::createTargetDescription(); + Context.expect(Target.has_value(), "RV64GC target must be available for composite selection"); + if (!Target) + return; + auto I1 = rcc::LLT::makeInteger(1); + auto I32 = rcc::LLT::makeInteger(32); + auto I64 = rcc::LLT::makeInteger(64); + auto F64 = rcc::LLT::makeFloat(rcc::FloatSemantics::IEEEDouble); + if (!I1 || !I32 || !I64 || !F64) + return; + + auto DivRem = createRegisterOnlyFunction(Context, *Target, rcc::GenericOpcode::G_SDIVREM, + {*I64, *I64}, {*I64, *I64}); + auto MultiplyHigh = createRegisterOnlyFunction(Context, *Target, rcc::GenericOpcode::G_UMULH, + {*I32}, {*I32, *I32}); + auto Minimum = createRegisterOnlyFunction(Context, *Target, rcc::GenericOpcode::G_SMIN, {*I64}, + {*I64, *I64}); + auto Absolute = + createRegisterOnlyFunction(Context, *Target, rcc::GenericOpcode::G_ABS, {*I32}, {*I32}); + auto IntegerSelect = createRegisterOnlyFunction(Context, *Target, rcc::GenericOpcode::G_SELECT, + {*I64}, {*I1, *I64, *I64}); + auto FloatSelect = createRegisterOnlyFunction(Context, *Target, rcc::GenericOpcode::G_SELECT, + {*F64}, {*I1, *F64, *F64}); + auto CopySign = createRegisterOnlyFunction(Context, *Target, rcc::GenericOpcode::G_FCOPYSIGN, + {*F64}, {*F64, *F64}); + const std::array Functions{DivRem.get(), MultiplyHigh.get(), Minimum.get(), Absolute.get(), + IntegerSelect.get(), FloatSelect.get(), CopySign.get()}; + for (rcc::MachineFunction *Function : Functions) { + Context.expect(Function != nullptr, "composite scalar fixture must be complete"); + if (Function == nullptr) + continue; + rcc::MachineInstr *Instruction = Function->firstBlock()->firstInstruction(); + auto Classification = rcc::riscv::legalizerInfo().classify(*Instruction, *Function, *Target); + Context.expect(Classification && *Classification == rcc::LegalizationAction::Legal, + "composite scalar operation must be legal on RV64GC"); + auto Selection = rcc::riscv::instructionSelector().select(*Instruction, *Function, *Target); + Context.expect(Selection.has_value(), "composite scalar operation must select on RV64GC"); + Context.expect(Selection && validatesSelectedFunction(*Target, *Function), + "composite scalar selection must satisfy all target descriptors"); + } + if (DivRem) { + const auto Instructions = DivRem->allInstructions(); + Context.expect(Instructions.size() == 2, "G_SDIVREM must emit separate quotient and remainder"); + if (Instructions.size() == 2) { + auto It = Instructions.begin(); + Context.expect((*It)->opcode() == rcc::Opcode::fromValue(static_cast( + rcc::riscv::InstructionOpcode::DIV)), + "G_SDIVREM quotient must use DIV"); + ++It; + Context.expect((*It)->opcode() == rcc::Opcode::fromValue(static_cast( + rcc::riscv::InstructionOpcode::REM)), + "G_SDIVREM remainder must use REM"); + } + } + if (MultiplyHigh) { + Context.expect(MultiplyHigh->allInstructions().size() >= 4, + "i32 G_UMULH must explicitly zero-extend before multiplication"); + } + if (CopySign) { + Context.expect(CopySign->firstBlock()->firstInstruction()->opcode() == + rcc::Opcode::fromValue( + static_cast(rcc::riscv::InstructionOpcode::FSGNJ_D)), + "G_FCOPYSIGN must select FSGNJ.D"); + } +} + +void testOverflowSelection(TestContext &Context) { + auto Target = rcc::riscv::createTargetDescription(); + if (!Target) + return; + auto I1 = rcc::LLT::makeInteger(1); + auto I32 = rcc::LLT::makeInteger(32); + auto I64 = rcc::LLT::makeInteger(64); + if (!I1 || !I32 || !I64) + return; + const std::array Opcodes{rcc::GenericOpcode::G_UADDO, rcc::GenericOpcode::G_USUBO, + rcc::GenericOpcode::G_SADDO, rcc::GenericOpcode::G_SSUBO, + rcc::GenericOpcode::G_UMULO, rcc::GenericOpcode::G_SMULO}; + for (rcc::GenericOpcode Generic : Opcodes) { + const rcc::LLT Type = Generic == rcc::GenericOpcode::G_UMULO ? *I32 : *I64; + auto Function = + createRegisterOnlyFunction(Context, *Target, Generic, {Type, *I1}, {Type, Type}); + Context.expect(Function != nullptr, "overflow fixture must be complete"); + if (!Function) + continue; + rcc::MachineInstr *Instruction = Function->firstBlock()->firstInstruction(); + auto Selection = rcc::riscv::instructionSelector().select(*Instruction, *Function, *Target); + Context.expect(Selection.has_value(), "RV64GC overflow operation must select"); + Context.expect(Selection && validatesSelectedFunction(*Target, *Function), + "overflow sequence must satisfy all target descriptors"); + Context.expect(Function->allInstructions().size() >= 2, + "overflow operation must materialize its status result"); + } + const std::array ExtendedOpcodes{rcc::GenericOpcode::G_UADDE, rcc::GenericOpcode::G_USUBE, + rcc::GenericOpcode::G_SADDE, rcc::GenericOpcode::G_SSUBE}; + for (rcc::GenericOpcode Generic : ExtendedOpcodes) { + auto Function = + createRegisterOnlyFunction(Context, *Target, Generic, {*I64, *I1}, {*I64, *I64, *I1}); + Context.expect(Function != nullptr, "extended overflow fixture must be complete"); + if (!Function) + continue; + rcc::MachineInstr *Instruction = Function->firstBlock()->firstInstruction(); + auto Selection = rcc::riscv::instructionSelector().select(*Instruction, *Function, *Target); + Context.expect(Selection.has_value(), "extended RV64GC overflow operation must select"); + Context.expect(Selection && validatesSelectedFunction(*Target, *Function), + "extended overflow sequence must satisfy all target descriptors"); + } +} + +void testAtomicSelection(TestContext &Context) { + auto Target = rcc::riscv::createTargetDescription(); + if (!Target) + return; + auto I64 = rcc::LLT::makeInteger(64); + auto Pointer = rcc::LLT::makePointer(0, 64); + auto Size = rcc::MemorySize::fixed(8); + auto Align = rcc::Alignment::create(8); + if (!I64 || !Pointer || !Size || !Align) + return; + + const auto MakeRMW = [&](rcc::GenericOpcode Generic, std::uint64_t ID) -> InstructionFixture { + auto Function = + rcc::MachineFunction::create("riscv_atomic", rcc::MachineFunctionStableID::fromValue(ID)); + if (!Function) + return {}; + auto Block = (*Function)->createBasicBlock(); + if (!Block || !(*Function)->appendBlock(**Block)) + return {}; + auto Result = (*Function)->registerInfo().createVirtualRegister(*I64); + auto Address = (*Function)->registerInfo().createVirtualRegister(*Pointer); + auto Value = (*Function)->registerInfo().createVirtualRegister(*I64); + if (!Result || !Address || !Value) + return {}; + rcc::MachineMemOperandDesc Description; + Description.Accesses = {rcc::MemoryAccessFlag::Load, rcc::MemoryAccessFlag::Store}; + Description.AtomicKind = rcc::AtomicOperation::ReadModifyWrite; + Description.SuccessOrdering = rcc::AtomicOrdering::SequentiallyConsistent; + Description.Size = *Size; + Description.Align = *Align; + Description.Base = rcc::MemoryBase::stack(); + Description.Offset = 32; + auto Memory = (*Function)->createMemOperand(std::move(Description)); + if (!Memory) + return {}; + rcc::MachineIRBuilder Builder(**Function, *Target); + if (!Builder.setInsertPoint(**Block)) + return {}; + auto Pending = Builder.buildInstr(rcc::opcode(Generic)); + if (!Pending || !Pending->addDef(*Result) || !Pending->addUse(*Address) || + !Pending->addUse(*Value) || !Pending->addMemOperand(**Memory)) { + return {}; + } + auto Built = Pending->build(); + if (!Built || !constrainVirtualRegisters(**Function)) + return {}; + return {std::move(*Function), *Built, *Result}; + }; + + auto Add = MakeRMW(rcc::GenericOpcode::G_ATOMICRMW_ADD, 1300); + auto Sub = MakeRMW(rcc::GenericOpcode::G_ATOMICRMW_SUB, 1301); + auto Nand = MakeRMW(rcc::GenericOpcode::G_ATOMICRMW_NAND, 1302); + Context.expect(Add.Instruction != nullptr && Sub.Instruction != nullptr && + Nand.Instruction != nullptr, + "atomic RMW fixtures must be complete"); + if (Add.Instruction != nullptr) { + auto Selection = + rcc::riscv::instructionSelector().select(*Add.Instruction, *Add.Function, *Target); + Context.expect(Selection.has_value() && Add.Instruction->opcode() == + rcc::Opcode::fromValue(static_cast( + rcc::riscv::InstructionOpcode::AMOADD_D)), + "G_ATOMICRMW_ADD must select AMOADD.D"); + Context.expect(Selection && validatesSelectedFunction(*Target, *Add.Function), + "selected AMOADD.D must satisfy its descriptor and MMO contract"); + } + if (Sub.Instruction != nullptr) { + auto Selection = + rcc::riscv::instructionSelector().select(*Sub.Instruction, *Sub.Function, *Target); + Context.expect(Selection.has_value() && Sub.Function->allInstructions().size() == 2, + "G_ATOMICRMW_SUB must negate its operand before AMOADD.D"); + Context.expect(Selection && validatesSelectedFunction(*Target, *Sub.Function), + "selected atomic subtraction must satisfy target descriptors"); + } + if (Nand.Instruction != nullptr) { + auto Classification = + rcc::riscv::legalizerInfo().classify(*Nand.Instruction, *Nand.Function, *Target); + Context.expect( + Classification && *Classification == rcc::LegalizationAction::Unsupported, + "an atomic operation without a single-instruction RV64GC mapping must be explicit"); + auto Match = + rcc::riscv::instructionSelector().canSelect(*Nand.Instruction, *Nand.Function, *Target); + Context.expect(!Match && Match.error().code() == rcc::ErrorCode::Unsupported, + "unsupported atomic RMW must fail before mutating MachineIR"); + Context.expect(!Match && Match.error().message().find("G_ATOMICRMW_NAND") != std::string::npos, + "unsupported atomic diagnostics must name the rejected opcode"); + } + + const auto MakeAtomicAccess = [&](bool Load, std::uint64_t ID) -> InstructionFixture { + auto Function = rcc::MachineFunction::create(Load ? "riscv_atomic_load" : "riscv_atomic_store", + rcc::MachineFunctionStableID::fromValue(ID)); + if (!Function) + return {}; + auto Block = (*Function)->createBasicBlock(); + if (!Block || !(*Function)->appendBlock(**Block)) + return {}; + auto Value = (*Function)->registerInfo().createVirtualRegister(*I64); + auto Address = (*Function)->registerInfo().createVirtualRegister(*Pointer); + if (!Value || !Address) + return {}; + rcc::MachineMemOperandDesc Description; + Description.Accesses = {Load ? rcc::MemoryAccessFlag::Load : rcc::MemoryAccessFlag::Store}; + Description.AtomicKind = Load ? rcc::AtomicOperation::Load : rcc::AtomicOperation::Store; + Description.SuccessOrdering = + Load ? rcc::AtomicOrdering::Acquire : rcc::AtomicOrdering::Release; + Description.Size = *Size; + Description.Align = *Align; + Description.Base = rcc::MemoryBase::stack(); + auto Memory = (*Function)->createMemOperand(std::move(Description)); + if (!Memory) + return {}; + rcc::MachineIRBuilder Builder(**Function, *Target); + if (!Builder.setInsertPoint(**Block)) + return {}; + auto Pending = Builder.buildInstr( + rcc::opcode(Load ? rcc::GenericOpcode::G_LOAD : rcc::GenericOpcode::G_STORE)); + if (!Pending) + return {}; + if (Load) { + if (!Pending->addDef(*Value) || !Pending->addUse(*Address)) + return {}; + } else if (!Pending->addUse(*Value) || !Pending->addUse(*Address)) { + return {}; + } + if (!Pending->addMemOperand(**Memory)) + return {}; + auto Built = Pending->build(); + if (!Built || !constrainVirtualRegisters(**Function)) + return {}; + return {std::move(*Function), *Built, *Value}; + }; + auto AtomicLoad = MakeAtomicAccess(true, 1303); + auto AtomicStore = MakeAtomicAccess(false, 1304); + if (AtomicLoad.Instruction != nullptr) { + auto Selection = rcc::riscv::instructionSelector().select(*AtomicLoad.Instruction, + *AtomicLoad.Function, *Target); + Context.expect(Selection && AtomicLoad.Instruction->opcode() == + rcc::Opcode::fromValue(static_cast( + rcc::riscv::InstructionOpcode::LR_D)), + "atomic G_LOAD must select LR.D"); + Context.expect(Selection && validatesSelectedFunction(*Target, *AtomicLoad.Function), + "selected atomic load must satisfy target descriptors"); + } + if (AtomicStore.Instruction != nullptr) { + auto Selection = rcc::riscv::instructionSelector().select(*AtomicStore.Instruction, + *AtomicStore.Function, *Target); + Context.expect(Selection && AtomicStore.Instruction->opcode() == + rcc::Opcode::fromValue(static_cast( + rcc::riscv::InstructionOpcode::AMOSWAP_D)), + "atomic G_STORE must select AMOSWAP.D with a discarded old value"); + Context.expect(Selection && validatesSelectedFunction(*Target, *AtomicStore.Function), + "selected atomic store must satisfy target descriptors"); + } +} + +void testUnsupportedTypes(TestContext &Context) { + auto Target = rcc::riscv::createTargetDescription(); + Context.expect(Target.has_value(), "RV64GC target must be available for type diagnostics"); + if (!Target) + return; + auto WideInteger = createBinary(Context, *Target, rcc::GenericOpcode::G_ADD, 128); + if (WideInteger.Function && WideInteger.Instruction != nullptr) { + auto Selection = rcc::riscv::instructionSelector().canSelect(*WideInteger.Instruction, + *WideInteger.Function, *Target); + Context.expect(!Selection && Selection.error().code() == rcc::ErrorCode::Unsupported, + "i128 arithmetic must produce an Unsupported diagnostic"); + Context.expect(!Selection && Selection.error().message().find("i128") != std::string::npos, + "unsupported type diagnostics must name the rejected LLT"); + } + auto Function = rcc::MachineFunction::create("unsupported_type", + rcc::MachineFunctionStableID::fromValue(100)); + if (!Function) + return; + auto Block = (*Function)->createBasicBlock(); + if (!Block || !(*Function)->appendBlock(**Block)) + return; + auto F64 = rcc::LLT::makeFloat(rcc::FloatSemantics::IEEEDouble); + if (!F64) + return; + auto Result = (*Function)->registerInfo().createVirtualRegister(*F64); + auto Source = (*Function)->registerInfo().createVirtualRegister(*F64); + if (!Result || !Source) + return; + rcc::MachineIRBuilder Builder(**Function, *Target); + if (!Builder.setInsertPoint(**Block)) + return; + auto Pending = Builder.buildInstr(rcc::opcode(rcc::GenericOpcode::G_FEXP)); + if (!Pending || !Pending->addDef(*Result) || !Pending->addUse(*Source)) + return; + if (!Pending->build()) + return; + const rcc::MachineInstr &Instruction = **Function->get()->allInstructions().begin(); + auto Action = rcc::riscv::legalizerInfo().classify(Instruction, **Function, *Target); + Context.expect(Action && *Action == rcc::LegalizationAction::Unsupported, + "unsupported floating operations must remain explicitly diagnosed"); + auto Selection = rcc::riscv::instructionSelector().canSelect(Instruction, **Function, *Target); + Context.expect(!Selection && Selection.error().code() == rcc::ErrorCode::Unsupported, + "selector pre-matching must reject unsupported floating operations"); + Context.expect(!Selection && Selection.error().message().find("G_FEXP") != std::string::npos, + "unsupported selection diagnostics must name the rejected opcode"); +} + +void testMemoryOffsetSelection(TestContext &Context) { + auto Target = rcc::riscv::createTargetDescription(); + Context.expect(Target.has_value(), "RV64GC target must be available for memory selection"); + if (!Target) + return; + auto Function = rcc::MachineFunction::create("riscv_memory_offset", + rcc::MachineFunctionStableID::fromValue(101)); + if (!Function) + return; + auto Block = (*Function)->createBasicBlock(); + if (!Block || !(*Function)->appendBlock(**Block)) + return; + auto I64 = rcc::LLT::makeInteger(64); + auto Pointer = rcc::LLT::makePointer(0, 64); + if (!I64 || !Pointer) + return; + auto Result = (*Function)->registerInfo().createVirtualRegister(*I64); + auto Address = (*Function)->registerInfo().createVirtualRegister(*Pointer); + if (!Result || !Address) + return; + rcc::MachineIRBuilder Builder(**Function, *Target); + if (!Builder.setInsertPoint(**Block)) + return; + auto Pending = Builder.buildInstr(rcc::opcode(rcc::GenericOpcode::G_LOAD)); + if (!Pending || !Pending->addDef(*Result) || !Pending->addUse(*Address)) + return; + auto Size = rcc::MemorySize::fixed(8); + auto Align = rcc::Alignment::create(8); + if (!Size || !Align) + return; + rcc::MachineMemOperandDesc Description; + Description.Accesses = {rcc::MemoryAccessFlag::Load, rcc::MemoryAccessFlag::Volatile}; + Description.Size = *Size; + Description.Align = *Align; + Description.Base = rcc::MemoryBase::stack(); + Description.Offset = 24; + auto Memory = (*Function)->createMemOperand(std::move(Description)); + if (!Memory || !Pending->addMemOperand(**Memory) || !Pending->build() || + !constrainVirtualRegisters(**Function)) + return; + rcc::MachineInstr *Instruction = (*Function)->firstBlock()->firstInstruction(); + Context.expect(Instruction != nullptr, "generic load must remain attached to its block"); + if (Instruction == nullptr) + return; + auto Action = rcc::riscv::legalizerInfo().classify(*Instruction, **Function, *Target); + Context.expect(Action && *Action == rcc::LegalizationAction::Legal, + "RV64GC legalizer must accept volatile scalar loads"); + auto Selection = rcc::riscv::instructionSelector().select(*Instruction, **Function, *Target); + Context.expect(Selection.has_value(), "RV64GC selector must select a volatile scalar load"); + if (!Selection) + return; + Context.expect(Instruction->opcode() == rcc::Opcode::fromValue(static_cast( + rcc::riscv::InstructionOpcode::LD)), + "i64 load with an offset must select LD"); + const rcc::MachineOperand *OffsetOperand = Instruction->operandAt(2); + const std::int64_t *Offset = + OffsetOperand == nullptr ? nullptr : OffsetOperand->getIf(); + Context.expect(Offset != nullptr && *Offset == 0, + "selected load must not reinterpret MMO provenance as an address immediate"); + Context.expect(Instruction->memOperands().size() == 1 && + Instruction->memOperands().front()->description().Offset == 24, + "selected load must preserve its MMO provenance offset"); + Context.expect(validatesSelectedFunction(*Target, **Function), + "selected volatile load must satisfy the target descriptor"); +} + +} // namespace + +int main() { + TestContext Context; + testClassification(Context); + testNarrowIntegerSelection(Context); + testScalarSelection(Context); + testComparisonsAndControl(Context); + testCompositeScalarSelection(Context); + testOverflowSelection(Context); + testAtomicSelection(Context); + testUnsupportedTypes(Context); + testMemoryOffsetSelection(Context); + return Context.result(); +} -- cgit v1.2.3